diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 96e5c2a274..7393f389fc 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -4,6 +4,7 @@ arpabet backfill canonicalized + canonicalizing cashapp cheermote cidrs diff --git a/Cargo.lock b/Cargo.lock index 9968a051b4..6061ba0a59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1043,7 +1043,6 @@ version = "0.0.1" dependencies = [ "dotenv", "easyenv", - "envvar", "errors", "hostname", "log", @@ -1677,6 +1676,17 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "dashmap" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", + "parking_lot 0.12.1", +] + [[package]] name = "data-encoding" version = "2.3.3" @@ -1948,7 +1958,6 @@ dependencies = [ "dotenv", "easyenv", "enums", - "envvar", "errors", "filesys", "google_drive_common", @@ -2024,11 +2033,12 @@ dependencies = [ [[package]] name = "easyenv" -version = "0.0.9" +version = "0.1.0" dependencies = [ "dotenv", "env_logger 0.10.0", "log", + "storyteller_root", ] [[package]] @@ -2106,16 +2116,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "envvar" -version = "0.0.1" -dependencies = [ - "dotenv", - "easyenv", - "errors", - "log", -] - [[package]] name = "errno" version = "0.3.1" @@ -2958,7 +2958,6 @@ dependencies = [ "dotenv", "easyenv", "enums", - "envvar", "errors", "filesys", "hashing", @@ -3822,11 +3821,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.1.18", + "hermit-abi 0.3.1", "libc", ] @@ -5117,13 +5116,37 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap", + "futures 0.3.26", + "lazy_static", + "log", + "parking_lot 0.12.1", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.26", +] + [[package]] name = "server_environment" version = "0.0.1" dependencies = [ "dotenv", "easyenv", - "envvar", "errors", "log", ] @@ -5535,7 +5558,6 @@ dependencies = [ "dotenv", "easyenv", "enums", - "envvar", "errors", "futures 0.1.31", "futures 0.3.26", @@ -5601,6 +5623,18 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "storyteller_root" +version = "0.0.1" +dependencies = [ + "dotenv", + "env_logger 0.10.0", + "errors", + "log", + "serial_test", + "tempdir", +] + [[package]] name = "string_cache" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index 2dd2a99afc..6781c87208 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,6 @@ members = [ "crates/lib/deprecated/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers "crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve. - "crates/lib/envvar", "crates/lib/errors", "crates/lib/files/archives", "crates/lib/files/filesys", @@ -44,6 +43,7 @@ members = [ "crates/lib/redis_common", "crates/lib/server_environment", "crates/lib/sqlx_mysql_helpers", + "crates/lib/storyteller_root", "crates/lib/subprocess_common", "crates/lib/testing", "crates/lib/tts_common", diff --git a/_sql/migrations/2023-05-23-090105_password_resets/up.sql b/_sql/migrations/2023-05-23-090105_password_resets/up.sql index ce3817f8c9..ea51f69305 100644 --- a/_sql/migrations/2023-05-23-090105_password_resets/up.sql +++ b/_sql/migrations/2023-05-23-090105_password_resets/up.sql @@ -2,6 +2,20 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- Per-user password resets. +-- Password resets are valid until any of the following are true: +-- +-- 1) database NOW() > password_rests.expires_at +-- 2) password_rests.is_redeemed == TRUE +-- 3) users.password_version > password_resets.current_password_version +-- +-- When creating a new password reset, send the user an email with the `secret_key` in the URL. +-- The user will have to supply their username and/or email address, plus the secret key to +-- redeem the password reset. +-- +-- Also when creating the password hash, be sure to set `expires_at` to be "NOW() + 3 hours", or +-- some other small, reasonable time frame. +-- CREATE TABLE password_resets ( -- Not used for anything except replication. id BIGINT(20) NOT NULL AUTO_INCREMENT, @@ -22,7 +36,7 @@ CREATE TABLE password_resets ( -- reset is no longer valid. current_password_version INT NOT NULL DEFAULT 0, - -- Username is a lookup key; display_name allows the user to add custom case. + -- Whether the password reset has been consumed. is_redeemed BOOLEAN NOT NULL DEFAULT false, -- For abuse tracking. @@ -44,6 +58,9 @@ CREATE TABLE password_resets ( -- INDICES -- PRIMARY KEY (id), UNIQUE KEY (token), + + UNIQUE KEY (user_token, secret_key), + KEY fk_user_token (user_token), KEY index_secret_key (secret_key), KEY index_expires_at (expires_at) diff --git a/_sql/migrations/2023-10-18-061204_alter_password_resets_add_unique_index/down.sql b/_sql/migrations/2023-10-18-061204_alter_password_resets_add_unique_index/down.sql new file mode 100644 index 0000000000..1699f599f8 --- /dev/null +++ b/_sql/migrations/2023-10-18-061204_alter_password_resets_add_unique_index/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE password_resets DROP INDEX unique_user_token_and_secret_key; diff --git a/_sql/migrations/2023-10-18-061204_alter_password_resets_add_unique_index/up.sql b/_sql/migrations/2023-10-18-061204_alter_password_resets_add_unique_index/up.sql new file mode 100644 index 0000000000..79a7a68c34 --- /dev/null +++ b/_sql/migrations/2023-10-18-061204_alter_password_resets_add_unique_index/up.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE password_resets + ADD UNIQUE INDEX unique_user_token_and_secret_key (user_token, secret_key); diff --git a/crates/lib/bootstrap/Cargo.toml b/crates/lib/bootstrap/Cargo.toml index 894ea268f4..f09c0b6bbf 100644 --- a/crates/lib/bootstrap/Cargo.toml +++ b/crates/lib/bootstrap/Cargo.toml @@ -16,7 +16,6 @@ path = "src/lib.rs" [dependencies] # Internal easyenv = { path = "../easyenv" } -envvar = { path = "../envvar" } errors = { path = "../errors" } server_environment = { path = "../server_environment" } diff --git a/crates/lib/bootstrap/src/bootstrap.rs b/crates/lib/bootstrap/src/bootstrap.rs index 3c8f64a3b5..bf8d74d4c4 100644 --- a/crates/lib/bootstrap/src/bootstrap.rs +++ b/crates/lib/bootstrap/src/bootstrap.rs @@ -120,7 +120,7 @@ fn load_env_config_files>(server_environment: ServerEnvironment, for env_config_file in env_config_file_names.into_iter() { info!("Loading environment variable config file: {}", &env_config_file); - let was_read = envvar::maybe_read_from_filename_and_paths( + let was_read = easyenv::maybe_read_env_config_from_filename_and_paths( &env_config_file, args.config_search_directories)?; diff --git a/crates/lib/easyenv/Cargo.toml b/crates/lib/easyenv/Cargo.toml index aa82f6e096..3746f151bf 100644 --- a/crates/lib/easyenv/Cargo.toml +++ b/crates/lib/easyenv/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "easyenv" edition = "2018" -version = "0.0.9" +version = "0.1.0" authors = [ "Brandon Thomas ", "Brandon Thomas ", @@ -13,6 +13,10 @@ name = "easyenv" path = "src/lib.rs" [dependencies] +# Internal +storyteller_root = { path = "../storyteller_root" } + +# External dotenv = "0.15.0" env_logger = "0.10.0" log = "0.4.14" diff --git a/crates/lib/easyenv/src/error.rs b/crates/lib/easyenv/src/error.rs new file mode 100644 index 0000000000..591f8b6b10 --- /dev/null +++ b/crates/lib/easyenv/src/error.rs @@ -0,0 +1,62 @@ +use std::error::Error; +use std::fmt; +use std::fmt::{Display, Formatter}; + +/// Errors with reading and parsing env variables. +#[derive(Debug)] +pub enum EnvError { + /// The environment variable value is not unicode. + NotUnicode, + /// Problem parsing the env variable as the desired type. + ParseError { + /// Explanation of the parsing failure. + reason: String + }, + /// The required environment variable wasn't present. + RequiredNotPresent, +} + +impl Display for EnvError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let reason = match self { + EnvError::NotUnicode => "EnvError::NotUnicode", + EnvError::ParseError { .. } => "EnvError::ParseError", + EnvError::RequiredNotPresent => "EnvError::RequiredNotPresent", + }; + write!(f, "{:?}", reason) + } +} + +impl Error for EnvError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} + +/// Errors while initializing the library +#[derive(Debug)] +pub enum InitError { + /// std::io error + IoError, + /// dotenv couldn't read a file + DotEnvError, + /// No env config file was found and read + NoConfigFileFoundError, +} + +impl Display for InitError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let reason = match self { + InitError::IoError => "InitError::IoError", + InitError::DotEnvError => "InitError::DotEnvError", + InitError::NoConfigFileFoundError => "InitError::NoConfigFileFoundError", + }; + write!(f, "{:?}", reason) + } +} + +impl Error for InitError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} diff --git a/crates/lib/easyenv/src/init.rs b/crates/lib/easyenv/src/init.rs new file mode 100644 index 0000000000..5c603d5c70 --- /dev/null +++ b/crates/lib/easyenv/src/init.rs @@ -0,0 +1,64 @@ +use std::path::{Path, PathBuf}; + +use log::error; + +use crate::error::InitError; +use crate::init_env_logger; + +/// Initialize dotenv (with `.env` file) and the env logger using the `RUST_LOG` variable setting. +pub fn init_all_with_default_logging(default_if_absent: Option<&str>) { + init_dotenv(); + init_env_logger(default_if_absent) +} + +/// Try loading in an environment file across many search paths (first found wins). +/// Returns an error if no file could be read or if in attempting to read a file there was an error. +pub fn read_env_config_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> Result<(), InitError> { + if do_read_env_config_from_filename_and_paths(&filename, paths)? { + Ok(()) + } else { + error!("No env file existed for filename {:?} in the search paths {:?}", + filename.as_ref(), + paths.iter().map(|p| p.as_ref().to_path_buf()).collect::>()); + Err(InitError::NoConfigFileFoundError) + } +} + +/// Try loading in an environment file across many search paths (first found wins). +/// Returns okay if no file was read, or an error if in attempting to read a file there was an error. +/// Returns boolean reflecting whether a file was read. +pub fn maybe_read_env_config_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> Result { + Ok(do_read_env_config_from_filename_and_paths(filename, paths)?) +} + +// Initialize dotenv with the default `.env` config file. +// Do not fail if the file cannot be loaded. +fn init_dotenv() { + match dotenv::dotenv() { + Ok(_) => println!("dotenv configs initialized"), + Err(e) => println!("Could not initialize dotenv: {:?}", e), + } +} + +fn do_read_env_config_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> Result { + for path in paths.iter() { + let path = path.as_ref().join(filename.as_ref()); + + if path.exists() && path.is_file() { + log::info!("Attempting to read env vars from file: {:?}", path); + let path = std::fs::canonicalize(path) + .map_err(|err| { + error!("Error canonicalizing path: {:?}", err); + InitError::IoError + })?; + dotenv::from_path(path) + .map_err(|err| { + error!("dotenv error reading config: {:?}", err); + InitError::DotEnvError + })?; + return Ok(true); + } + } + + Ok(false) +} diff --git a/crates/lib/easyenv/src/lib.rs b/crates/lib/easyenv/src/lib.rs index 3c97245930..03ac3ef4e6 100644 --- a/crates/lib/easyenv/src/lib.rs +++ b/crates/lib/easyenv/src/lib.rs @@ -1,7 +1,24 @@ // Copyright (c) 2020 Brandon Thomas -//! Very simple helper functions for environment variables and environment variable-driven -//! `env_logger` configuration. +//! easyenv +//! +//! The interface between an app and its environment variables. This library makes it easy to: +//! +//! - read in strongly-typed environment variables. +//! - load default environment variables from one or more files in an order that supports +//! intentional overriding. +//! +//! Things that might want to move to another package : +//! +//! - configure env logging +//! - build conventions around system paths such that paths imported from environment variables +//! respect pathing conventions. +//! +//! This library does not: +//! +//! - handle app-specific conventions +//! - care about servers or deployed environments (eg. "staging" vs "production") +//! #![deny(dead_code)] #![deny(missing_docs)] @@ -11,101 +28,31 @@ #![deny(unused_qualifications)] #![deny(unused_qualifications)] -use env_logger; -use std::error::Error; -use std::fmt::{Display, Debug, Formatter}; -use std::{env, fmt}; +// NB: These were exported from the root in previous versions. +// It's easy to just keep them exported here to maintain compatibility. +pub use error::EnvError; +pub use error::InitError; +pub use init::init_all_with_default_logging; +pub use init::maybe_read_env_config_from_filename_and_paths; +pub use init::read_env_config_from_filename_and_paths; +pub use logging::DEFAULT_LOG_LEVEL; +pub use logging::ENV_RUST_LOG; +pub use logging::init_env_logger; +pub use types::boolean::get_env_bool_optional; +pub use types::boolean::get_env_bool_or_default; +pub use types::boolean::get_env_bool_required; +pub use types::duration::get_env_duration_seconds_optional; +pub use types::duration::get_env_duration_seconds_or_default; +pub use types::duration::get_env_duration_seconds_required; +pub use types::num::get_env_num; +pub use types::pathbuf::get_env_pathbuf_optional; +pub use types::pathbuf::get_env_pathbuf_or_default; +pub use types::pathbuf::get_env_pathbuf_required; +pub use types::string::get_env_string_optional; +pub use types::string::get_env_string_or_default; +pub use types::string::get_env_string_required; -mod boolean; -mod duration; -mod pathbuf; -mod string; -mod num; - -/// Name of the environment variable Rust's env logger uses -pub const ENV_RUST_LOG : &str = "RUST_LOG"; - -const DEFAULT_LOG_LEVEL: &str = "info"; - -/// Errors with env variables. -#[derive(Debug)] -pub enum EnvError { - /// The environment variable value is not unicode. - NotUnicode, - /// Problem parsing the env variable as the desired type. - ParseError { - /// Explanation of the parsing failure. - reason: String - }, - /// The required environment variable wasn't present. - RequiredNotPresent, -} - -impl Display for EnvError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "EnvError::ParseError") - } -} - -impl Error for EnvError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - None - } -} - -pub use boolean::get_env_bool_optional; -pub use boolean::get_env_bool_or_default; -pub use boolean::get_env_bool_required; - -pub use duration::get_env_duration_seconds_optional; -pub use duration::get_env_duration_seconds_or_default; -pub use duration::get_env_duration_seconds_required; - -pub use num::get_env_num; - -pub use pathbuf::get_env_pathbuf_optional; -pub use pathbuf::get_env_pathbuf_or_default; -pub use pathbuf::get_env_pathbuf_required; - -pub use string::get_env_string_optional; -pub use string::get_env_string_or_default; -pub use string::get_env_string_required; - -/// Initialize Rust's env logger. -/// -/// The Rust logger reads the desired log level from the `RUST_LOG` environment variable. If this -/// isn't set, the provided default is used. If a default fallback isn't provided to this function, -/// we fall back to `"info"`. -/// -/// A more robust logging config might configure on a per-component basis, eg. -/// `"tokio_reactor=warn,hyper=info,debug"`. You can read more in the `log` and `env_logger` crate -/// docs. -pub fn init_env_logger(default_if_absent: Option<&str>) { - if env::var(ENV_RUST_LOG) - .as_ref() - .ok() - .is_none() - { - let default_log_level = default_if_absent.unwrap_or(DEFAULT_LOG_LEVEL); - println!("Setting default logging level to \"{}\", override with env var {}.", - default_log_level, ENV_RUST_LOG); - env::set_var(ENV_RUST_LOG, default_log_level); - } - - env_logger::init(); -} - -/// Initialize dotenv with the default `.env` config file. -pub fn init_dotenv() { - match dotenv::dotenv() { - Ok(_) => println!("dotenv configs initialized"), - Err(e) => println!("Could not initialize dotenv: {:?}", e), - } -} - -/// Initialize dotenv and env logger. -/// See `init_dotenv()` and `init_env_logger(Option<&str>)` for further details. -pub fn init_all_with_default_logging(default_if_absent: Option<&str>) { - init_dotenv(); - init_env_logger(default_if_absent) -} +mod error; +mod init; +mod logging; +pub (crate) mod types; diff --git a/crates/lib/easyenv/src/logging.rs b/crates/lib/easyenv/src/logging.rs new file mode 100644 index 0000000000..770823d65e --- /dev/null +++ b/crates/lib/easyenv/src/logging.rs @@ -0,0 +1,35 @@ +use std::env; + + +/// Name of the environment variable Rust's env logger uses +pub const ENV_RUST_LOG : &str = "RUST_LOG"; + +/// The default logging level +pub const DEFAULT_LOG_LEVEL: &str = "info"; + + + +/// Initialize Rust's env logger. +/// +/// The Rust logger reads the desired log level from the `RUST_LOG` environment variable. If this +/// isn't set, the provided default is used. If a default fallback isn't provided to this function, +/// we fall back to `"info"`. +/// +/// A more robust logging config might configure on a per-component basis, eg. +/// `"tokio_reactor=warn,hyper=info,debug"`. You can read more in the `log` and `env_logger` crate +/// docs. +pub fn init_env_logger(default_if_absent: Option<&str>) { + if env::var(ENV_RUST_LOG) + .as_ref() + .ok() + .is_none() + { + let default_log_level = default_if_absent.unwrap_or(DEFAULT_LOG_LEVEL); + println!("Setting default logging level to \"{}\", override with env var {}.", + default_log_level, ENV_RUST_LOG); + env::set_var(ENV_RUST_LOG, default_log_level); + } + + env_logger::init(); +} + diff --git a/crates/lib/easyenv/src/boolean.rs b/crates/lib/easyenv/src/types/boolean.rs similarity index 98% rename from crates/lib/easyenv/src/boolean.rs rename to crates/lib/easyenv/src/types/boolean.rs index 445103883b..d3f269d191 100644 --- a/crates/lib/easyenv/src/boolean.rs +++ b/crates/lib/easyenv/src/types/boolean.rs @@ -1,7 +1,9 @@ use std::env; use std::env::VarError; + use log::warn; -use crate::EnvError; + +use crate::error::EnvError; /// Get an environment variable as a bool. /// If not present or there is an error in parsing, return `None`. diff --git a/crates/lib/easyenv/src/duration.rs b/crates/lib/easyenv/src/types/duration.rs similarity index 98% rename from crates/lib/easyenv/src/duration.rs rename to crates/lib/easyenv/src/types/duration.rs index b8031a6194..01728ac3c7 100644 --- a/crates/lib/easyenv/src/duration.rs +++ b/crates/lib/easyenv/src/types/duration.rs @@ -1,8 +1,10 @@ -use crate::EnvError; -use log::warn; use std::env; use std::time::Duration; +use log::warn; + +use crate::error::EnvError; + /// Get an environment variable as a `Duration` in seconds. /// If not provided or cannot parse, return an error. pub fn get_env_duration_seconds_required(env_name: &str) -> Result { diff --git a/crates/lib/easyenv/src/types/mod.rs b/crates/lib/easyenv/src/types/mod.rs new file mode 100644 index 0000000000..5a3813d4f5 --- /dev/null +++ b/crates/lib/easyenv/src/types/mod.rs @@ -0,0 +1,5 @@ +pub (crate) mod boolean; +pub (crate) mod duration; +pub (crate) mod pathbuf; +pub (crate) mod string; +pub (crate) mod num; diff --git a/crates/lib/easyenv/src/num.rs b/crates/lib/easyenv/src/types/num.rs similarity index 96% rename from crates/lib/easyenv/src/num.rs rename to crates/lib/easyenv/src/types/num.rs index 1e164fa024..d47b63bfce 100644 --- a/crates/lib/easyenv/src/num.rs +++ b/crates/lib/easyenv/src/types/num.rs @@ -1,8 +1,10 @@ use std::env; use std::fmt::{Debug, Display}; use std::str::FromStr; + use log::{error, warn}; -use crate::EnvError; + +use crate::error::EnvError; /// Get an environment variable as a number, or fall back to the provided default if not set. /// If the env var is present but can't be parsed, an error is returned instead. diff --git a/crates/lib/easyenv/src/pathbuf.rs b/crates/lib/easyenv/src/types/pathbuf.rs similarity index 83% rename from crates/lib/easyenv/src/pathbuf.rs rename to crates/lib/easyenv/src/types/pathbuf.rs index fe6ce6f6db..d95d6833f5 100644 --- a/crates/lib/easyenv/src/pathbuf.rs +++ b/crates/lib/easyenv/src/types/pathbuf.rs @@ -1,9 +1,12 @@ -use crate::EnvError; -use log::warn; -use std::env::VarError; use std::env; +use std::env::VarError; use std::path::{Path, PathBuf}; -use std::str::FromStr; + +use log::warn; + +use storyteller_root::get_substituted_path; + +use crate::error::EnvError; /// Get an environment variable as a `PathBuf`. /// If not provided or cannot parse, return an error. @@ -54,7 +57,7 @@ pub fn get_env_pathbuf_or_default>(env_name: &str, default_value: }) } -fn get_env_pathbuf_internal(env_name: &str) -> Result, EnvError> { +pub (crate) fn get_env_pathbuf_internal(env_name: &str) -> Result, EnvError> { match env::var(env_name).as_ref() { Err(err) => match err { // TODO: EnvError enum variant for equals sign in env var name @@ -62,12 +65,10 @@ fn get_env_pathbuf_internal(env_name: &str) -> Result, EnvError> VarError::NotUnicode(_) => Err(EnvError::NotUnicode), } Ok(val) => { - match PathBuf::from_str(val) { - Ok(path) => Ok(Some(path)), - Err(_err) => Err(EnvError::ParseError { - reason: "error parsing PathBuf from value".to_string() - }), - } + // TODO(bt,2023-10-17): The error handling and type juggling under the hood is pretty gnarly + // and needs to be revisited to make the failure modes safer. + let path = get_substituted_path(val); + Ok(Some(path)) } } } diff --git a/crates/lib/easyenv/src/string.rs b/crates/lib/easyenv/src/types/string.rs similarity index 97% rename from crates/lib/easyenv/src/string.rs rename to crates/lib/easyenv/src/types/string.rs index fe43cca95b..39640a8755 100644 --- a/crates/lib/easyenv/src/string.rs +++ b/crates/lib/easyenv/src/types/string.rs @@ -1,7 +1,9 @@ -use crate::EnvError; -use log::warn; use std::env; +use log::warn; + +use crate::error::EnvError; + /// Get an environment variable as an optional `String`. pub fn get_env_string_optional(env_name: &str) -> Option { match env::var(env_name).as_ref().ok() { diff --git a/crates/lib/envvar/src/lib.rs b/crates/lib/envvar/src/lib.rs deleted file mode 100644 index 48e2d072a4..0000000000 --- a/crates/lib/envvar/src/lib.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! envvar -//! -//! The purpose of this library is to handle reading environment variables. -//! - -// Never allow these -#![forbid(private_in_public)] -#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things - -// Okay to toggle -#![forbid(unreachable_patterns)] -#![forbid(unused_imports)] -#![forbid(unused_mut)] -#![forbid(unused_variables)] - -// Always allow -#![allow(dead_code)] -#![allow(non_snake_case)] - -use std::path::{Path, PathBuf}; - -use errors::{anyhow, AnyhowResult}; - -pub fn read_from_filename>(filename: P) -> AnyhowResult { - let result = dotenv::from_filename(filename)?; - Ok(result) -} - -/// Try loading in an environment file across many search paths (first found wins). -/// Returns an error if no file could be read or if in attempting to read a file there was an error. -pub fn read_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> AnyhowResult<()> { - if do_read_from_filename_and_paths(&filename, paths)? { - Ok(()) - } else { - Err(anyhow!("No env file existed for filename {:?} in the search paths {:?}", - filename.as_ref(), - paths.iter().map(|p| p.as_ref().to_path_buf()).collect::>())) - } -} - -/// Try loading in an environment file across many search paths (first found wins). -/// Returns okay if no file was read, or an error if in attempting to read a file there was an error. -/// Returns boolean reflecting whether a file was read. -pub fn maybe_read_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> AnyhowResult { - Ok(do_read_from_filename_and_paths(filename, paths)?) -} - -pub fn do_read_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> AnyhowResult { - for path in paths.iter() { - let path = path.as_ref().join(filename.as_ref()); - - if path.exists() && path.is_file() { - log::info!("Attempting to read env vars from file: {:?}", path); - let path = std::fs::canonicalize(path)?; - dotenv::from_path(path)?; - return Ok(true); - } - } - - Ok(false) -} diff --git a/crates/lib/mysql_queries/sqlx-data.json b/crates/lib/mysql_queries/sqlx-data.json index 42dd51c4d3..a580175ea1 100644 --- a/crates/lib/mysql_queries/sqlx-data.json +++ b/crates/lib/mysql_queries/sqlx-data.json @@ -13338,6 +13338,16 @@ }, "query": "\nUPDATE tts_models\nSET\n maybe_default_pretrained_vocoder = ?,\n maybe_custom_vocoder_token = ?,\n text_pipeline_type = ?,\n title = ?,\n description_markdown = ?,\n description_rendered_html = ?,\n ietf_language_tag = ?,\n ietf_primary_language_subtag = ?,\n creator_set_visibility = ?,\n maybe_mod_user_token = ?,\n version = version + 1\nWHERE token = ?\nLIMIT 1\n " }, + "b4308b1aae016b0f2559c88c90008c35b20fcf14cf8a37585af26958be62a398": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 7 + } + }, + "query": "\n INSERT INTO zs_voice_dataset_samples SET\n\n token = ?,\n\n uuid_idempotency_token = ?,\n\n dataset_token = ?,\n media_file_token = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?\n " + }, "b483867c3f767a01cc80e4c0e0f1950c65154e06c13fc2fcc6e3ec96e8e2c956": { "describe": { "columns": [], @@ -14939,6 +14949,44 @@ }, "query": "\nUPDATE tts_inference_jobs\nSET status='complete_failure'\nWHERE\n status IN ('pending')\n AND priority_level = 0;\n " }, + "cc27f21042c3d8ed5a2ef37e3e1d59f1aded7ba78acb938663cd439baf1c58ef": { + "describe": { + "columns": [ + { + "name": "token: tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken", + "ordinal": 0, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4229 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "media_file_token: tokens::tokens::media_files::MediaFileToken", + "ordinal": 1, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4233 + }, + "max_size": 128, + "type": "VarString" + } + } + ], + "nullable": [ + false, + false + ], + "parameters": { + "Right": 1 + } + }, + "query": "\nSELECT\n token as `token: tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken`,\n media_file_token as `media_file_token: tokens::tokens::media_files::MediaFileToken`\nFROM zs_voice_dataset_samples\nWHERE\n uuid_idempotency_token = ?\n AND user_deleted_at IS NULL\n AND mod_deleted_at IS NULL\n " + }, "cc4372cdd1a65999cbaf008f200683eedc93ab83e17256e47e011667ba8384d8": { "describe": { "columns": [], @@ -15871,6 +15919,16 @@ }, "query": "\nUPDATE api_tokens\nSET\n deleted_at = CURRENT_TIMESTAMP,\n ip_address_last_update = ?\nWHERE\n user_token = ?\nAND api_token NOT IN (?, ?, ?, ?, ?)\n " }, + "d66d409272c3aae14e81cf52b1c54e017889752eed9c7fc79e9a91317d95e4d7": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 10 + } + }, + "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n creator_set_visibility = ?\n\n " + }, "d71ef2cfed337f9e3f9e1c2255bdec6347763910fd589482f0b0e56b794d7bb7": { "describe": { "columns": [], diff --git a/crates/lib/server_environment/Cargo.toml b/crates/lib/server_environment/Cargo.toml index 61cd26d1e1..82d1767e4b 100644 --- a/crates/lib/server_environment/Cargo.toml +++ b/crates/lib/server_environment/Cargo.toml @@ -16,7 +16,6 @@ path = "src/lib.rs" [dependencies] # Internal easyenv = { path = "../easyenv" } -envvar = { path = "../envvar" } errors = { path = "../errors" } # External diff --git a/crates/lib/envvar/Cargo.toml b/crates/lib/storyteller_root/Cargo.toml similarity index 60% rename from crates/lib/envvar/Cargo.toml rename to crates/lib/storyteller_root/Cargo.toml index bd9c7b650e..243fb161ae 100644 --- a/crates/lib/envvar/Cargo.toml +++ b/crates/lib/storyteller_root/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "envvar" +name = "storyteller_root" edition = "2018" version = "0.0.1" authors = [ @@ -9,19 +9,18 @@ authors = [ publish = false [lib] -# NB: Public "envvar" crate is real, but doesn't seem useful. -# NB: We shouldn't use "env" because that overlaps std::env. -name = "envvar" +name = "storyteller_root" path = "src/lib.rs" [dependencies] # Internal -easyenv = { path = "../easyenv" } errors = { path = "../errors" } # External dotenv = "0.15.0" +env_logger = "0.10.0" log = "0.4.14" [dev-dependencies] -# None yet +serial_test = "2.0.0" +tempdir = "0.3.7" diff --git a/crates/lib/storyteller_root/src/lib.rs b/crates/lib/storyteller_root/src/lib.rs new file mode 100644 index 0000000000..009fc5f5a7 --- /dev/null +++ b/crates/lib/storyteller_root/src/lib.rs @@ -0,0 +1,38 @@ +//! storyteller_root +//! +//! This crate defines functions that locate the development root for Storyteller projects, +//! as well as our major monorepos. This can be used to build relative paths that should work +//! across developer machines and environments. +//! +//! These functions should not be relied upon in production. +//! +//! The following environment variables are read: +//! +//! - STORYTELLER_ROOT +//! - STORYTELLER_FRONTEND_ROOT +//! - STORYTELLER_ML_ROOT +//! - STORYTELLER_RUST_ROOT +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +mod paths; +mod substituted_path; + +pub use paths::storyteller_frontend_root::get_storyteller_frontend_root; +pub use paths::storyteller_ml_root::get_storyteller_ml_root; +pub use paths::storyteller_root::get_storyteller_root; +pub use paths::storyteller_rust_root::get_storyteller_rust_root; +pub use substituted_path::get_substituted_path; diff --git a/crates/lib/storyteller_root/src/paths/env_get_path.rs b/crates/lib/storyteller_root/src/paths/env_get_path.rs new file mode 100644 index 0000000000..00ebe3e64c --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/env_get_path.rs @@ -0,0 +1,81 @@ +use std::env::VarError; +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::path::PathBuf; +use std::str::FromStr; + +/// Errors with reading and parsing env variables. +#[derive(Debug, Eq, PartialEq)] +pub enum EnvError { + /// The environment variable value is not unicode. + NotUnicode, + /// Problem parsing the env variable as the desired type. + ParseError { + /// Explanation of the parsing failure. + reason: String + }, + /// The required environment variable wasn't present. + RequiredNotPresent, +} + +impl Display for EnvError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let reason = match self { + EnvError::NotUnicode => "EnvError::NotUnicode", + EnvError::ParseError { .. } => "EnvError::ParseError", + EnvError::RequiredNotPresent => "EnvError::RequiredNotPresent", + }; + write!(f, "{:?}", reason) + } +} + +impl Error for EnvError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} + +// NB: Pulled from `easyenv` crate. +pub (crate) fn env_get_path(env_name: &str) -> Result, EnvError> { + match std::env::var(env_name).as_ref() { + Err(err) => match err { + // TODO: EnvError enum variant for equals sign in env var name + VarError::NotPresent => Ok(None), + VarError::NotUnicode(_) => Err(EnvError::NotUnicode), + } + Ok(val) => { + match PathBuf::from_str(val) { + Ok(path) => Ok(Some(path)), + Err(_err) => Err(EnvError::ParseError { + reason: "error parsing PathBuf from value".to_string() + }), + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::env_get_path; + + #[test] + fn test_read_unset_env() { + assert_eq!(env_get_path("ASDFGH_FOO"), Ok(None)); + } + + #[test] + fn test_read_set_env() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + const ENV_VAR_NAME : &str = "ENV_TEST_VAR_DO_NOT_REUSE"; + + std::env::set_var(ENV_VAR_NAME, "/foo/bar"); + + assert_eq!(env_get_path(ENV_VAR_NAME), Ok(Some(PathBuf::from("/foo/bar")))); + + std::env::remove_var(ENV_VAR_NAME); + } +} diff --git a/crates/lib/storyteller_root/src/paths/mod.rs b/crates/lib/storyteller_root/src/paths/mod.rs new file mode 100644 index 0000000000..2d2a0798cc --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/mod.rs @@ -0,0 +1,5 @@ +pub (crate) mod env_get_path; +pub (crate) mod storyteller_frontend_root; +pub (crate) mod storyteller_ml_root; +pub (crate) mod storyteller_root; +pub (crate) mod storyteller_rust_root; diff --git a/crates/lib/storyteller_root/src/paths/storyteller_frontend_root.rs b/crates/lib/storyteller_root/src/paths/storyteller_frontend_root.rs new file mode 100644 index 0000000000..5a48af8f51 --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/storyteller_frontend_root.rs @@ -0,0 +1,108 @@ +use std::path::PathBuf; + +use crate::get_storyteller_root; +use crate::paths::env_get_path::env_get_path; + +/// The env var that declares where the storyteller frontend monorepo lives. +pub const STORYTELLER_FRONTEND_ROOT : &str = "STORYTELLER_FRONTEND_ROOT"; + +// DO NOT LEAK THIS. THIS IS FOR TESTING ONLY. +pub const TEST_STORYTELLER_FRONTEND_ROOT : &str = "ENV_TEST_STORYTELLER_FRONTEND_ROOT_DO_NOT_LEAK_"; + +/// Get the root of the Rust monorepo +pub fn get_storyteller_frontend_root() -> PathBuf { + // 1) Try "STORYTELLER_FRONTEND_ROOT" env var override. Do not check for path existence. + let root = if cfg!(test) { TEST_STORYTELLER_FRONTEND_ROOT } else { STORYTELLER_FRONTEND_ROOT }; + + if let Ok(Some(path)) = env_get_path(root) { + return path; + } + + // 2) Relative to "storyteller root" + let mut dir = get_storyteller_root(); + dir.push("storyteller-frontend"); + dir +} + +#[cfg(test)] +mod tests { + use std::env::{current_dir, temp_dir}; + use std::fs::{create_dir, remove_dir}; + use std::path::PathBuf; + + use serial_test::serial; + + use crate::paths::storyteller_root::TEST_HOME; + use crate::paths::storyteller_frontend_root::{get_storyteller_frontend_root, TEST_STORYTELLER_FRONTEND_ROOT}; + + #[test] + #[serial] + fn test_storyteller_frontend_home_env_var() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_FRONTEND_ROOT, "/testing/storyteller/root/custom-storyteller-frontend"); + assert_eq!(get_storyteller_frontend_root(), PathBuf::from("/testing/storyteller/root/custom-storyteller-frontend")); + std::env::remove_var(TEST_STORYTELLER_FRONTEND_ROOT); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_no_directories_exist() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, "/testing/fake_home"); + assert_eq!(get_storyteller_frontend_root(), PathBuf::from("/testing/fake_home/code/storyteller/storyteller-frontend")); + std::env::remove_var(TEST_HOME); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_with_existing_directory() { + let mut fake_home = temp_dir(); + fake_home.push("temp_dir_for_testing"); + + let mut fake_dev = fake_home.clone(); + fake_dev.push("dev"); + + // NB: Final path should be /tmp/temp_dir_for_testing/dev/storyteller + let mut fake_storyteller = fake_dev.clone(); + fake_storyteller.push("storyteller"); + + // NB: Ordered for repeatability + let _r = remove_dir(&fake_home).ok(); + let _r = create_dir(&fake_home).ok(); + let _r = create_dir(&fake_dev).ok(); + let _r = create_dir(&fake_storyteller).ok(); + + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, fake_home.to_str().expect("should be string")); + + let mut expected = fake_storyteller.clone(); + expected.push("storyteller-frontend"); + + assert_eq!(get_storyteller_frontend_root(), expected); + + std::env::remove_var(TEST_HOME); + + // NB: Must remove in reverse order, or we leave dangling pointers + let _r = remove_dir(&fake_storyteller).ok(); + let _r = remove_dir(&fake_dev).ok(); + let _r = remove_dir(&fake_home).ok(); + } + + #[test] + #[serial] + fn test_relative_to_current_working_directory() { + // NB: In this test we have no "TEST_HOME" directory. + let current_working_dir = current_dir().expect("Should have working dir."); + + let mut expected_dir = current_working_dir.clone(); + expected_dir.push("code/storyteller/storyteller-frontend"); + + assert_eq!(get_storyteller_frontend_root(), expected_dir); + } +} diff --git a/crates/lib/storyteller_root/src/paths/storyteller_ml_root.rs b/crates/lib/storyteller_root/src/paths/storyteller_ml_root.rs new file mode 100644 index 0000000000..c9233b062f --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/storyteller_ml_root.rs @@ -0,0 +1,108 @@ +use std::path::PathBuf; + +use crate::get_storyteller_root; +use crate::paths::env_get_path::env_get_path; + +/// The env var that declares where the storyteller ml monorepo lives. +pub const STORYTELLER_ML_ROOT : &str = "STORYTELLER_ML_ROOT"; + +// DO NOT LEAK THIS. THIS IS FOR TESTING ONLY. +pub const TEST_STORYTELLER_ML_ROOT : &str = "ENV_TEST_STORYTELLER_ML_ROOT_DO_NOT_LEAK_"; + +/// Get the root of the Rust monorepo +pub fn get_storyteller_ml_root() -> PathBuf { + // 1) Try "STORYTELLER_ML_ROOT" env var override. Do not check for path existence. + let root = if cfg!(test) { TEST_STORYTELLER_ML_ROOT } else { STORYTELLER_ML_ROOT }; + + if let Ok(Some(path)) = env_get_path(root) { + return path; + } + + // 2) Relative to "storyteller root" + let mut dir = get_storyteller_root(); + dir.push("storyteller-ml"); + dir +} + +#[cfg(test)] +mod tests { + use std::env::{current_dir, temp_dir}; + use std::fs::{create_dir, remove_dir}; + use std::path::PathBuf; + + use serial_test::serial; + + use crate::paths::storyteller_ml_root::{get_storyteller_ml_root, TEST_STORYTELLER_ML_ROOT}; + use crate::paths::storyteller_root::TEST_HOME; + + #[test] + #[serial] + fn test_storyteller_ml_home_env_var() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_ML_ROOT, "/testing/storyteller/root/custom-storyteller-ml"); + assert_eq!(get_storyteller_ml_root(), PathBuf::from("/testing/storyteller/root/custom-storyteller-ml")); + std::env::remove_var(TEST_STORYTELLER_ML_ROOT); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_no_directories_exist() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, "/testing/fake_home"); + assert_eq!(get_storyteller_ml_root(), PathBuf::from("/testing/fake_home/code/storyteller/storyteller-ml")); + std::env::remove_var(TEST_HOME); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_with_existing_directory() { + let mut fake_home = temp_dir(); + fake_home.push("temp_dir_for_testing"); + + let mut fake_dev = fake_home.clone(); + fake_dev.push("dev"); + + // NB: Final path should be /tmp/temp_dir_for_testing/dev/storyteller + let mut fake_storyteller = fake_dev.clone(); + fake_storyteller.push("storyteller"); + + // NB: Ordered for repeatability + let _r = remove_dir(&fake_home).ok(); + let _r = create_dir(&fake_home).ok(); + let _r = create_dir(&fake_dev).ok(); + let _r = create_dir(&fake_storyteller).ok(); + + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, fake_home.to_str().expect("should be string")); + + let mut expected = fake_storyteller.clone(); + expected.push("storyteller-ml"); + + assert_eq!(get_storyteller_ml_root(), expected); + + std::env::remove_var(TEST_HOME); + + // NB: Must remove in reverse order, or we leave dangling pointers + let _r = remove_dir(&fake_storyteller).ok(); + let _r = remove_dir(&fake_dev).ok(); + let _r = remove_dir(&fake_home).ok(); + } + + #[test] + #[serial] + fn test_relative_to_current_working_directory() { + // NB: In this test we have no "TEST_HOME" directory. + let current_working_dir = current_dir().expect("Should have working dir."); + + let mut expected_dir = current_working_dir.clone(); + expected_dir.push("code/storyteller/storyteller-ml"); + + assert_eq!(get_storyteller_ml_root(), expected_dir); + } +} diff --git a/crates/lib/storyteller_root/src/paths/storyteller_root.rs b/crates/lib/storyteller_root/src/paths/storyteller_root.rs new file mode 100644 index 0000000000..5d68e45c80 --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/storyteller_root.rs @@ -0,0 +1,151 @@ +use std::path::{Path, PathBuf}; + +use crate::paths::env_get_path::env_get_path; + +/// The env var that declares where the storyteller root directory lives. +pub const STORYTELLER_ROOT : &str = "STORYTELLER_ROOT"; + +/// Default OS home directory. +pub const HOME : &str = "HOME"; + +// DO NOT LEAK THIS. THIS IS FOR TESTING ONLY. +pub (crate) const TEST_STORYTELLER_ROOT : &str = "ENV_TEST_STORYTELLER_ROOT_DO_NOT_LEAK_"; +pub (crate) const TEST_HOME : &str = "ENV_TEST_HOME_DO_NOT_LEAK_"; + +// TODO: Test on Mac, Windows, and WSL. +/// Use several heuristics to find "storyteller root", where all of our monorepos are located. +pub fn get_storyteller_root() -> PathBuf { + // 1) Try "STORYTELLER_ROOT" env var override. Do not check for path existence. + let env_var_name = if cfg!(test) { TEST_STORYTELLER_ROOT } else { STORYTELLER_ROOT }; + + if let Ok(Some(path)) = env_get_path(env_var_name) { + return path; + } + + // TODO: Consider `dirs` or `directories` crates for cross-platform support: + // https://crates.io/crates/dirs + // https://crates.io/crates/directories + + // 2) Try to construct paths relative to the "HOME" directory. Prefer existing directories. + let env_var_name = if cfg!(test) { TEST_HOME } else { HOME }; + + if let Ok(Some(home_path)) = env_get_path(env_var_name) { + if let Some(path) = try_subdirectories(home_path) { + return path; + } + } + + // 3) Try to construct paths relative to the current working directory. + if let Ok(current_working_dir) = std::env::current_dir() { + if let Some(path) = try_subdirectories(current_working_dir) { + return path; + } + } + + // 4) Rooted directory (since this function is infallible) + PathBuf::from("/storyteller") +} + +fn try_subdirectories(root_path: PathBuf) -> Option { + None + // First try to see if the directories exist (in order) + .or_else(|| try_subdirectory(&root_path, "code/storyteller", true)) + .or_else(|| try_subdirectory(&root_path, "dev/storyteller", true)) + // Then just use whatever parses. + .or_else(|| try_subdirectory(&root_path, "code/storyteller", false)) +} + +fn try_subdirectory<'a>(dir: &Path, sub_path: &str, check_if_exists: bool) -> Option { + let mut home = dir.to_path_buf(); + home.push(sub_path); + + if let Ok(canonical) = home.canonicalize() { + home = canonical; // NB: Ignore errors + } + + if check_if_exists && !(home.exists() && home.is_dir()) { + return None; + } + + Some(home) +} + +#[cfg(test)] +mod tests { + use std::env::{current_dir, temp_dir}; + use std::fs::{create_dir, remove_dir}; + use std::path::PathBuf; + + use serial_test::serial; + + use crate::get_storyteller_root; + use crate::paths::storyteller_root::{TEST_HOME, TEST_STORYTELLER_ROOT}; + + #[test] + #[serial] + fn test_storyteller_home_env_var() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_ROOT, "/testing/storyteller/root"); + assert_eq!(get_storyteller_root(), PathBuf::from("/testing/storyteller/root")); + std::env::remove_var(TEST_STORYTELLER_ROOT); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_no_directories_exist() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, "/testing/fake_home"); + assert_eq!(get_storyteller_root(), PathBuf::from("/testing/fake_home/code/storyteller")); + std::env::remove_var(TEST_HOME); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_with_existing_directory() { + let mut fake_home = temp_dir(); + fake_home.push("temp_dir_for_testing"); + + let mut fake_dev = fake_home.clone(); + fake_dev.push("dev"); + + // NB: Final path should be /tmp/temp_dir_for_testing/dev/storyteller + let mut fake_storyteller = fake_dev.clone(); + fake_storyteller.push("storyteller"); + + // NB: Ordered for repeatability + let _r = remove_dir(&fake_home).ok(); + let _r = create_dir(&fake_home).ok(); + let _r = create_dir(&fake_dev).ok(); + let _r = create_dir(&fake_storyteller).ok(); + + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, fake_home.to_str().expect("should be string")); + + assert_eq!(get_storyteller_root(), fake_storyteller); + + std::env::remove_var(TEST_HOME); + + // NB: Must remove in reverse order, or we leave dangling pointers + let _r = remove_dir(&fake_storyteller).ok(); + let _r = remove_dir(&fake_dev).ok(); + let _r = remove_dir(&fake_home).ok(); + } + + #[test] + #[serial] + fn test_relative_to_current_working_directory() { + // NB: In this test we have no "TEST_HOME" directory. + let current_working_dir = current_dir().expect("Should have working dir."); + + let mut storyteller_dir = current_working_dir.clone(); + storyteller_dir.push("code/storyteller"); + + assert_eq!(get_storyteller_root(), storyteller_dir); + } +} diff --git a/crates/lib/storyteller_root/src/paths/storyteller_rust_root.rs b/crates/lib/storyteller_root/src/paths/storyteller_rust_root.rs new file mode 100644 index 0000000000..fd339e73c7 --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/storyteller_rust_root.rs @@ -0,0 +1,108 @@ +use std::path::PathBuf; + +use crate::get_storyteller_root; +use crate::paths::env_get_path::env_get_path; + +/// The env var that declares where the storyteller rust monorepo lives. +pub const STORYTELLER_RUST_ROOT : &str = "STORYTELLER_RUST_ROOT"; + +// DO NOT LEAK THIS. THIS IS FOR TESTING ONLY. +pub (crate) const TEST_STORYTELLER_RUST_ROOT : &str = "ENV_TEST_STORYTELLER_RUST_ROOT_DO_NOT_LEAK_"; + +/// Get the root of the Rust monorepo +pub fn get_storyteller_rust_root() -> PathBuf { + // 1) Try "STORYTELLER_RUST_ROOT" env var override. Do not check for path existence. + let root = if cfg!(test) { TEST_STORYTELLER_RUST_ROOT } else { STORYTELLER_RUST_ROOT }; + + if let Ok(Some(path)) = env_get_path(root) { + return path; + } + + // 2) Relative to "storyteller root" + let mut dir = get_storyteller_root(); + dir.push("storyteller-rust"); + dir +} + +#[cfg(test)] +mod tests { + use std::env::{current_dir, temp_dir}; + use std::fs::{create_dir, remove_dir}; + use std::path::PathBuf; + + use serial_test::serial; + + use crate::paths::storyteller_root::TEST_HOME; + use crate::paths::storyteller_rust_root::{get_storyteller_rust_root, TEST_STORYTELLER_RUST_ROOT}; + + #[test] + #[serial] + fn test_storyteller_rust_home_env_var() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_RUST_ROOT, "/testing/storyteller/root/custom-storyteller-rust"); + assert_eq!(get_storyteller_rust_root(), PathBuf::from("/testing/storyteller/root/custom-storyteller-rust")); + std::env::remove_var(TEST_STORYTELLER_RUST_ROOT); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_no_directories_exist() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, "/testing/fake_home"); + assert_eq!(get_storyteller_rust_root(), PathBuf::from("/testing/fake_home/code/storyteller/storyteller-rust")); + std::env::remove_var(TEST_HOME); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_with_existing_directory() { + let mut fake_home = temp_dir(); + fake_home.push("temp_dir_for_testing"); + + let mut fake_dev = fake_home.clone(); + fake_dev.push("dev"); + + // NB: Final path should be /tmp/temp_dir_for_testing/dev/storyteller + let mut fake_storyteller = fake_dev.clone(); + fake_storyteller.push("storyteller"); + + // NB: Ordered for repeatability + let _r = remove_dir(&fake_home).ok(); + let _r = create_dir(&fake_home).ok(); + let _r = create_dir(&fake_dev).ok(); + let _r = create_dir(&fake_storyteller).ok(); + + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, fake_home.to_str().expect("should be string")); + + let mut expected = fake_storyteller.clone(); + expected.push("storyteller-rust"); + + assert_eq!(get_storyteller_rust_root(), expected); + + std::env::remove_var(TEST_HOME); + + // NB: Must remove in reverse order, or we leave dangling pointers + let _r = remove_dir(&fake_storyteller).ok(); + let _r = remove_dir(&fake_dev).ok(); + let _r = remove_dir(&fake_home).ok(); + } + + #[test] + #[serial] + fn test_relative_to_current_working_directory() { + // NB: In this test we have no "TEST_HOME" directory. + let current_working_dir = current_dir().expect("Should have working dir."); + + let mut expected_dir = current_working_dir.clone(); + expected_dir.push("code/storyteller/storyteller-rust"); + + assert_eq!(get_storyteller_rust_root(), expected_dir); + } +} diff --git a/crates/lib/storyteller_root/src/substituted_path.rs b/crates/lib/storyteller_root/src/substituted_path.rs new file mode 100644 index 0000000000..15bb9fb0de --- /dev/null +++ b/crates/lib/storyteller_root/src/substituted_path.rs @@ -0,0 +1,123 @@ +use std::path::PathBuf; + +use crate::{get_storyteller_frontend_root, get_storyteller_ml_root, get_storyteller_root, get_storyteller_rust_root}; + +const STORYTELLER_ROOT_REPLACEMENT : &str = "%storyteller_root%"; + +const STORYTELLER_FRONTEND_REPLACEMENT : &str = "%storyteller_frontend%"; +const STORYTELLER_ML_REPLACEMENT : &str = "%storyteller_ml%"; +const STORYTELLER_RUST_REPLACEMENT : &str = "%storyteller_rust%"; + +/// Transform a given string path, substituting "magic" variables for the various storyteller +/// project directories. +/// +/// Substitution variables: +/// +/// - %storyteller_root% +/// - %storyteller_rust% +/// - %storyteller_ml% +/// - %storyteller_frontend% +/// +pub fn get_substituted_path(path: &str) -> PathBuf { + // TODO(bt,2023-10-17): Should this be infallible? If not, where in the stack of transformations + // should the failure manifest? ie, if infallible, should we make the final paths "closer" to + // the de-substituted string, or the environment variable? Which makes it easier to detect the + // error? + // + // TODO(bt,2023-10-17): I'm not happy with all of the internal type juggling and allocations, + // but I shouldn't spend time on refactoring this now. + // + if path.contains(STORYTELLER_ROOT_REPLACEMENT) { + let replacement = type_juggle_path(get_storyteller_root()); + let path = path.replacen(STORYTELLER_ROOT_REPLACEMENT, &replacement, 1); + return PathBuf::from(path); + } else if path.contains(STORYTELLER_RUST_REPLACEMENT) { + let replacement = type_juggle_path(get_storyteller_rust_root()); + let path = path.replacen(STORYTELLER_RUST_REPLACEMENT, &replacement, 1); + return PathBuf::from(path); + } else if path.contains(STORYTELLER_ML_REPLACEMENT) { + let replacement = type_juggle_path(get_storyteller_ml_root()); + let path = path.replacen(STORYTELLER_ML_REPLACEMENT, &replacement, 1); + return PathBuf::from(path); + } else if path.contains(STORYTELLER_FRONTEND_REPLACEMENT) { + let replacement = type_juggle_path(get_storyteller_frontend_root()); + let path = path.replacen(STORYTELLER_FRONTEND_REPLACEMENT, &replacement, 1); + return PathBuf::from(path); + } + + PathBuf::from(path) +} + +fn type_juggle_path(path: PathBuf) -> String { + // NB: The Path/PathBuf library does some silly stuff regarding non-UTF-8 paths. + path.to_str() + .map(|s| s.to_string()) + .unwrap_or_else(|| path.to_string_lossy().to_string()) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serial_test::serial; + + use crate::paths::storyteller_root::TEST_STORYTELLER_ROOT; + use crate::substituted_path::get_substituted_path; + + #[test] + fn test_no_substitutions() { + assert_eq!(get_substituted_path("/foo/bar"), PathBuf::from("/foo/bar")); + } + + #[test] + #[serial] + fn test_storyteller_root_substitution() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_ROOT, "/testing/storyteller/root"); + + assert_eq!(get_substituted_path("%storyteller_root%/foo/bar"), PathBuf::from("/testing/storyteller/root/foo/bar")); + + std::env::remove_var(TEST_STORYTELLER_ROOT); + } + + #[test] + #[serial] + fn test_storyteller_rust_root_substitution() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_ROOT, "/testing/storyteller/root"); + + assert_eq!(get_substituted_path("%storyteller_rust%/foo/bar"), PathBuf::from("/testing/storyteller/root/storyteller-rust/foo/bar")); + + std::env::remove_var(TEST_STORYTELLER_ROOT); + } + + #[test] + #[serial] + fn test_storyteller_ml_root_substitution() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_ROOT, "/testing/storyteller/root"); + + assert_eq!(get_substituted_path("%storyteller_ml%/foo/bar"), PathBuf::from("/testing/storyteller/root/storyteller-ml/foo/bar")); + + std::env::remove_var(TEST_STORYTELLER_ROOT); + } + + #[test] + #[serial] + fn test_storyteller_frontend_root_substitution() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_STORYTELLER_ROOT, "/testing/storyteller/root"); + + assert_eq!(get_substituted_path("%storyteller_frontend%/foo/bar"), PathBuf::from("/testing/storyteller/root/storyteller-frontend/foo/bar")); + + std::env::remove_var(TEST_STORYTELLER_ROOT); + } +} diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index 18060d8522..aafe4c50f4 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -23,7 +23,6 @@ container_common = { path = "../../../lib/deprecated/container_common" } crockford = { path = "../../../lib/crockford" } easyenv = { path = "../../../lib/easyenv" } enums = { path = "../../../_types/enums" } -envvar = { path = "../../../lib/envvar" } errors = { path = "../../../lib/errors" } filesys = { path = "../../../lib/files/filesys" } google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } diff --git a/crates/service/job/download_job/config/download-job.development.env b/crates/service/job/download_job/config/download-job.development.env index bdaed04a7c..bd1d27eb0d 100644 --- a/crates/service/job/download_job/config/download-job.development.env +++ b/crates/service/job/download_job/config/download-job.development.env @@ -19,10 +19,10 @@ WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d0c49bc35097 # # # ================================= # -RVC_V2_MODEL_CHECK_ROOT_DIRECTORY="/home/bt/dev/storyteller/storyteller-ml/voice_conversion/rvc-v2-brandon" +RVC_V2_MODEL_CHECK_ROOT_DIRECTORY="%storyteller_ml%/voice_conversion/rvc-v2-brandon" RVC_V2_MODEL_CHECK_COMMAND="python fakeyou_infer.py" RVC_V2_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate" -RVC_V2_MODEL_CHECK_MAYBE_DEFAULT_TEST_WAV_PATH="/home/bt/dev/storyteller/storyteller-ml/voice_conversion/rvc-v2-brandon/test.wav" +RVC_V2_MODEL_CHECK_MAYBE_DEFAULT_TEST_WAV_PATH="%storyteller_ml%/voice_conversion/rvc-v2-brandon/test.wav" #RVC_V2_MODEL_CHECK_COMMAND="/common/so-vits-svc-code/install/python/bin/python3.10 -m so_vits_svc_fork.fakeyou_infer" #RVC_V2_MODEL_CHECK_MAYBE_DOCKER_IMAGE="8acacf30bb2e" #RVC_V2_MODEL_CHECK_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vits-svc/example_config.json" diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 25a8dc74ea..18dbb60d83 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -24,7 +24,6 @@ config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } enums = { path = "../../../_types/enums" } -envvar = { path = "../../../lib/envvar" } errors = { path = "../../../lib/errors" } filesys = { path = "../../../lib/files/filesys" } hashing = { path = "../../../lib/files/hashing" } diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 864967c6e4..303534d758 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -22,13 +22,18 @@ VALL_E_X_INFERENCE_COMMAND = "python3 main.py" VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .venv/bin/activate" # VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE = "" # fill this out for dev? +#VALL_E_X_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/tts/VALL-E-X" +#VALL_E_X_INFERENCE_COMMAND="python3 main.py" +#VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND="source .env/bin/activate" +#VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE="" # fill this out for dev? + # ================================= # # # # RVC (v2) # # # # ================================= # -RVC_V2_INFERENCE_ROOT_DIRECTORY="/home/bt/dev/storyteller/storyteller-ml/voice_conversion/rvc-v2-brandon" +RVC_V2_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/voice_conversion/rvc-v2-brandon" RVC_V2_INFERENCE_COMMAND="python fakeyou_infer.py" RVC_V2_INFERENCE_MAYBE_VENV_COMMAND="source python/bin/activate" @@ -38,7 +43,7 @@ RVC_V2_INFERENCE_MAYBE_VENV_COMMAND="source python/bin/activate" # # # ================================= # -SAD_TALKER_INFERENCE_ROOT_DIRECTORY="/home/bt/dev/storyteller/storyteller-ml/animation/SadTalker" +SAD_TALKER_INFERENCE_ROOT_DIRECTORY="%storyteller_ml%/animation/SadTalker" SAD_TALKER_INFERENCE_COMMAND="python inference.py" SAD_TALKER_INFERENCE_MAYBE_VENV_COMMAND="source python/bin/activate" @@ -48,7 +53,7 @@ SAD_TALKER_INFERENCE_MAYBE_VENV_COMMAND="source python/bin/activate" # # # ================================= # -FFMPEG_LOGO_LOGO_PATH="/home/bt/dev/storyteller/storyteller-rust/includes/container_includes/image_assets/fakeyou_watermark.png" +FFMPEG_LOGO_LOGO_PATH="%storyteller_rust%/includes/container_includes/image_assets/fakeyou_watermark.png" # ================================= # # # diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 75c6c9f87a..dbc5cde46c 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -25,7 +25,6 @@ container_common = { path = "../../../lib/deprecated/container_common" } datetimes = { path = "../../../lib/datetimes" } easyenv = { path = "../../../lib/easyenv" } enums = { path = "../../../_types/enums" } -envvar = { path = "../../../lib/envvar" } errors = { path = "../../../lib/errors" } google_drive_common = { path = "../../../lib/api_clients/google_drive_common" } hashing = { path = "../../../lib/files/hashing" }