Better env var support

.idea/dictionaries/bt.xml

@@ -4,6 +4,7 @@
       <w>arpabet</w>
       <w>backfill</w>
       <w>canonicalized</w>
+      <w>canonicalizing</w>
       <w>cashapp</w>
       <w>cheermote</w>
       <w>cidrs</w>

Cargo.lock

@@ -1043,7 +1043,6 @@ version = "0.0.1"
 dependencies = [
  "dotenv",
  "easyenv",
- "envvar",
  "errors",
  "hostname",
  "log",
@@ -1948,7 +1947,6 @@ dependencies = [
  "dotenv",
  "easyenv",
  "enums",
- "envvar",
  "errors",
  "filesys",
  "google_drive_common",
@@ -2024,7 +2022,7 @@ dependencies = [
 
 [[package]]
 name = "easyenv"
-version = "0.0.9"
+version = "0.1.0"
 dependencies = [
  "dotenv",
  "env_logger 0.10.0",
@@ -2106,16 +2104,6 @@ dependencies = [
  "termcolor",
 ]
 
-[[package]]
-name = "envvar"
-version = "0.0.1"
-dependencies = [
- "dotenv",
- "easyenv",
- "errors",
- "log",
-]
-
 [[package]]
 name = "errno"
 version = "0.3.1"
@@ -2958,7 +2946,6 @@ dependencies = [
  "dotenv",
  "easyenv",
  "enums",
- "envvar",
  "errors",
  "filesys",
  "hashing",
@@ -5123,7 +5110,6 @@ version = "0.0.1"
 dependencies = [
  "dotenv",
  "easyenv",
- "envvar",
  "errors",
  "log",
 ]
@@ -5535,7 +5521,6 @@ dependencies = [
  "dotenv",
  "easyenv",
  "enums",
- "envvar",
  "errors",
  "futures 0.1.31",
  "futures 0.3.26",

Cargo.toml

@@ -30,7 +30,6 @@ members = [
   "crates/lib/deprecated/container_common", # TODO(bt, 2023-02-13): Pull apart and rehome pieces
   "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers
   "crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve.
-  "crates/lib/envvar",
   "crates/lib/errors",
   "crates/lib/files/archives",
   "crates/lib/files/filesys",

crates/lib/bootstrap/Cargo.toml

@@ -16,7 +16,6 @@ path = "src/lib.rs"
 [dependencies]
 # Internal
 easyenv = { path = "../easyenv" }
-envvar = { path = "../envvar" }
 errors = { path = "../errors" }
 server_environment = { path = "../server_environment" }
 

crates/lib/bootstrap/src/bootstrap.rs

@@ -120,7 +120,7 @@ fn load_env_config_files<P: AsRef<Path>>(server_environment: ServerEnvironment,
   for env_config_file in env_config_file_names.into_iter() {
     info!("Loading environment variable config file: {}", &env_config_file);
 
-    let was_read = envvar::maybe_read_from_filename_and_paths(
+    let was_read = easyenv::maybe_read_env_config_from_filename_and_paths(
       &env_config_file,
       args.config_search_directories)?;
 

crates/lib/easyenv/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "easyenv"
 edition = "2018"
-version = "0.0.9"
+version = "0.1.0"
 authors = [
   "Brandon Thomas <bt@brand.io>",
   "Brandon Thomas <echelon@gmail.com>",

crates/lib/easyenv/src/error.rs

@@ -0,0 +1,62 @@
+use std::error::Error;
+use std::fmt;
+use std::fmt::{Display, Formatter};
+
+/// Errors with reading and parsing env variables.
+#[derive(Debug)]
+pub enum EnvError {
+  /// The environment variable value is not unicode.
+  NotUnicode,
+  /// Problem parsing the env variable as the desired type.
+  ParseError {
+    /// Explanation of the parsing failure.
+    reason: String
+  },
+  /// The required environment variable wasn't present.
+  RequiredNotPresent,
+}
+
+impl Display for EnvError {
+  fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+    let reason = match self {
+      EnvError::NotUnicode => "EnvError::NotUnicode",
+      EnvError::ParseError { .. } => "EnvError::ParseError",
+      EnvError::RequiredNotPresent => "EnvError::RequiredNotPresent",
+    };
+    write!(f, "{:?}", reason)
+  }
+}
+
+impl Error for EnvError {
+  fn source(&self) -> Option<&(dyn Error + 'static)> {
+    None
+  }
+}
+
+/// Errors while initializing the library
+#[derive(Debug)]
+pub enum InitError {
+  /// std::io error
+  IoError,
+  /// dotenv couldn't read a file
+  DotEnvError,
+  /// No env config file was found and read
+  NoConfigFileFoundError,
+}
+
+impl Display for InitError {
+  fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+    let reason = match self {
+      InitError::IoError => "InitError::IoError",
+      InitError::DotEnvError => "InitError::DotEnvError",
+      InitError::NoConfigFileFoundError => "InitError::NoConfigFileFoundError",
+    };
+    write!(f, "{:?}", reason)
+  }
+}
+
+impl Error for InitError {
+  fn source(&self) -> Option<&(dyn Error + 'static)> {
+    None
+  }
+}

crates/lib/easyenv/src/init.rs

@@ -0,0 +1,64 @@
+use std::path::{Path, PathBuf};
+
+use log::error;
+
+use crate::error::InitError;
+use crate::init_env_logger;
+
+/// Initialize dotenv (with `.env` file) and the env logger using the `RUST_LOG` variable setting.
+pub fn init_all_with_default_logging(default_if_absent: Option<&str>) {
+  init_dotenv();
+  init_env_logger(default_if_absent)
+}
+
+/// Try loading in an environment file across many search paths (first found wins).
+/// Returns an error if no file could be read or if in attempting to read a file there was an error.
+pub fn read_env_config_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> Result<(), InitError> {
+  if do_read_env_config_from_filename_and_paths(&filename, paths)? {
+    Ok(())
+  } else {
+    error!("No env file existed for filename {:?} in the search paths {:?}",
+      filename.as_ref(),
+      paths.iter().map(|p| p.as_ref().to_path_buf()).collect::<Vec<PathBuf>>());
+    Err(InitError::NoConfigFileFoundError)
+  }
+}
+
+/// Try loading in an environment file across many search paths (first found wins).
+/// Returns okay if no file was read, or an error if in attempting to read a file there was an error.
+/// Returns boolean reflecting whether a file was read.
+pub fn maybe_read_env_config_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> Result<bool, InitError> {
+  Ok(do_read_env_config_from_filename_and_paths(filename, paths)?)
+}
+
+// Initialize dotenv with the default `.env` config file.
+// Do not fail if the  file cannot be loaded.
+fn init_dotenv() {
+  match dotenv::dotenv() {
+    Ok(_) => println!("dotenv configs initialized"),
+    Err(e) => println!("Could not initialize dotenv: {:?}", e),
+  }
+}
+
+fn do_read_env_config_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> Result<bool, InitError> {
+  for path in paths.iter() {
+    let path = path.as_ref().join(filename.as_ref());
+
+    if path.exists() && path.is_file() {
+      log::info!("Attempting to read env vars from file: {:?}", path);
+      let path = std::fs::canonicalize(path)
+          .map_err(|err| {
+            error!("Error canonicalizing path: {:?}", err);
+            InitError::IoError
+          })?;
+      dotenv::from_path(path)
+          .map_err(|err| {
+            error!("dotenv error reading config: {:?}", err);
+            InitError::DotEnvError
+          })?;
+      return Ok(true);
+    }
+  }
+
+  Ok(false)
+}

crates/lib/easyenv/src/lib.rs

@@ -1,7 +1,24 @@
 // Copyright (c) 2020 Brandon Thomas <bt@brand.io>
 
-//! Very simple helper functions for environment variables and environment variable-driven
-//! `env_logger` configuration.
+//! easyenv
+//!
+//! The interface between an app and its environment variables. This library makes it easy to:
+//!
+//!   - read in strongly-typed environment variables.
+//!   - load default environment variables from one or more files in an order that supports
+//!     intentional overriding.
+//!
+//! Things that might want to move to another package :
+//!
+//!   - configure env logging
+//!   - build conventions around system paths such that paths imported from environment variables
+//!     respect pathing conventions.
+//!
+//! This library does not:
+//!
+//!   - handle app-specific conventions
+//!   - care about servers or deployed environments (eg. "staging" vs "production")
+//!
 
 #![deny(dead_code)]
 #![deny(missing_docs)]
@@ -11,101 +28,32 @@
 #![deny(unused_qualifications)]
 #![deny(unused_qualifications)]
 
-use env_logger;
-use std::error::Error;
-use std::fmt::{Display, Debug, Formatter};
-use std::{env, fmt};
+// NB: These were exported from the root in previous versions.
+// It's easy to just keep them exported here to maintain compatibility.
+pub use error::EnvError;
+pub use error::InitError;
+pub use init::init_all_with_default_logging;
+pub use init::maybe_read_env_config_from_filename_and_paths;
+pub use init::read_env_config_from_filename_and_paths;
+pub use logging::DEFAULT_LOG_LEVEL;
+pub use logging::ENV_RUST_LOG;
+pub use logging::init_env_logger;
+pub use types::boolean::get_env_bool_optional;
+pub use types::boolean::get_env_bool_or_default;
+pub use types::boolean::get_env_bool_required;
+pub use types::duration::get_env_duration_seconds_optional;
+pub use types::duration::get_env_duration_seconds_or_default;
+pub use types::duration::get_env_duration_seconds_required;
+pub use types::num::get_env_num;
+pub use types::pathbuf::get_env_pathbuf_optional;
+pub use types::pathbuf::get_env_pathbuf_or_default;
+pub use types::pathbuf::get_env_pathbuf_required;
+pub use types::string::get_env_string_optional;
+pub use types::string::get_env_string_or_default;
+pub use types::string::get_env_string_required;
 
-mod boolean;
-mod duration;
-mod pathbuf;
-mod string;
-mod num;
+mod error;
+mod init;
+mod logging;
+mod types;
 
-/// Name of the environment variable Rust's env logger uses
-pub const ENV_RUST_LOG : &str = "RUST_LOG";
-
-const DEFAULT_LOG_LEVEL: &str = "info";
-
-/// Errors with env variables.
-#[derive(Debug)]
-pub enum EnvError {
-  /// The environment variable value is not unicode.
-  NotUnicode,
-  /// Problem parsing the env variable as the desired type.
-  ParseError {
-    /// Explanation of the parsing failure.
-    reason: String
-  },
-  /// The required environment variable wasn't present.
-  RequiredNotPresent,
-}
-
-impl Display for EnvError {
-  fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-    write!(f, "EnvError::ParseError")
-  }
-}
-
-impl Error for EnvError {
-  fn source(&self) -> Option<&(dyn Error + 'static)> {
-    None
-  }
-}
-
-pub use boolean::get_env_bool_optional;
-pub use boolean::get_env_bool_or_default;
-pub use boolean::get_env_bool_required;
-
-pub use duration::get_env_duration_seconds_optional;
-pub use duration::get_env_duration_seconds_or_default;
-pub use duration::get_env_duration_seconds_required;
-
-pub use num::get_env_num;
-
-pub use pathbuf::get_env_pathbuf_optional;
-pub use pathbuf::get_env_pathbuf_or_default;
-pub use pathbuf::get_env_pathbuf_required;
-
-pub use string::get_env_string_optional;
-pub use string::get_env_string_or_default;
-pub use string::get_env_string_required;
-
-/// Initialize Rust's env logger.
-///
-/// The Rust logger reads the desired log level from the `RUST_LOG` environment variable. If this
-/// isn't set, the provided default is used. If a default fallback isn't provided to this function,
-/// we fall back to `"info"`.
-///
-/// A more robust logging config might configure on a per-component basis, eg.
-/// `"tokio_reactor=warn,hyper=info,debug"`. You can read more in the `log` and `env_logger` crate
-/// docs.
-pub fn init_env_logger(default_if_absent: Option<&str>) {
-  if env::var(ENV_RUST_LOG)
-    .as_ref()
-    .ok()
-    .is_none()
-  {
-    let default_log_level = default_if_absent.unwrap_or(DEFAULT_LOG_LEVEL);
-    println!("Setting default logging level to \"{}\", override with env var {}.",
-             default_log_level, ENV_RUST_LOG);
-    env::set_var(ENV_RUST_LOG, default_log_level);
-  }
-
-  env_logger::init();
-}
-
-/// Initialize dotenv with the default `.env` config file.
-pub fn init_dotenv() {
-  match dotenv::dotenv() {
-    Ok(_) => println!("dotenv configs initialized"),
-    Err(e) => println!("Could not initialize dotenv: {:?}", e),
-  }
-}
-
-/// Initialize dotenv and env logger.
-/// See `init_dotenv()` and `init_env_logger(Option<&str>)` for further details.
-pub fn init_all_with_default_logging(default_if_absent: Option<&str>) {
-  init_dotenv();
-  init_env_logger(default_if_absent)
-}

crates/lib/easyenv/src/logging.rs

@@ -0,0 +1,35 @@
+use std::env;
+
+
+/// Name of the environment variable Rust's env logger uses
+pub const ENV_RUST_LOG : &str = "RUST_LOG";
+
+/// The default logging level
+pub const DEFAULT_LOG_LEVEL: &str = "info";
+
+
+
+/// Initialize Rust's env logger.
+///
+/// The Rust logger reads the desired log level from the `RUST_LOG` environment variable. If this
+/// isn't set, the provided default is used. If a default fallback isn't provided to this function,
+/// we fall back to `"info"`.
+///
+/// A more robust logging config might configure on a per-component basis, eg.
+/// `"tokio_reactor=warn,hyper=info,debug"`. You can read more in the `log` and `env_logger` crate
+/// docs.
+pub fn init_env_logger(default_if_absent: Option<&str>) {
+  if env::var(ENV_RUST_LOG)
+      .as_ref()
+      .ok()
+      .is_none()
+  {
+    let default_log_level = default_if_absent.unwrap_or(DEFAULT_LOG_LEVEL);
+    println!("Setting default logging level to \"{}\", override with env var {}.",
+             default_log_level, ENV_RUST_LOG);
+    env::set_var(ENV_RUST_LOG, default_log_level);
+  }
+
+  env_logger::init();
+}
+

crates/lib/easyenv/src/types/boolean.rs

@@ -1,7 +1,9 @@
 use std::env;
 use std::env::VarError;
+
 use log::warn;
-use crate::EnvError;
+
+use crate::error::EnvError;
 
 /// Get an environment variable as a bool.
 /// If not present or there is an error in parsing, return `None`.

crates/lib/easyenv/src/types/duration.rs

@@ -1,8 +1,10 @@
-use crate::EnvError;
-use log::warn;
 use std::env;
 use std::time::Duration;
 
+use log::warn;
+
+use crate::error::EnvError;
+
 /// Get an environment variable as a `Duration` in seconds.
 /// If not provided or cannot parse, return an error.
 pub fn get_env_duration_seconds_required(env_name: &str) -> Result<Duration, EnvError> {

crates/lib/easyenv/src/types/mod.rs

@@ -0,0 +1,5 @@
+pub mod boolean;
+pub mod duration;
+pub mod pathbuf;
+pub mod string;
+pub mod num;

crates/lib/easyenv/src/types/num.rs

@@ -1,8 +1,10 @@
 use std::env;
 use std::fmt::{Debug, Display};
 use std::str::FromStr;
+
 use log::{error, warn};
-use crate::EnvError;
+
+use crate::error::EnvError;
 
 /// Get an environment variable as a number, or fall back to the provided default if not set.
 /// If the env var is present but can't be parsed, an error is returned instead.

crates/lib/easyenv/src/types/pathbuf.rs

@@ -1,10 +1,12 @@
-use crate::EnvError;
-use log::warn;
-use std::env::VarError;
 use std::env;
+use std::env::VarError;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
+use log::warn;
+
+use crate::error::EnvError;
+
 /// Get an environment variable as a `PathBuf`.
 /// If not provided or cannot parse, return an error.
 pub fn get_env_pathbuf_required(env_name: &str) -> Result<PathBuf, EnvError> {

crates/lib/easyenv/src/types/string.rs

@@ -1,7 +1,9 @@
-use crate::EnvError;
-use log::warn;
 use std::env;
 
+use log::warn;
+
+use crate::error::EnvError;
+
 /// Get an environment variable as an optional `String`.
 pub fn get_env_string_optional(env_name: &str) -> Option<String> {
   match env::var(env_name).as_ref().ok() {

crates/lib/envvar/Cargo.toml

@@ -1,27 +0,0 @@
-[package]
-name = "envvar"
-edition = "2018"
-version = "0.0.1"
-authors = [
-    "Brandon Thomas <bt@brand.io>",
-    "Brandon Thomas <echelon@gmail.com>",
-]
-publish = false
-
-[lib]
-# NB: Public "envvar" crate is real, but doesn't seem useful.
-# NB: We shouldn't use "env" because that overlaps std::env.
-name = "envvar"
-path = "src/lib.rs"
-
-[dependencies]
-# Internal
-easyenv = { path = "../easyenv" }
-errors = { path = "../errors" }
-
-# External
-dotenv = "0.15.0"
-log = "0.4.14"
-
-[dev-dependencies]
-# None yet

crates/lib/envvar/src/lib.rs

@@ -1,61 +0,0 @@
-//! envvar
-//!
-//! The purpose of this library is to handle reading environment variables.
-//!
-
-// Never allow these
-#![forbid(private_in_public)]
-#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
-
-// Okay to toggle
-#![forbid(unreachable_patterns)]
-#![forbid(unused_imports)]
-#![forbid(unused_mut)]
-#![forbid(unused_variables)]
-
-// Always allow
-#![allow(dead_code)]
-#![allow(non_snake_case)]
-
-use std::path::{Path, PathBuf};
-
-use errors::{anyhow, AnyhowResult};
-
-pub fn read_from_filename<P: AsRef<Path>>(filename: P) -> AnyhowResult<PathBuf> {
-  let result = dotenv::from_filename(filename)?;
-  Ok(result)
-}
-
-/// Try loading in an environment file across many search paths (first found wins).
-/// Returns an error if no file could be read or if in attempting to read a file there was an error.
-pub fn read_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> AnyhowResult<()> {
-  if do_read_from_filename_and_paths(&filename, paths)? {
-    Ok(())
-  } else {
-    Err(anyhow!("No env file existed for filename {:?} in the search paths {:?}",
-      filename.as_ref(),
-      paths.iter().map(|p| p.as_ref().to_path_buf()).collect::<Vec<PathBuf>>()))
-  }
-}
-
-/// Try loading in an environment file across many search paths (first found wins).
-/// Returns okay if no file was read, or an error if in attempting to read a file there was an error.
-/// Returns boolean reflecting whether a file was read.
-pub fn maybe_read_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> AnyhowResult<bool> {
-  Ok(do_read_from_filename_and_paths(filename, paths)?)
-}
-
-pub fn do_read_from_filename_and_paths<P: AsRef<Path>, Q: AsRef<Path>>(filename: P, paths: &[Q]) -> AnyhowResult<bool> {
-  for path in paths.iter() {
-    let path = path.as_ref().join(filename.as_ref());
-
-    if path.exists() && path.is_file() {
-      log::info!("Attempting to read env vars from file: {:?}", path);
-      let path = std::fs::canonicalize(path)?;
-      dotenv::from_path(path)?;
-      return Ok(true);
-    }
-  }
-
-  Ok(false)
-}

crates/lib/server_environment/Cargo.toml

@@ -16,7 +16,6 @@ path = "src/lib.rs"
 [dependencies]
 # Internal
 easyenv = { path = "../easyenv" }
-envvar = { path = "../envvar" }
 errors = { path = "../errors" }
 
 # External

crates/service/job/download_job/Cargo.toml

@@ -23,7 +23,6 @@ container_common = { path = "../../../lib/deprecated/container_common" }
 crockford = { path = "../../../lib/crockford" }
 easyenv = { path = "../../../lib/easyenv" }
 enums = { path = "../../../_types/enums" }
-envvar = { path = "../../../lib/envvar" }
 errors = { path = "../../../lib/errors" }
 filesys = { path = "../../../lib/files/filesys" }
 google_drive_common = { path = "../../../lib/api_clients/google_drive_common" }

crates/service/job/inference_job/Cargo.toml

@@ -24,7 +24,6 @@ config = { path = "../../../lib/deprecated/config" }
 container_common = { path = "../../../lib/deprecated/container_common" }
 easyenv = { path = "../../../lib/easyenv" }
 enums = { path = "../../../_types/enums" }
-envvar = { path = "../../../lib/envvar" }
 errors = { path = "../../../lib/errors" }
 filesys = { path = "../../../lib/files/filesys" }
 hashing = { path = "../../../lib/files/hashing" }

crates/service/web/storyteller_web/Cargo.toml

@@ -25,7 +25,6 @@ container_common = { path = "../../../lib/deprecated/container_common" }
 datetimes = { path = "../../../lib/datetimes" }
 easyenv = { path = "../../../lib/easyenv" }
 enums = { path = "../../../_types/enums" }
-envvar = { path = "../../../lib/envvar" }
 errors = { path = "../../../lib/errors" }
 google_drive_common = { path = "../../../lib/api_clients/google_drive_common" }
 hashing = { path = "../../../lib/files/hashing" }