mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-20 08:35:29 +00:00
197 lines
7.4 KiB
Rust
197 lines
7.4 KiB
Rust
use std::io::Write;
|
|
|
|
use chrono::Local;
|
|
use log::{info, warn, LevelFilter};
|
|
use sqlx::{MySql, Pool};
|
|
use sqlx::mysql::MySqlPoolOptions;
|
|
|
|
use easyenv::env_logger::Builder;
|
|
use easyenv::init_all_with_default_logging;
|
|
use errors::AnyhowResult;
|
|
|
|
use crate::args::{Command, parse_cli_args};
|
|
use crate::operations::backfill_user_spend_events::backfill_user_spend_events::backfill_user_spend_events;
|
|
use crate::operations::backfill_user_spend_events::stripe_api::StripeApi;
|
|
use crate::operations::backfill_user_spend_events::sub_args::parse_backfill_user_spend_events_args;
|
|
use crate::operations::calculate_legacy_tts_results_usages::calculate_legacy_tts_results_usages::calculate_legacy_tts_result_usages;
|
|
use crate::operations::calculate_model_weights_usages::run_migration::run_migration;
|
|
|
|
pub mod args;
|
|
pub mod operations;
|
|
|
|
//#[tokio::main(flavor = "multi_thread", worker_threads = 16)]
|
|
#[tokio::main]
|
|
async fn main() -> AnyhowResult<()> {
|
|
println!("db-backfill: run backfill or migration operations");
|
|
|
|
// rustls 0.23 requires a process-wide crypto provider before the first TLS
|
|
// handshake (the replica's `ssl-mode=required` connection and Stripe HTTPS).
|
|
// The dependency graph enables BOTH ring and aws-lc-rs, so rustls refuses to
|
|
// auto-select and panics — install one explicitly. (`Err` = already installed.)
|
|
let _ = rustls::crypto::ring::default_provider().install_default();
|
|
|
|
// init_all_with_default_logging(None);
|
|
Builder::new()
|
|
.format(|buf, record| {
|
|
writeln!(
|
|
buf,
|
|
"{} [{}] {}",
|
|
Local::now().format("%Y-%m-%dT%H:%M:%S"),
|
|
record.level(),
|
|
record.args()
|
|
)
|
|
})
|
|
.filter(None, LevelFilter::Info)
|
|
.init();
|
|
|
|
let command = parse_cli_args()?;
|
|
|
|
// NB: This secrets file differs from the rest because we might want to backfill production from local dev.
|
|
// (Hopefully this isn't getting out of hand at this point.)
|
|
// The path is resolved relative to the current working directory — log the
|
|
// absolute path so a "not found" is actionable.
|
|
let secrets_filename = ".env-db-backfill-secrets";
|
|
let secrets_path = std::env::current_dir()
|
|
.map(|dir| dir.join(secrets_filename))
|
|
.unwrap_or_else(|_| std::path::PathBuf::from(secrets_filename));
|
|
info!("Loading secrets from: {}", secrets_path.display());
|
|
// NB: We deliberately do NOT use `dotenv` here. Its parser treats `$`, spaces,
|
|
// and `\` as special INSIDE the value (quoted or not), which mangles DB
|
|
// connection URLs and generated passwords. This loader takes each value
|
|
// verbatim, so `MYSQL_READ_URL=mysql://user:p$ss w@host:25060/db?ssl-mode=required`
|
|
// just works.
|
|
load_secrets_file(&secrets_path)?;
|
|
|
|
info!("dispatching command: {:?}", command);
|
|
|
|
match command.sub_command {
|
|
Command::CalculateModelWeightsUsages => {
|
|
let mysql = get_mysql("MYSQL_PRODUCTION_URL").await?;
|
|
run_migration(mysql).await?
|
|
}
|
|
Command::CalculateLegacyTtsResultsUsages => {
|
|
let mysql = get_mysql("MYSQL_PRODUCTION_URL").await?;
|
|
calculate_legacy_tts_result_usages(mysql).await?
|
|
}
|
|
Command::BackfillUserSpendEvents => {
|
|
// Read pool = the replica (source); write pool = your target DB (local in
|
|
// a dry run, prod when you go live). Stripe key = the ArtCraft account.
|
|
let sub_args = parse_backfill_user_spend_events_args();
|
|
let read_pool = get_mysql("MYSQL_READ_URL").await?;
|
|
let write_pool = get_mysql("MYSQL_WRITE_URL").await?;
|
|
let stripe = StripeApi::new(easyenv::get_env_string_required("STRIPE_ARTCRAFT_SECRET_KEY")?)?;
|
|
backfill_user_spend_events(&read_pool, &write_pool, &stripe, sub_args).await?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Minimal, forgiving `.env`-style loader that sets process env vars from a
|
|
/// `KEY=VALUE` file. The value is the remainder of the line taken VERBATIM
|
|
/// (after one optional layer of surrounding quotes is stripped). Blank lines and
|
|
/// `#` comment lines are ignored. Unlike `dotenv`, it does not interpret `$`,
|
|
/// spaces, `\`, or `#` inside the value — so DB URLs and generated passwords
|
|
/// pass through unchanged.
|
|
fn load_secrets_file(path: &std::path::Path) -> AnyhowResult<()> {
|
|
let contents = std::fs::read_to_string(path)
|
|
.map_err(|err| anyhow::anyhow!("Could not read secrets file at {} : {}", path.display(), err))?;
|
|
|
|
let mut loaded_keys: Vec<String> = Vec::new();
|
|
for (line_index, raw_line) in contents.lines().enumerate() {
|
|
let line = raw_line.trim();
|
|
if line.is_empty() || line.starts_with('#') {
|
|
continue;
|
|
}
|
|
let (key, value) = match line.split_once('=') {
|
|
Some((key, value)) => (key.trim(), strip_matching_quotes(value.trim())),
|
|
None => {
|
|
warn!("secrets {}:{}: ignoring line without '=': {:?}", path.display(), line_index + 1, line);
|
|
continue;
|
|
}
|
|
};
|
|
if key.is_empty() {
|
|
warn!("secrets {}:{}: ignoring empty key", path.display(), line_index + 1);
|
|
continue;
|
|
}
|
|
std::env::set_var(key, value);
|
|
loaded_keys.push(key.to_string());
|
|
}
|
|
|
|
info!(
|
|
"Loaded {} key(s) from {} ({} bytes): [{}]",
|
|
loaded_keys.len(),
|
|
path.display(),
|
|
contents.len(),
|
|
loaded_keys.join(", "),
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod secrets_loader_tests {
|
|
use super::strip_matching_quotes;
|
|
|
|
// The whole point: a DB URL with a password containing `$`, a space, `#`, and a
|
|
// query string with `=` must survive verbatim (this is exactly what `dotenv` mangled).
|
|
#[test]
|
|
fn url_value_is_taken_verbatim() {
|
|
let line = r#"MYSQL_READ_URL=mysql://user:p$ss w#x@host:25060/db?ssl-mode=required"#;
|
|
let (key, value) = line.split_once('=').unwrap();
|
|
assert_eq!(key.trim(), "MYSQL_READ_URL");
|
|
assert_eq!(
|
|
strip_matching_quotes(value.trim()),
|
|
"mysql://user:p$ss w#x@host:25060/db?ssl-mode=required",
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn strips_one_layer_of_matching_quotes_only() {
|
|
assert_eq!(strip_matching_quotes("\"abc\""), "abc");
|
|
assert_eq!(strip_matching_quotes("'abc'"), "abc");
|
|
assert_eq!(strip_matching_quotes("abc"), "abc");
|
|
assert_eq!(strip_matching_quotes("\"abc'"), "\"abc'"); // mismatched: untouched
|
|
assert_eq!(strip_matching_quotes("\""), "\""); // single char: untouched
|
|
}
|
|
}
|
|
|
|
/// Strip a single layer of matching surrounding single or double quotes, if any.
|
|
fn strip_matching_quotes(value: &str) -> &str {
|
|
let bytes = value.as_bytes();
|
|
if bytes.len() >= 2 {
|
|
let first = bytes[0];
|
|
let last = bytes[bytes.len() - 1];
|
|
if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') {
|
|
return &value[1..value.len() - 1];
|
|
}
|
|
}
|
|
value
|
|
}
|
|
|
|
async fn get_mysql(env_var_name: &str) -> AnyhowResult<Pool<MySql>> {
|
|
let url = easyenv::get_env_string_required(env_var_name)?;
|
|
// Redacted so you can SEE the host + database + params actually being used
|
|
// (e.g. whether a database path is present) without leaking the password.
|
|
info!("Connecting to MySQL {env_var_name}: {}", redact_db_url(&url));
|
|
|
|
let pool = MySqlPoolOptions::new()
|
|
.max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 20)?)
|
|
.connect(&url)
|
|
.await?;
|
|
|
|
Ok(pool)
|
|
}
|
|
|
|
/// Mask the user:password portion of a DB URL for logging, leaving scheme, host,
|
|
/// port, database path, and query params visible.
|
|
fn redact_db_url(url: &str) -> String {
|
|
if let Some(scheme_end) = url.find("://") {
|
|
let after = &url[scheme_end + 3..];
|
|
if let Some(at) = after.rfind('@') {
|
|
return format!("{}://***@{}", &url[..scheme_end], &after[at + 1..]);
|
|
}
|
|
}
|
|
url.to_string()
|
|
}
|