From 3c5b7cc9cb371669ba17afc1582bfff972e8fdfd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 14 Feb 2026 07:16:44 +0700 Subject: [PATCH] 2 spaces --- .gitignore | 3 + script/database/migrate.py | 134 ++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 67 deletions(-) mode change 100644 => 100755 script/database/migrate.py diff --git a/.gitignore b/.gitignore index 038ff5c676..3f50bd4eda 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ script/production_database_secret_backup.sh script/production_database_secret_connect.sh script/socat_dev_database_connect.sh +# Python virtual envs (named by convention) +venv/ + # Scratch data scratch/ test_data/temp/ diff --git a/script/database/migrate.py b/script/database/migrate.py old mode 100644 new mode 100755 index 40acc8249b..5ef8cb164b --- a/script/database/migrate.py +++ b/script/database/migrate.py @@ -4,11 +4,11 @@ Database migration script for running queries in a loop with failure fallback. Reads MySQL credentials from a TOML secrets file. Requirements: - pip install pymysql + pip install pymysql Secrets file (script/database/secrets.toml): - [database] - url = "mysql://user:password@host:3306/dbname" + [database] + url = "mysql://user:password@host:3306/dbname" """ import sys @@ -18,13 +18,13 @@ from pathlib import Path from urllib.parse import urlparse try: - import pymysql + import pymysql except ImportError: - print("pymysql is required: pip install pymysql") - sys.exit(1) + print("pymysql is required: pip install pymysql") + sys.exit(1) -SECRETS_PATH = Path(__file__).parent / "secrets.toml" +SECRETS_PATH = "secrets.toml" # -- Migration query to run in batches -- # Tweak this as needed. Use LIMIT to control batch size. @@ -40,82 +40,82 @@ FAILURE_BACKOFF_SECONDS = 5 def load_config(): - with open(SECRETS_PATH, "rb") as f: - config = tomllib.load(f) - url = config["database"]["url"] - parsed = urlparse(url) - return { - "host": parsed.hostname, - "port": parsed.port or 3306, - "user": parsed.username, - "password": parsed.password, - "database": parsed.path.lstrip("/"), - } + with open(SECRETS_PATH, "rb") as f: + config = tomllib.load(f) + url = config["database"]["url"] + parsed = urlparse(url) + return { + "host": parsed.hostname, + "port": parsed.port or 3306, + "user": parsed.username, + "password": parsed.password, + "database": parsed.path.lstrip("/"), + } def connect(config): - return pymysql.connect( - host=config["host"], - port=config["port"], - user=config["user"], - password=config["password"], - database=config["database"], - connect_timeout=10, - read_timeout=30, - write_timeout=30, - autocommit=True, - ) + return pymysql.connect( + host=config["host"], + port=config["port"], + user=config["user"], + password=config["password"], + database=config["database"], + connect_timeout=10, + read_timeout=30, + write_timeout=30, + autocommit=True, + ) def run_migration(): - config = load_config() - conn = connect(config) - total_affected = 0 - failures = 0 + config = load_config() + conn = connect(config) + total_affected = 0 + failures = 0 - print(f"Connected to {config['host']}/{config['database']}") - print(f"Running query:\n{QUERY.strip()}\n") + print(f"Connected to {config['host']}/{config['database']}") + print(f"Running query:\n{QUERY.strip()}\n") - try: - while True: - try: - with conn.cursor() as cursor: - cursor.execute(QUERY) - affected = cursor.rowcount + try: + while True: + try: + with conn.cursor() as cursor: + cursor.execute(QUERY) + affected = cursor.rowcount - total_affected += affected - failures = 0 - print(f" Batch: {affected} rows | Total: {total_affected}") + total_affected += affected + failures = 0 + print(f" Batch: {affected} rows | Total: {total_affected}") - if affected == 0: - print("No more rows to process. Done.") - break + if affected == 0: + print("No more rows to process. Done.") + break - time.sleep(BATCH_PAUSE_SECONDS) + time.sleep(BATCH_PAUSE_SECONDS) - except (pymysql.OperationalError, pymysql.InterfaceError) as e: - failures += 1 - print(f" Error ({failures}/{MAX_CONSECUTIVE_FAILURES}): {e}") + except (pymysql.OperationalError, pymysql.InterfaceError) as e: + failures += 1 + print(f" Error ({failures}/{MAX_CONSECUTIVE_FAILURES}): {e}") - if failures >= MAX_CONSECUTIVE_FAILURES: - print("Too many consecutive failures. Aborting.") - sys.exit(1) + if failures >= MAX_CONSECUTIVE_FAILURES: + print("Too many consecutive failures. Aborting.") + sys.exit(1) - print(f" Reconnecting in {FAILURE_BACKOFF_SECONDS}s...") - time.sleep(FAILURE_BACKOFF_SECONDS) - try: - conn.close() - except Exception: - pass - conn = connect(config) + print(f" Reconnecting in {FAILURE_BACKOFF_SECONDS}s...") + time.sleep(FAILURE_BACKOFF_SECONDS) + try: + conn.close() + except Exception: + pass + conn = connect(config) - except KeyboardInterrupt: - print(f"\nInterrupted. Total rows affected: {total_affected}") - finally: - conn.close() + except KeyboardInterrupt: + print(f"\nInterrupted. Total rows affected: {total_affected}") + finally: + conn.close() - print(f"Finished. Total rows affected: {total_affected}") + print(f"Finished. Total rows affected: {total_affected}") if __name__ == "__main__": - run_migration() + run_migration()