From ec1257ccf62f9582bfffbb3696a05d9e1e8e753c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jul 2025 20:26:57 -0400 Subject: [PATCH] fix sqlx sqlite build for dev and CI (#716) --- .github/workflows/artcraft-macos-publish.yml | 2 + .../workflows/artcraft-windows-publish.yml | 2 + ...b66a60a985acaa1ee5753f30f5e6e107e080e.json | 12 ++ crates/desktop/artcraft/build.rs | 2 - crates/schema/database/sqlite_tasks/build.rs | 44 +------ .../sqlite_tasks/src/queries/create_task.rs | 4 +- script/artcraft/unix_rust_dev.sh | 21 +++- script/rust/sqlx_codegen_database.sh | 118 ++++++++++++++---- 8 files changed, 132 insertions(+), 73 deletions(-) create mode 100644 .sqlx/query-0848cd9ddf8f61f2dd4d57f0c58b66a60a985acaa1ee5753f30f5e6e107e080e.json diff --git a/.github/workflows/artcraft-macos-publish.yml b/.github/workflows/artcraft-macos-publish.yml index aff78f71be..8b17ca782f 100644 --- a/.github/workflows/artcraft-macos-publish.yml +++ b/.github/workflows/artcraft-macos-publish.yml @@ -220,6 +220,8 @@ jobs: APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} APPLE_SIGNING_IDENTITY: ${{ env.APPLE_SIGNING_IDENTITY }} APPLE_DEVELOPMENT_TEAM: ${{ env.APPLE_DEVELOPMENT_TEAM }} + # Use the SQLx query cache for SQLite + SQLX_OFFLINE: true with: projectPath: ./crates/desktop/artcraft tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version. diff --git a/.github/workflows/artcraft-windows-publish.yml b/.github/workflows/artcraft-windows-publish.yml index 9e1cc3831c..3dd9f6bd1a 100644 --- a/.github/workflows/artcraft-windows-publish.yml +++ b/.github/workflows/artcraft-windows-publish.yml @@ -84,6 +84,8 @@ jobs: uses: tauri-apps/tauri-action@v0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Use the SQLx query cache for SQLite + SQLX_OFFLINE: true with: projectPath: ./crates/desktop/artcraft tagName: artcraft-windows-v__VERSION__ # the action automatically replaces __VERSION__ with the app version. diff --git a/.sqlx/query-0848cd9ddf8f61f2dd4d57f0c58b66a60a985acaa1ee5753f30f5e6e107e080e.json b/.sqlx/query-0848cd9ddf8f61f2dd4d57f0c58b66a60a985acaa1ee5753f30f5e6e107e080e.json new file mode 100644 index 0000000000..3a77f3e997 --- /dev/null +++ b/.sqlx/query-0848cd9ddf8f61f2dd4d57f0c58b66a60a985acaa1ee5753f30f5e6e107e080e.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO tasks (\n id,\n task_status,\n task_type,\n provider,\n provider_job_id,\n frontend_subscriber_id,\n frontend_subscriber_payload\n )\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + }, + "hash": "0848cd9ddf8f61f2dd4d57f0c58b66a60a985acaa1ee5753f30f5e6e107e080e" +} diff --git a/crates/desktop/artcraft/build.rs b/crates/desktop/artcraft/build.rs index e6358f18c0..795b9b7c83 100644 --- a/crates/desktop/artcraft/build.rs +++ b/crates/desktop/artcraft/build.rs @@ -1,5 +1,3 @@ fn main() { - // For SQLx - std::env::set_var("DATABASE_URL", format!("sqlite:///Users/bt/Artcraft/state/tasks_v1.sqlite")); tauri_build::build() } diff --git a/crates/schema/database/sqlite_tasks/build.rs b/crates/schema/database/sqlite_tasks/build.rs index bdc637a1a7..e07fe0d245 100644 --- a/crates/schema/database/sqlite_tasks/build.rs +++ b/crates/schema/database/sqlite_tasks/build.rs @@ -1,41 +1,9 @@ - -use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode}; -use sqlx::SqlitePool; - pub fn main() { + // NB: This should help JetBrains' RustRover from highlighting failing query macros, + // but we do not want to interfere with the following: + // + // (1) User development builds + // (2) Tooling to cache offline queries + // (3) CI builds println!("cargo:rustc-env=DATABASE_URL=sqlite:/tmp/tasks.sqlite"); - - // // From: https://github.com/launchbadge/sqlx/issues/121#issuecomment-1913373602 - // // Create a new temporary file. We need to `.keep()` it so that it isn't - // // deleted when it goes out of scope at the end of this function. - // let (_, tempfile) = tempfile::NamedTempFile::new().unwrap().keep().unwrap(); - // let database_path = tempfile.as_path().to_str().unwrap(); - - // let schema_file = "../schema.sql"; - - // // Ensure that the build script is rerun if schema.sql changes - // println!("cargo:rerun-if-changed={schema_file}"); - - // // Execute SQL commands from schema.sql on the temporary database - // //rusqlite::Connection::open(database_path) - // // .unwrap() - // // .execute_batch(&std::fs::read_to_string(schema_file).unwrap()) - // // .unwrap(); - - // //let connection_options = SqliteConnectOptions::new() - // // .filename(schema_file) - // // .create_if_missing(true) - // // .journal_mode(SqliteJournalMode::Wal); - - // //let pool = SqlitePool::connect_lazy_with(connection_options); - - // //// Run migrations regardless of whether the database is new, SQLx will track which migrations - // //// have been run. - // //// The migrations text get compiled into the binary, so no worries about build inclusion. - // //// Since the task database is being treated as ephemeral, we can always run migrations without - // //// worrying about previous state if we simply blow away old versions of the schema. - // //sqlx::migrate!("../../../../_sql/artcraft_migrations"). - - // // Prepare the DATABASE_URL in the format that sqlx expects for SQLite - // println!("cargo:rustc-env=DATABASE_URL=sqlite:{database_path}"); } \ No newline at end of file diff --git a/crates/schema/database/sqlite_tasks/src/queries/create_task.rs b/crates/schema/database/sqlite_tasks/src/queries/create_task.rs index 902b6c7fe3..94323eeaf8 100644 --- a/crates/schema/database/sqlite_tasks/src/queries/create_task.rs +++ b/crates/schema/database/sqlite_tasks/src/queries/create_task.rs @@ -28,7 +28,7 @@ pub async fn create_task( let query = sqlx::query!(r#" INSERT INTO tasks ( - id, + id, task_status, task_type, provider, @@ -46,7 +46,7 @@ pub async fn create_task( args.frontend_subscriber_id, args.frontend_subscriber_payload ); - + let _r = query.execute(args.db.get_pool()).await?; Ok(task_id) diff --git a/script/artcraft/unix_rust_dev.sh b/script/artcraft/unix_rust_dev.sh index cc23003b70..6d1b9c4e8f 100755 --- a/script/artcraft/unix_rust_dev.sh +++ b/script/artcraft/unix_rust_dev.sh @@ -27,10 +27,23 @@ export TAURI_APP_PATH="${rust_crate_path}" echo "Create database file..." touch "${sqlite_db_file}" -echo "Migrate database file..." -sqlx migrate run \ - --database-url "sqlite:${sqlite_db_file}" \ - --source "${root_dir}/_sql/artcraft_migrations" +sqlx_migrate() { + echo "SQLx migrate database file..." + sqlx migrate run \ + --database-url "sqlite:${sqlite_db_file}" \ + --source "${root_dir}/_sql/artcraft_migrations" + + export DATABASE_URL="sqlite:${sqlite_db_file}" +} + +sqlx_offline() { + echo "SQLx offline..." + export SQLX_OFFLINE=true +} + +# NB: Switch this for actively developing queries. +#sqlx_migrate +sqlx_offline echo "Build and run..." RUSTFLAGS="-Awarnings" cargo tauri dev \ diff --git a/script/rust/sqlx_codegen_database.sh b/script/rust/sqlx_codegen_database.sh index d111def0a4..5cb81ee8c0 100755 --- a/script/rust/sqlx_codegen_database.sh +++ b/script/rust/sqlx_codegen_database.sh @@ -5,38 +5,102 @@ set -euxo pipefail -# NB: Database used by desktop app for AiChatBot sidecar. -# Make sure it has been migrated (use aichatbot-sidecar app directory as root). -# SQLITE_DATABASE_PATH=./runtime_data/database.db +# TODO: Fix up root dir determination +root_dir=$(pwd) +sqlite_db_file="/tmp/tasks.sqlite" -# NB: OpenSSL is broken on Ubuntu 22.04 (this is documented elsewhere) -# See https://askubuntu.com/a/1403961 -# For some reason, the choice of "rustls" or the vendored openssl is not persistent between calls. -#cargo install sqlx-cli --features openssl-vendored,mysql +sqlite_package_path="${root_dir}/crates/schema/database/sqlite_tasks" +mysql_package_path="${root_dir}/crates/schema/database/mysql_queries" -build_shared_mysql_database_library() { - # NB: For now, this is our monolithic DB that serves all of our microservices (gross) - # It's a single package so that queries can be shared (again, gross) - #pushd crates/lib/mysql_queries - #SQLX_OFFLINE=true cargo sqlx prepare - #popd - cargo sqlx prepare --workspace +query_cache_dir="${root_dir}/.sqlx/" + +remove_old_query_cache() { + echo "Removing old query cache files..." + pushd "${query_cache_dir}" + rm -f *.json + popd } -#build_shared_sqlite_database_library() { -# # NB(1): These are *SQLite* queries for the desktop AiChatBot app. -# # NB(2): Make sure to run migrations first. -# # NB(3): SqLite has trouble with relative paths, so we copy the file to / -# cp $SQLITE_DATABASE_PATH /tmp -# pushd crates/lib/sqlite_queries -# DATABASE_URL=sqlite:///tmp/database.db cargo sqlx prepare --merged -# popd -#} +prepare_sqlite_tasks() { + echo 'Create Tauri SQLite tasks database file...' + touch "${sqlite_db_file}" -echo 'mysql prepare (if this takes a while, it may need to rebuild all the rust code)' -build_shared_mysql_database_library + echo "Migrate Tauri SQLite tasks database file..." + cargo sqlx migrate run \ + --database-url "sqlite:${sqlite_db_file}" \ + --source "${root_dir}/_sql/artcraft_migrations" -#echo 'sqlite prepare' -#build_shared_sqlite_database_library + echo "Prepare Tauri SQLite tasks query cache..." + pushd "${sqlite_package_path}" + cargo sqlx prepare \ + --database-url "sqlite:${sqlite_db_file}" + popd + + echo "Move query cache files..." + pushd "${sqlite_package_path}/.sqlx" + mv *.json "${query_cache_dir}" + popd +} + +prepare_mysql() { + echo "Prepare MySQL database query cache..." + pushd "${mysql_package_path}" + cargo sqlx prepare + popd + + echo "Move query cache files..." + pushd "${mysql_package_path}/.sqlx" + mv *.json "${query_cache_dir}" + popd +} + +remove_old_query_cache +prepare_sqlite_tasks +prepare_mysql + +# NB(bt): Below are several generational revisions of sqlx query caching. +# The most recent was for a workspace with only one sqlx query package. +# There are older remnants from a multi-package configuration, but I believe +# the tooling has improved since then. Keeping this here just in case. +# +# #export DATABASE_URL="sqlite:${sqlite_db_file}" +# +# # NB: Database used by desktop app for AiChatBot sidecar. +# # Make sure it has been migrated (use aichatbot-sidecar app directory as root). +# # SQLITE_DATABASE_PATH=./runtime_data/database.db +# +# # NB: OpenSSL is broken on Ubuntu 22.04 (this is documented elsewhere) +# # See https://askubuntu.com/a/1403961 +# # For some reason, the choice of "rustls" or the vendored openssl is not persistent between calls. +# #cargo install sqlx-cli --features openssl-vendored,mysql +# +# build_shared_mysql_database_library() { +# +# # NB: For now, this is our monolithic DB that serves all of our microservices (gross) +# +# # It's a single package so that queries can be shared (again, gross) +# #pushd crates/lib/mysql_queries +# #SQLX_OFFLINE=true cargo sqlx prepare +# #popd +# cargo sqlx prepare --workspace +# } +# +# #build_shared_sqlite_database_library() { +# # # NB(1): These are *SQLite* queries for the desktop AiChatBot app. +# # # NB(2): Make sure to run migrations first. +# +# # # NB(3): SqLite has trouble with relative paths, so we copy the file to / +# # cp $SQLITE_DATABASE_PATH /tmp +# # pushd crates/lib/sqlite_queries +# # DATABASE_URL=sqlite:///tmp/database.db cargo sqlx prepare --merged +# # popd +# #} +# +# +# echo 'mysql prepare (if this takes a while, it may need to rebuild all the rust code)' +# build_shared_mysql_database_library +# +# #echo 'sqlite prepare' +# #build_shared_sqlite_database_library echo 'done'