Merge pull request #7141 from FredApps/fix-openssl-ia32cap

fix(startup): an OPENSSL_ia32cap naming a missing CPU feature no longer stops PdfCraft (#2916)

.cargo/config.toml

@@ -5,3 +5,11 @@ xtask = "run -p xtask --"
 # engine's kernels (rten-simd) and other vectorised code only use it when compiled in.
 [target.wasm32-unknown-unknown]
 rustflags = ["-C", "target-feature=+simd128"]
+
+# aws-lc (digital signatures and TLS, through aws-lc-rs) detects the CPU when `main` asks it to
+# (`pdfcraft_sign::init_crypto`), not in a static initializer before `main`. On x86 it reads
+# `OPENSSL_ia32cap` then, and aborts when that asks for a feature the processor lacks; each
+# executable first runs itself again without the variable (#2916, `pdfcraft_platform::crypto_env`),
+# which it can only do once `main` runs.
+[env]
+AWS_LC_SYS_CFLAGS = "-DBORINGSSL_NO_STATIC_INITIALIZER"

apps/pdfcraft-cli/src/main.rs

@@ -98,6 +98,13 @@ help and feedback: https://discord.gg/artcraft"
 );
 
 fn main() -> ExitCode {
+    // Before anything uses aws-lc: an `OPENSSL_ia32cap` naming a CPU feature this processor lacks
+    // would stop it, so the CLI runs again without it (#2916, `pdfcraft_platform::crypto_env`).
+    if let Some(code) = pdfcraft_platform::crypto_env::run_without_cpu_override() {
+        std::process::exit(code);
+    }
+    // aws-lc detects the CPU's features here, where it used to before `main` (`init_crypto`).
+    pdfcraft_engine::sign::init_crypto();
     // `std::env::args` panics on an argument that is not UTF-8 ("called `Result::unwrap()` on an
     // `Err` value"), which a Latin-1 file name is, and it panics while collecting them: even
     // `--help` ended that way. Such a name cannot be passed on either, since the paths here are

apps/pdfcraft-cli/tests/it/cpu_override.rs

@@ -0,0 +1,33 @@
+//! An `OPENSSL_ia32cap` that asks for a CPU feature the processor lacks stops neither the CLI nor
+//! its signing (#2916). aws-lc printed "Fatal Error: HW capability found: …" and aborted before
+//! `main`. Built to detect the CPU on first use, it would abort at the first signature instead.
+
+// macOS intentionally does not relaunch: Finder delivers files to the original process.
+#![cfg(not(target_os = "macos"))]
+
+use std::process::{Command, Output};
+
+/// CPUID leaf 1, EDX bit 10: reserved, so no x86 processor reports it.
+const MISSING_FEATURE: &str = "0x400";
+
+fn run(args: &[&str]) -> Output {
+    Command::new(env!("CARGO_BIN_EXE_pdfcraft-cli")).args(args).env("OPENSSL_ia32cap", MISSING_FEATURE).output().unwrap()
+}
+
+#[test]
+fn a_cpu_override_naming_a_missing_feature_stops_neither_the_cli_nor_signing() {
+    let out = run(&["--help"]);
+    assert!(out.status.success(), "--help: {:?} {}", out.status, String::from_utf8_lossy(&out.stderr));
+    assert!(String::from_utf8_lossy(&out.stdout).contains("pdfcraft-cli render"));
+
+    let dir = std::env::temp_dir().join(format!("pdfcraft-cli-cpu-override-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    // A new RSA key: the CLI's first use of aws-lc.
+    let id = dir.join("id.p12");
+    let path = format!("path={}", id.display());
+    let out = run(&["run", "sign_id_create", "name=Test", "password=correct horse", &path, "--root", dir.to_str().unwrap()]);
+    assert!(out.status.success(), "sign_id_create: {:?} {}", out.status, String::from_utf8_lossy(&out.stderr));
+    assert!(std::fs::metadata(&id).is_ok_and(|m| m.len() > 0), "no digital ID at {}", id.display());
+    let _ = std::fs::remove_dir_all(&dir);
+}

apps/pdfcraft-cli/tests/it/main.rs

@@ -5,6 +5,7 @@ mod automation_roots;
 mod batch;
 mod batch_exit;
 mod check;
+mod cpu_override;
 mod edit;
 mod extract;
 mod help;

apps/pdfcraft/src/main.rs

@@ -201,6 +201,13 @@ fn rejoin_split_paths(args: Vec<String>, exists: impl Fn(&str) -> bool) -> Vec<S
 }
 
 fn main() -> eframe::Result {
+    // Before anything uses aws-lc: an `OPENSSL_ia32cap` naming a CPU feature this processor lacks
+    // would stop it, so PdfCraft runs again without it (#2916, `crypto_env`).
+    if let Some(code) = pdfcraft_platform::crypto_env::run_without_cpu_override() {
+        std::process::exit(code);
+    }
+    // aws-lc detects the CPU's features here, where it used to before `main` (`init_crypto`).
+    pdfcraft_engine::sign::init_crypto();
     // Windows: the update helper is a copy of this program started with `--finish-update`.
     #[cfg(windows)]
     if let Some(result) = craft_update::self_update::finish(&pdfcraft_ui_egui::UPDATE_APP, &std::env::args_os().skip(1).collect::<Vec<_>>()) {
@@ -208,6 +215,10 @@ fn main() -> eframe::Result {
     }
     // First, so the panic hook and every start-up warning are recorded (`logging`).
     let logger = logging::install();
+    if let Some(value) = pdfcraft_platform::crypto_env::dropped() {
+        let name = pdfcraft_platform::crypto_env::IA32CAP;
+        log::warn!("{name} was set to {value}; PdfCraft started again without it (aws-lc stops on a CPU feature the processor lacks)");
+    }
     // Last-resort guard (AGENTS.md §4): commands, edits, opens and saves catch panics and report
     // them; this hook logs every panic, caught or not, with a backtrace when RUST_BACKTRACE is set.
     std::panic::set_hook(Box::new(|info| {

apps/pdfcraft/tests/help.rs

@@ -8,7 +8,12 @@ const BIN: &str = env!("CARGO_BIN_EXE_pdfcraft");
 
 /// Run the app with `args`, failing if it is still running after 60 s (it opened its window).
 fn run(args: &[&str], stdout: Stdio) -> Output {
-    let mut child = Command::new(BIN).args(args).stdin(Stdio::null()).stdout(stdout).stderr(Stdio::piped()).spawn().unwrap();
+    run_with(Command::new(BIN).args(args), stdout)
+}
+
+fn run_with(command: &mut Command, stdout: Stdio) -> Output {
+    let args: Vec<_> = command.get_args().map(|a| a.to_string_lossy().into_owned()).collect();
+    let mut child = command.stdin(Stdio::null()).stdout(stdout).stderr(Stdio::piped()).spawn().unwrap();
     let deadline = Instant::now() + Duration::from_secs(60);
     while child.try_wait().unwrap().is_none() {
         if Instant::now() > deadline {
@@ -42,6 +47,18 @@ fn version_prints_the_version_and_exits() {
     assert!(String::from_utf8_lossy(&out.stdout).starts_with("pdfcraft "), "{}", String::from_utf8_lossy(&out.stdout));
 }
 
+/// An `OPENSSL_ia32cap` that asks for a CPU feature the processor lacks (CPUID leaf 1, EDX bit
+/// 10, reserved on every x86) doesn't stop the app (#2916): aws-lc aborted before `main`, with no
+/// window and nothing in the log.
+// macOS intentionally does not relaunch: Finder delivers files to the original process.
+#[cfg(not(target_os = "macos"))]
+#[test]
+fn a_cpu_override_naming_a_missing_feature_does_not_stop_the_app() {
+    let out = run_with(Command::new(BIN).arg("--version").env("OPENSSL_ia32cap", "0x400"), Stdio::piped());
+    assert!(out.status.success(), "{:?} {}", out.status, String::from_utf8_lossy(&out.stderr));
+    assert!(String::from_utf8_lossy(&out.stdout).starts_with("pdfcraft "), "{}", String::from_utf8_lossy(&out.stdout));
+}
+
 /// A standard output that can't be written (here a full device) ends the run normally: `println!`
 /// would panic and exit 101.
 #[cfg(target_os = "linux")]

crates/platform/src/crypto_env.rs

@@ -0,0 +1,67 @@
+//! An `OPENSSL_ia32cap` left in the environment must not stop PdfCraft (#2916).
+//!
+//! aws-lc, the cryptography library behind digital signatures (through `aws-lc-rs`) and TLS,
+//! reads `OPENSSL_ia32cap` when it detects the CPU's features on x86 and x86-64. A value that
+//! asks for a feature the processor doesn't have makes it print "Fatal Error: HW capability
+//! found: …, but HW capability requested: …" and call `abort()`. On Windows that is exit code
+//! 0xC0000409 with no window, no message and nothing in the log. Tuning guides and other
+//! programs leave the variable set for every program, so PdfCraft never started on those PCs
+//! while other apps did.
+//!
+//! Two parts keep PdfCraft running:
+//! - `.cargo/config.toml` builds aws-lc with `BORINGSSL_NO_STATIC_INITIALIZER`, so it detects
+//!   the CPU when `CRYPTO_library_init` is called instead of in a static initializer before
+//!   `main`.
+//! - [`run_without_cpu_override`], the first thing each executable's `main` does, runs the
+//!   program again without the variable. Then `main` calls `pdfcraft_sign::init_crypto`, so the
+//!   CPU is detected at start as before, and every operation gets its fast code.
+//!
+//! Removing the variable from this process instead wouldn't work: aws-lc reads it through the C
+//! runtime's `getenv`, whose copy of the environment `std::env::remove_var` doesn't change on
+//! Windows, and the workspace has no `unsafe` to call the C runtime with.
+
+use std::ffi::OsString;
+use std::io::Write as _;
+
+/// The variable aws-lc reads on x86 and x86-64.
+pub const IA32CAP: &str = "OPENSSL_ia32cap";
+
+/// Set in the program started again to the value it was started without, for its log.
+pub const DROPPED: &str = "PDFCRAFT_DROPPED_OPENSSL_IA32CAP";
+
+/// When `OPENSSL_ia32cap` is set: run this executable again, with the same arguments and
+/// standard streams but without the variable, and return its exit code once it ends.
+///
+/// `None` when the variable isn't set, on processors aws-lc doesn't read it for (not x86), on
+/// macOS (Finder hands files to the process it started, through Apple events, so a second one
+/// would never get them), or when the program can't be started again. Then the caller carries
+/// on: a value that asks for a missing CPU feature still stops it, at its first use of aws-lc.
+pub fn run_without_cpu_override() -> Option<i32> {
+    if !cfg!(any(target_arch = "x86", target_arch = "x86_64")) || cfg!(target_os = "macos") {
+        return None;
+    }
+    let value = std::env::var_os(IA32CAP)?;
+    let run = || {
+        let exe = std::env::current_exe()?;
+        std::process::Command::new(exe).args(std::env::args_os().skip(1)).env_remove(IA32CAP).env(DROPPED, &value).status()
+    };
+    match run() {
+        // A process ended by a signal (Unix) has no code: report a failure.
+        Ok(status) => Some(status.code().unwrap_or(1)),
+        Err(e) => {
+            // No logger yet; unlike `eprintln!`, this never panics on a closed standard error.
+            let _ = writeln!(std::io::stderr(), "{IA32CAP} is set to {}, and starting again without it failed: {e}", shown(&value));
+            None
+        }
+    }
+}
+
+/// The value this program was started without, when [`run_without_cpu_override`] started it.
+pub fn dropped() -> Option<String> {
+    std::env::var_os(DROPPED).map(|v| shown(&v))
+}
+
+/// A value for a message: lossy where it isn't Unicode, and short.
+fn shown(value: &OsString) -> String {
+    value.to_string_lossy().chars().take(64).collect()
+}

crates/platform/src/lib.rs

@@ -1,9 +1,11 @@
 //! pdfcraft-platform — L7 operating-system services shared by the desktop app (`ui-egui`) and the
 //! automation tools (`automation`), which sit on the same layer and can't depend on each other,
-//! and by the two executables (`owner_only`, for the `--control` file).
+//! and by the two executables (`owner_only`, for the `--control` file; `crypto_env`, before
+//! anything uses aws-lc).
 
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 
+pub mod crypto_env;
 pub mod desktop_theme;
 pub mod owner_only;
 pub mod staging;

crates/sign/src/lib.rs

@@ -37,6 +37,16 @@ pub use status_layers::{StatusCaptions, has_status_layers, show_status_layers, s
 pub use timestamp::{TimestampAuthority, TimestampQuery, TimestampToken};
 pub use x509::{Certificate, Name};
 
+/// Detect the CPU's features for aws-lc now. This build has it do so on first use rather than in
+/// a static initializer before `main` (`.cargo/config.toml`, `pdfcraft_platform::crypto_env`), and
+/// aws-lc-rs asks for it before most operations but not all: an RSA key generation went without,
+/// and ran on aws-lc's portable code. So each executable calls this once at start, after
+/// `crypto_env` has run it again without an `OPENSSL_ia32cap` that would stop it here.
+#[cfg(not(target_arch = "wasm32"))]
+pub fn init_crypto() {
+    aws_lc_rs::init();
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum SignError {
     #[error("malformed data: {0}")]