Integration: bound the nvidia-smi query, smallest of same-id cards (#9042)

nvidia_video_memory ran `nvidia-smi` with Command::output(), which has no
deadline. It runs on the UI thread at startup, under the video-memory cache
lock, so a driver that never answers the query kept the window from opening.
It now runs with a 2 s deadline (stdout read on a helper thread, try_wait
polling), and a late process is killed and reaped on a detached thread.

parse_nvidia_smi took the first line naming the vendor/device id. Two cards
of one model with different memory (`0x2C0510DE, 16303` then
`0x2C0510DE, 8192`) gave 16 GB though wgpu may be on the 8 GB one; it now
takes the smallest.

Co-authored-by: Vishnu Vardhan Varma Byrraju <112673081+VishnuByrraju@users.noreply.github.com>
Co-authored-by: Khawar Zamman Wani <28915582+wanikhawar@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
committed 2026-10-27 02:02:57 +01:00
1 parent d0b08abc65
commit ffd4d480bb
1 file changed
+129 -19
+129 -19
View File
@@ -1505,11 +1505,11 @@ fn detect_physical_memory() -> Option<u64> {
}
/// The memory of `adapter` itself, in bytes, when it is a discrete GPU and the platform reports
/// it: DXGI's dedicated video memory on Windows, amdgpu's VRAM size on Linux. `None` for
/// integrated and software adapters, whose textures live in system memory (the RAM-based
/// budgets already cover them), and where nothing reports it. `PHOTOCRAFT_VRAM_MB` overrides it
/// for any adapter, simulating a discrete card of that size: on an integrated GPU the budget then
/// stops charging the document's pixels too, as for a real card (a development switch).
/// it: DXGI's dedicated video memory on Windows, amdgpu's VRAM size or `nvidia-smi`'s report on
/// Linux. `None` for integrated and software adapters, whose textures live in system memory (the
/// RAM-based budgets already cover them), and where nothing reports it. `PHOTOCRAFT_VRAM_MB`
/// overrides it for any adapter, simulating a discrete card of that size: on an integrated GPU the
/// budget then stops charging the document's pixels too, as for a real card (a development switch).
pub fn video_memory(adapter: &wgpu::AdapterInfo) -> Option<u64> {
if let Some(mb) = std::env::var("PHOTOCRAFT_VRAM_MB").ok().and_then(|v| v.trim().parse::<u64>().ok()) {
return mb.checked_mul(1 << 20).filter(|b| *b > 0);
@@ -1558,32 +1558,91 @@ fn detect_video_memory(vendor: u32, device: u32) -> Option<u64> {
#[cfg(any(target_os = "linux", test))]
const NVIDIA_VENDOR: u32 = 0x10de;
/// How long `nvidia-smi` gets to answer: it runs on the UI thread while the window opens, and a
/// driver that hangs the query must not keep PhotoCraft from starting.
#[cfg(target_os = "linux")]
const NVIDIA_SMI_DEADLINE: std::time::Duration = std::time::Duration::from_secs(2);
/// NVIDIA's own memory size on Linux, from `nvidia-smi` (installed with the driver that put this
/// adapter there in the first place): absent, slow to answer or odd output all read as `None`,
/// never a hang or a panic.
/// adapter there in the first place): absent, slower than [`NVIDIA_SMI_DEADLINE`] or odd output
/// all read as `None`, never a hang or a panic.
#[cfg(target_os = "linux")]
fn nvidia_video_memory(vendor: u32, device: u32) -> Option<u64> {
if vendor != NVIDIA_VENDOR {
return None;
}
let out = std::process::Command::new("nvidia-smi").args(["--query-gpu=pci.device_id,memory.total", "--format=csv,noheader,nounits"]).output().ok()?;
out.status.success().then(|| parse_nvidia_smi(&String::from_utf8_lossy(&out.stdout), vendor, device)).flatten()
let mut cmd = std::process::Command::new("nvidia-smi");
cmd.args(["--query-gpu=pci.device_id,memory.total", "--format=csv,noheader,nounits"]);
let out = bounded_output(cmd, NVIDIA_SMI_DEADLINE)?;
parse_nvidia_smi(&String::from_utf8_lossy(&out), vendor, device)
}
/// The first 64 KiB of `cmd`'s stdout when it exits successfully within `deadline`; `None` when
/// it can't start, fails or is still running by then. A late one is killed and reaped on a thread
/// of its own (a process stuck in the driver may not die at once), so the caller waits about
/// `deadline` at most.
#[cfg(any(target_os = "linux", test))]
fn bounded_output(mut cmd: std::process::Command, deadline: std::time::Duration) -> Option<Vec<u8>> {
use std::io::Read;
use std::process::{Child, Stdio};
use std::time::{Duration, Instant};
fn abandon(mut child: Child) {
let _ = child.kill();
let _ = std::thread::Builder::new().name("reap".into()).spawn(move || child.wait());
}
let end = Instant::now().checked_add(deadline)?;
let mut child = cmd.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().ok()?;
let Some(stdout) = child.stdout.take() else {
abandon(child);
return None;
};
let (tx, rx) = std::sync::mpsc::channel();
let reader = std::thread::Builder::new().name("child-stdout".into()).spawn(move || {
let mut buf = Vec::new();
let _ = stdout.take(64 << 10).read_to_end(&mut buf);
let _ = tx.send(buf);
});
if reader.is_err() {
abandon(child);
return None;
}
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) if Instant::now() < end => std::thread::sleep(Duration::from_millis(10)),
_ => {
abandon(child);
return None;
}
}
};
if !status.success() {
return None;
}
// The child has exited, so its stdout is at EOF unless it handed the pipe on to a process of
// its own: bounded too, rather than joining the reader.
rx.recv_timeout(end.saturating_duration_since(Instant::now()).max(Duration::from_millis(50))).ok()
}
/// `nvidia-smi --query-gpu=pci.device_id,memory.total --format=csv,noheader,nounits`'s own
/// format: one `0xDDDDVVVV, size-in-MiB` line per card (device id the high half, vendor id the
/// low, as `lspci -n` and wgpu's own ids read them). The matching line's size, in bytes; `None`
/// when no line names this `vendor`/`device`, or the output isn't shaped like that at all.
/// low, as `lspci -n` and wgpu's own ids read them). The matching line's size, in bytes, and the
/// smallest when several cards share those ids (the ids can't tell which one wgpu picked, so the
/// budget assumes the smaller); `None` when no line names this `vendor`/`device`, or the output
/// isn't shaped like that at all.
#[cfg(any(target_os = "linux", test))]
fn parse_nvidia_smi(output: &str, vendor: u32, device: u32) -> Option<u64> {
output.lines().find_map(|line| {
let (id, mb) = line.split_once(',')?;
let id = u32::from_str_radix(id.trim().trim_start_matches("0x").trim_start_matches("0X"), 16).ok()?;
if id & 0xffff != vendor || id >> 16 != device {
return None;
}
mb.trim().parse::<u64>().ok()?.checked_mul(1 << 20)
})
output
.lines()
.filter_map(|line| {
let (id, mb) = line.split_once(',')?;
let id = u32::from_str_radix(id.trim().trim_start_matches("0x").trim_start_matches("0X"), 16).ok()?;
if id & 0xffff != vendor || id >> 16 != device {
return None;
}
mb.trim().parse::<u64>().ok()?.checked_mul(1 << 20).filter(|b| *b > 0)
})
.min()
}
/// amdgpu's VRAM size (`<card>/device/mem_info_vram_total`, bytes) of the card under `drm` (the
@@ -2900,6 +2959,57 @@ mod tests {
assert_eq!(parse_nvidia_smi(&format!("0x2C0510DE, {}\n", u64::MAX), NVIDIA_VENDOR, 0x2c05), None, "would overflow");
}
/// Two cards of one model (same vendor and device id) with different memory: wgpu's ids can't
/// tell which one it picked, so the budget assumes the smaller rather than whichever came first.
#[test]
fn nvidia_smi_cards_sharing_an_id_report_the_smaller_memory() {
assert_eq!(parse_nvidia_smi("0x2C0510DE, 16303\n0x2C0510DE, 8192\n", NVIDIA_VENDOR, 0x2c05), Some(8192 << 20));
assert_eq!(parse_nvidia_smi("0x2C0510DE, 8192\n0x2C0510DE, 16303\n", NVIDIA_VENDOR, 0x2c05), Some(8192 << 20));
assert_eq!(parse_nvidia_smi("0x2C0510DE, lots\n0x2C0510DE, 16303\n", NVIDIA_VENDOR, 0x2c05), Some(16303 << 20), "a bad line is skipped");
assert_eq!(parse_nvidia_smi("0x2C0510DE, 0\n0x2C0510DE, 16303\n", NVIDIA_VENDOR, 0x2c05), Some(16303 << 20), "a zero size is no answer");
}
/// `nvidia-smi` runs on the UI thread at startup, so a query the driver never answers must
/// give up at the deadline instead of keeping the window from opening.
#[cfg(any(unix, windows))]
#[test]
fn a_child_that_never_answers_gives_up_at_the_deadline() {
let slow = if cfg!(windows) {
let mut c = std::process::Command::new("ping");
c.args(["-n", "11", "127.0.0.1"]);
c
} else {
let mut c = std::process::Command::new("sleep");
c.arg("10");
c
};
let start = std::time::Instant::now();
assert_eq!(bounded_output(slow, std::time::Duration::from_millis(200)), None);
assert!(start.elapsed() < std::time::Duration::from_secs(2), "took {:?}", start.elapsed());
let quick = if cfg!(windows) {
let mut c = std::process::Command::new("cmd");
c.args(["/C", "echo 0x2C0510DE, 16303"]);
c
} else {
let mut c = std::process::Command::new("printf");
c.arg("0x2C0510DE, 16303\n");
c
};
let out = bounded_output(quick, std::time::Duration::from_secs(10)).expect("a quick command answers");
assert_eq!(parse_nvidia_smi(&String::from_utf8_lossy(&out), NVIDIA_VENDOR, 0x2c05), Some(16303 << 20));
let failing = if cfg!(windows) {
let mut c = std::process::Command::new("cmd");
c.args(["/C", "exit 3"]);
c
} else {
std::process::Command::new("false")
};
assert_eq!(bounded_output(failing, std::time::Duration::from_secs(10)), None, "a failed run");
assert_eq!(bounded_output(std::process::Command::new("photocraft-no-such-tool"), std::time::Duration::from_secs(1)), None, "not installed");
}
/// Only a discrete NVIDIA adapter asks `nvidia-smi`: nothing else spawns a process whose
/// answer it would then ignore anyway (`nvidia_video_memory` gates on the vendor id first).
#[cfg(target_os = "linux")]