mirror of
https://github.com/storytold/photocraft.git
synced 2026-11-21 09:26:43 +00:00
Integration: bound the nvidia-smi query, smallest of same-id cards (#9042)
nvidia_video_memory ran `nvidia-smi` with Command::output(), which has no deadline. It runs on the UI thread at startup, under the video-memory cache lock, so a driver that never answers the query kept the window from opening. It now runs with a 2 s deadline (stdout read on a helper thread, try_wait polling), and a late process is killed and reaped on a detached thread. parse_nvidia_smi took the first line naming the vendor/device id. Two cards of one model with different memory (`0x2C0510DE, 16303` then `0x2C0510DE, 8192`) gave 16 GB though wgpu may be on the 8 GB one; it now takes the smallest. Co-authored-by: Vishnu Vardhan Varma Byrraju <112673081+VishnuByrraju@users.noreply.github.com> Co-authored-by: Khawar Zamman Wani <28915582+wanikhawar@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d0b08abc65
commit
ffd4d480bb
1 file changed
+129
-19
@@ -1505,11 +1505,11 @@ fn detect_physical_memory() -> Option<u64> {
|
||||
}
|
||||
|
||||
/// The memory of `adapter` itself, in bytes, when it is a discrete GPU and the platform reports
|
||||
/// it: DXGI's dedicated video memory on Windows, amdgpu's VRAM size on Linux. `None` for
|
||||
/// integrated and software adapters, whose textures live in system memory (the RAM-based
|
||||
/// budgets already cover them), and where nothing reports it. `PHOTOCRAFT_VRAM_MB` overrides it
|
||||
/// for any adapter, simulating a discrete card of that size: on an integrated GPU the budget then
|
||||
/// stops charging the document's pixels too, as for a real card (a development switch).
|
||||
/// it: DXGI's dedicated video memory on Windows, amdgpu's VRAM size or `nvidia-smi`'s report on
|
||||
/// Linux. `None` for integrated and software adapters, whose textures live in system memory (the
|
||||
/// RAM-based budgets already cover them), and where nothing reports it. `PHOTOCRAFT_VRAM_MB`
|
||||
/// overrides it for any adapter, simulating a discrete card of that size: on an integrated GPU the
|
||||
/// budget then stops charging the document's pixels too, as for a real card (a development switch).
|
||||
pub fn video_memory(adapter: &wgpu::AdapterInfo) -> Option<u64> {
|
||||
if let Some(mb) = std::env::var("PHOTOCRAFT_VRAM_MB").ok().and_then(|v| v.trim().parse::<u64>().ok()) {
|
||||
return mb.checked_mul(1 << 20).filter(|b| *b > 0);
|
||||
@@ -1558,32 +1558,91 @@ fn detect_video_memory(vendor: u32, device: u32) -> Option<u64> {
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
const NVIDIA_VENDOR: u32 = 0x10de;
|
||||
|
||||
/// How long `nvidia-smi` gets to answer: it runs on the UI thread while the window opens, and a
|
||||
/// driver that hangs the query must not keep PhotoCraft from starting.
|
||||
#[cfg(target_os = "linux")]
|
||||
const NVIDIA_SMI_DEADLINE: std::time::Duration = std::time::Duration::from_secs(2);
|
||||
|
||||
/// NVIDIA's own memory size on Linux, from `nvidia-smi` (installed with the driver that put this
|
||||
/// adapter there in the first place): absent, slow to answer or odd output all read as `None`,
|
||||
/// never a hang or a panic.
|
||||
/// adapter there in the first place): absent, slower than [`NVIDIA_SMI_DEADLINE`] or odd output
|
||||
/// all read as `None`, never a hang or a panic.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn nvidia_video_memory(vendor: u32, device: u32) -> Option<u64> {
|
||||
if vendor != NVIDIA_VENDOR {
|
||||
return None;
|
||||
}
|
||||
let out = std::process::Command::new("nvidia-smi").args(["--query-gpu=pci.device_id,memory.total", "--format=csv,noheader,nounits"]).output().ok()?;
|
||||
out.status.success().then(|| parse_nvidia_smi(&String::from_utf8_lossy(&out.stdout), vendor, device)).flatten()
|
||||
let mut cmd = std::process::Command::new("nvidia-smi");
|
||||
cmd.args(["--query-gpu=pci.device_id,memory.total", "--format=csv,noheader,nounits"]);
|
||||
let out = bounded_output(cmd, NVIDIA_SMI_DEADLINE)?;
|
||||
parse_nvidia_smi(&String::from_utf8_lossy(&out), vendor, device)
|
||||
}
|
||||
|
||||
/// The first 64 KiB of `cmd`'s stdout when it exits successfully within `deadline`; `None` when
|
||||
/// it can't start, fails or is still running by then. A late one is killed and reaped on a thread
|
||||
/// of its own (a process stuck in the driver may not die at once), so the caller waits about
|
||||
/// `deadline` at most.
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
fn bounded_output(mut cmd: std::process::Command, deadline: std::time::Duration) -> Option<Vec<u8>> {
|
||||
use std::io::Read;
|
||||
use std::process::{Child, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
fn abandon(mut child: Child) {
|
||||
let _ = child.kill();
|
||||
let _ = std::thread::Builder::new().name("reap".into()).spawn(move || child.wait());
|
||||
}
|
||||
let end = Instant::now().checked_add(deadline)?;
|
||||
let mut child = cmd.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().ok()?;
|
||||
let Some(stdout) = child.stdout.take() else {
|
||||
abandon(child);
|
||||
return None;
|
||||
};
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let reader = std::thread::Builder::new().name("child-stdout".into()).spawn(move || {
|
||||
let mut buf = Vec::new();
|
||||
let _ = stdout.take(64 << 10).read_to_end(&mut buf);
|
||||
let _ = tx.send(buf);
|
||||
});
|
||||
if reader.is_err() {
|
||||
abandon(child);
|
||||
return None;
|
||||
}
|
||||
let status = loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(status)) => break status,
|
||||
Ok(None) if Instant::now() < end => std::thread::sleep(Duration::from_millis(10)),
|
||||
_ => {
|
||||
abandon(child);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
};
|
||||
if !status.success() {
|
||||
return None;
|
||||
}
|
||||
// The child has exited, so its stdout is at EOF unless it handed the pipe on to a process of
|
||||
// its own: bounded too, rather than joining the reader.
|
||||
rx.recv_timeout(end.saturating_duration_since(Instant::now()).max(Duration::from_millis(50))).ok()
|
||||
}
|
||||
|
||||
/// `nvidia-smi --query-gpu=pci.device_id,memory.total --format=csv,noheader,nounits`'s own
|
||||
/// format: one `0xDDDDVVVV, size-in-MiB` line per card (device id the high half, vendor id the
|
||||
/// low, as `lspci -n` and wgpu's own ids read them). The matching line's size, in bytes; `None`
|
||||
/// when no line names this `vendor`/`device`, or the output isn't shaped like that at all.
|
||||
/// low, as `lspci -n` and wgpu's own ids read them). The matching line's size, in bytes, and the
|
||||
/// smallest when several cards share those ids (the ids can't tell which one wgpu picked, so the
|
||||
/// budget assumes the smaller); `None` when no line names this `vendor`/`device`, or the output
|
||||
/// isn't shaped like that at all.
|
||||
#[cfg(any(target_os = "linux", test))]
|
||||
fn parse_nvidia_smi(output: &str, vendor: u32, device: u32) -> Option<u64> {
|
||||
output.lines().find_map(|line| {
|
||||
let (id, mb) = line.split_once(',')?;
|
||||
let id = u32::from_str_radix(id.trim().trim_start_matches("0x").trim_start_matches("0X"), 16).ok()?;
|
||||
if id & 0xffff != vendor || id >> 16 != device {
|
||||
return None;
|
||||
}
|
||||
mb.trim().parse::<u64>().ok()?.checked_mul(1 << 20)
|
||||
})
|
||||
output
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let (id, mb) = line.split_once(',')?;
|
||||
let id = u32::from_str_radix(id.trim().trim_start_matches("0x").trim_start_matches("0X"), 16).ok()?;
|
||||
if id & 0xffff != vendor || id >> 16 != device {
|
||||
return None;
|
||||
}
|
||||
mb.trim().parse::<u64>().ok()?.checked_mul(1 << 20).filter(|b| *b > 0)
|
||||
})
|
||||
.min()
|
||||
}
|
||||
|
||||
/// amdgpu's VRAM size (`<card>/device/mem_info_vram_total`, bytes) of the card under `drm` (the
|
||||
@@ -2900,6 +2959,57 @@ mod tests {
|
||||
assert_eq!(parse_nvidia_smi(&format!("0x2C0510DE, {}\n", u64::MAX), NVIDIA_VENDOR, 0x2c05), None, "would overflow");
|
||||
}
|
||||
|
||||
/// Two cards of one model (same vendor and device id) with different memory: wgpu's ids can't
|
||||
/// tell which one it picked, so the budget assumes the smaller rather than whichever came first.
|
||||
#[test]
|
||||
fn nvidia_smi_cards_sharing_an_id_report_the_smaller_memory() {
|
||||
assert_eq!(parse_nvidia_smi("0x2C0510DE, 16303\n0x2C0510DE, 8192\n", NVIDIA_VENDOR, 0x2c05), Some(8192 << 20));
|
||||
assert_eq!(parse_nvidia_smi("0x2C0510DE, 8192\n0x2C0510DE, 16303\n", NVIDIA_VENDOR, 0x2c05), Some(8192 << 20));
|
||||
assert_eq!(parse_nvidia_smi("0x2C0510DE, lots\n0x2C0510DE, 16303\n", NVIDIA_VENDOR, 0x2c05), Some(16303 << 20), "a bad line is skipped");
|
||||
assert_eq!(parse_nvidia_smi("0x2C0510DE, 0\n0x2C0510DE, 16303\n", NVIDIA_VENDOR, 0x2c05), Some(16303 << 20), "a zero size is no answer");
|
||||
}
|
||||
|
||||
/// `nvidia-smi` runs on the UI thread at startup, so a query the driver never answers must
|
||||
/// give up at the deadline instead of keeping the window from opening.
|
||||
#[cfg(any(unix, windows))]
|
||||
#[test]
|
||||
fn a_child_that_never_answers_gives_up_at_the_deadline() {
|
||||
let slow = if cfg!(windows) {
|
||||
let mut c = std::process::Command::new("ping");
|
||||
c.args(["-n", "11", "127.0.0.1"]);
|
||||
c
|
||||
} else {
|
||||
let mut c = std::process::Command::new("sleep");
|
||||
c.arg("10");
|
||||
c
|
||||
};
|
||||
let start = std::time::Instant::now();
|
||||
assert_eq!(bounded_output(slow, std::time::Duration::from_millis(200)), None);
|
||||
assert!(start.elapsed() < std::time::Duration::from_secs(2), "took {:?}", start.elapsed());
|
||||
|
||||
let quick = if cfg!(windows) {
|
||||
let mut c = std::process::Command::new("cmd");
|
||||
c.args(["/C", "echo 0x2C0510DE, 16303"]);
|
||||
c
|
||||
} else {
|
||||
let mut c = std::process::Command::new("printf");
|
||||
c.arg("0x2C0510DE, 16303\n");
|
||||
c
|
||||
};
|
||||
let out = bounded_output(quick, std::time::Duration::from_secs(10)).expect("a quick command answers");
|
||||
assert_eq!(parse_nvidia_smi(&String::from_utf8_lossy(&out), NVIDIA_VENDOR, 0x2c05), Some(16303 << 20));
|
||||
|
||||
let failing = if cfg!(windows) {
|
||||
let mut c = std::process::Command::new("cmd");
|
||||
c.args(["/C", "exit 3"]);
|
||||
c
|
||||
} else {
|
||||
std::process::Command::new("false")
|
||||
};
|
||||
assert_eq!(bounded_output(failing, std::time::Duration::from_secs(10)), None, "a failed run");
|
||||
assert_eq!(bounded_output(std::process::Command::new("photocraft-no-such-tool"), std::time::Duration::from_secs(1)), None, "not installed");
|
||||
}
|
||||
|
||||
/// Only a discrete NVIDIA adapter asks `nvidia-smi`: nothing else spawns a process whose
|
||||
/// answer it would then ignore anyway (`nvidia_video_memory` gates on the vendor id first).
|
||||
#[cfg(target_os = "linux")]
|
||||
|
||||
Reference in new issue
Block a user