diff --git a/apps/solidcraft/src/device_lost.rs b/apps/solidcraft/src/device_lost.rs index b892723c3..2f598c69c 100644 --- a/apps/solidcraft/src/device_lost.rs +++ b/apps/solidcraft/src/device_lost.rs @@ -5,12 +5,25 @@ //! for recovery as a forced quit does, and starts again ([`restart`]), and the new start says what //! happened ([`note`]). A device lost again soon after such a restart starts the next one on the //! software renderer, which doesn't depend on the graphics driver. +//! +//! The old process ends without tearing down its GPU objects ([`end`]): on a lost NVIDIA device +//! destroying the window's swap chain busy-waits in the driver and never returns (#9044). use crate::startup; /// Set on the start after a lost device: when the device was lost (seconds since 1970). pub const LOST_ENV: &str = "SOLIDCRAFT_DEVICE_LOST"; +/// Set on the start after a lost device: why it was lost, as wgpu said it. The old process's +/// session journal, which recorded it, is gone by then. +pub const REASON_ENV: &str = "SOLIDCRAFT_DEVICE_LOST_REASON"; + +/// The longest reason passed on ([`REASON_ENV`]) and shown, in characters. +const REASON_CHARS: usize = 300; + +/// How long the process may take to end ([`end`]) before it is aborted. +const END_GRACE: std::time::Duration = std::time::Duration::from_secs(3); + /// A loss this recent (seconds) counts for the start after it: its note, and a second loss /// within it starts on the software renderer. Inherited by later starts, an older stamp is /// ignored. @@ -32,27 +45,54 @@ pub fn restart_on_software(after_recent_loss: bool, cpu_renderer: bool) -> bool after_recent_loss || cpu_renderer } +/// `reason` on one line, at most [`REASON_CHARS`] long; None when there is nothing to say. +pub fn reason(reason: Option<&str>) -> Option { + let line = reason?.split_whitespace().collect::>().join(" "); + (!line.is_empty()).then(|| line.chars().take(REASON_CHARS).collect()) +} + /// What the start after a lost device tells the user, `software` when it runs on the software -/// renderer. -pub fn note(software: bool) -> String { +/// renderer, with what the driver reported ([`REASON_ENV`]). +pub fn note(software: bool, reason: Option<&str>) -> String { let mut note = String::from( "The graphics device was reset (the graphics driver stopped responding), so SolidCraft closed and started again. Unsaved changes were kept for recovery.", ); if software { note.push_str(" This start uses software rendering, which doesn't depend on the graphics driver."); } + if let Some(reason) = self::reason(reason) { + note.push_str(&format!(" The graphics driver reported: {reason}")); + } note } -/// Start SolidCraft again with the same arguments, the loss stamped at `lost_at`, on the software -/// renderer when `software`. -pub fn restart(software: bool, lost_at: u64) { - let Ok(exe) = std::env::current_exe() else { return }; +/// The command that starts `exe` again with `args`, the loss stamped at `lost_at` and its +/// `reason`, on the software renderer when `software`. +fn command( + exe: std::path::PathBuf, + args: impl IntoIterator, + software: bool, + lost_at: u64, + reason: &str, +) -> std::process::Command { let mut command = std::process::Command::new(exe); - command.args(std::env::args_os().skip(1)).env(LOST_ENV, lost_at.to_string()); + command.args(args).env(LOST_ENV, lost_at.to_string()); + match self::reason(Some(reason)) { + Some(reason) => command.env(REASON_ENV, reason), + // Not one inherited from an earlier loss. + None => command.env_remove(REASON_ENV), + }; if software { command.env(startup::RENDERER_ENV, "cpu"); } + command +} + +/// Start SolidCraft again with the same arguments, the loss stamped at `lost_at` and its `reason`, +/// on the software renderer when `software`. +pub fn restart(software: bool, lost_at: u64, reason: &str) { + let Ok(exe) = std::env::current_exe() else { return }; + let mut command = command(exe, std::env::args_os().skip(1), software, lost_at, reason); #[cfg(windows)] { use std::os::windows::process::CommandExt; @@ -64,6 +104,23 @@ pub fn restart(software: bool, lost_at: u64) { } } +/// End the process now, without tearing down what is left on the lost device: the window's swap +/// chain, the viewport's buffers and pipelines. On NVIDIA's Vulkan driver destroying the swap +/// chain of a lost device busy-waits forever, which left the old window frozen at 100% of a core +/// beside the new one (#9044). Should the exit itself get stuck in a library's exit handler, the +/// process is aborted after [`END_GRACE`]. +pub fn end() -> ! { + let watchdog = std::thread::Builder::new().name("solidcraft-exit".into()).spawn(|| { + std::thread::sleep(END_GRACE); + eprintln!("SolidCraft graphics: still closing after the device was lost; aborting"); + std::process::abort(); + }); + if let Err(e) = watchdog { + log::warn!("graphics: no exit watchdog: {e}"); + } + std::process::exit(0) +} + #[cfg(test)] mod tests { /// Only a recent, well-formed stamp counts: a later start that inherited it says nothing. @@ -84,8 +141,34 @@ mod tests { assert!(!super::restart_on_software(false, false)); assert!(super::restart_on_software(true, false)); assert!(super::restart_on_software(false, true)); - assert!(super::note(true).contains("software rendering")); - assert!(!super::note(false).contains("software rendering")); - assert!(super::note(false).contains("kept for recovery")); + assert!(super::note(true, None).contains("software rendering")); + assert!(!super::note(false, None).contains("software rendering")); + assert!(super::note(false, None).contains("kept for recovery")); + } + + /// The start after a loss is told why (#9044): the old process's journal held the reason, and + /// it is deleted as that process closes. + #[test] + fn the_next_start_is_told_why_the_device_was_lost() { + let exe = std::path::PathBuf::from("solidcraft"); + let args = ["--control".into(), "7979".into()]; + let command = super::command(exe.clone(), args, true, 1_000, "Lost: Device lost\n (vkQueueSubmit)"); + let env = |command: &std::process::Command, name: &str| { + command.get_envs().find(|(k, _)| *k == name).map(|(_, v)| v.map(|v| v.to_string_lossy().into_owned())) + }; + assert_eq!(env(&command, super::LOST_ENV), Some(Some("1000".into()))); + assert_eq!(env(&command, super::REASON_ENV), Some(Some("Lost: Device lost (vkQueueSubmit)".into())), "on one line"); + assert_eq!(env(&command, crate::startup::RENDERER_ENV), Some(Some("cpu".into()))); + assert_eq!(command.get_args().collect::>(), ["--control", "7979"]); + // No reason: none inherited from an earlier loss either. + let command = super::command(exe, Vec::new(), false, 1_000, " "); + assert_eq!(env(&command, super::REASON_ENV), Some(None)); + assert_eq!(env(&command, crate::startup::RENDERER_ENV), None); + + let note = super::note(false, Some("Lost: Device lost")); + assert!(note.ends_with("The graphics driver reported: Lost: Device lost"), "{note}"); + assert!(!super::note(false, Some(" \n ")).contains("reported")); + let long = "x".repeat(1_000); + assert_eq!(super::reason(Some(&long)).map(|r| r.chars().count()), Some(300)); } } diff --git a/apps/solidcraft/src/main.rs b/apps/solidcraft/src/main.rs index d9c6fbc9f..c4cfcf40f 100644 --- a/apps/solidcraft/src/main.rs +++ b/apps/solidcraft/src/main.rs @@ -204,13 +204,29 @@ impl eframe::App for App { } solidcraft_ui_egui::recovery_ui::close(&mut self.solve); solidcraft_ui_egui::open_elsewhere::close(&mut self.solve); - // Before a restart below starts the next instance, which would otherwise get nothing from - // the 3D mouse driver (`solidcraft_3dx_client::stop`). + // Before a restart (`Drop for App`) starts the next instance, which would otherwise get + // nothing from the 3D mouse driver (`solidcraft_3dx_client::stop`). #[cfg(target_os = "macos")] solidcraft_3dx_client::stop(); solidcraft_engine::crash::end_session(); - if let Some(lost_at) = self.gpu.as_ref().and_then(|g| g.lost_at) { - device_lost::restart(self.gpu.as_ref().is_some_and(|g| g.restart_on_software), lost_at); + // The next start (after a lost device, or into an update) begins once the app is dropped + // (`Drop for App`), when the settings are on disk. + } +} + +impl Drop for App { + /// eframe drops its settings storage first, which waits for `app.ron` to be written, then the + /// app, then the window and its GPU surface. So the next start begins here: started from + /// `on_exit`, it read `app.ron` while the old process was still writing it, found it + /// unreadable and moved it aside, resetting every setting (#9044). + fn drop(&mut self) { + if let Some(gpu) = &self.gpu + && let Some(lost_at) = gpu.lost_at + { + device_lost::restart(gpu.restart_on_software, lost_at, &gpu.reason); + // The window's GPU surface is still to be dropped, and on a lost device that may + // never return (#9044). + device_lost::end(); } // An update asked to restart into the new version or to run its installer. if let Some((program, args)) = solidcraft_ui_egui::update::take_launch(&mut self.solve) { @@ -238,6 +254,8 @@ struct LostWatch { restart_on_software: bool, /// When the loss was seen: the app is closing to start again. lost_at: Option, + /// Why, as wgpu said it: passed to the next start ([`device_lost::REASON_ENV`]). + reason: String, } impl App { @@ -251,6 +269,7 @@ impl App { eprintln!("SolidCraft graphics: the device was lost ({reason}); starting again"); solidcraft_engine::crash::phase(&format!("graphics device lost: {reason}")); gpu.lost_at = Some(device_lost::now_secs()); + gpu.reason = reason; self.solve.quit_requested = true; } } @@ -839,7 +858,11 @@ fn main() -> eframe::Result { // The launch before this one closed because its graphics device was lost (`App::watch_device`). let after_loss = device_lost::recent(std::env::var(device_lost::LOST_ENV).ok().as_deref(), device_lost::now_secs()); if after_loss { - notes.push(device_lost::note(cpu_renderer)); + let reason = std::env::var(device_lost::REASON_ENV).ok(); + if let Some(reason) = device_lost::reason(reason.as_deref()) { + log::warn!("graphics: the last launch lost its device ({reason})"); + } + notes.push(device_lost::note(cpu_renderer, reason.as_deref())); } if safe_mode { notes.push("Safe mode: default settings (yours are kept for the next normal start), no recovery offer, software rendering.".into()); @@ -979,6 +1002,7 @@ fn main() -> eframe::Result { lost: rs.device_lost.clone(), restart_on_software: device_lost::restart_on_software(after_loss, cpu_renderer), lost_at: None, + reason: String::new(), }); if !safe_mode { app.preferences.graphics_adapter = diff --git a/docs/book/src/guide/troubleshooting.md b/docs/book/src/guide/troubleshooting.md index edfe0c9b4..685ee7ca2 100644 --- a/docs/book/src/guide/troubleshooting.md +++ b/docs/book/src/guide/troubleshooting.md @@ -158,9 +158,12 @@ When the graphics driver stops responding, the system resets it (on Windows, "Di stopped responding and has recovered"), and the graphics device SolidCraft draws with is gone for good: the window can't draw anything more. SolidCraft then closes, keeps your unsaved changes for recovery as it does after a crash, and starts again; the new window says what -happened and offers the changes back. If the device is reset again within ten minutes of that -restart, the next start uses software rendering, which doesn't depend on the graphics driver. -Updating the graphics driver from the GPU maker usually stops the resets. +happened, with what the graphics driver reported, and offers the changes back. The old window +closes on its own; SolidCraft doesn't wait for the driver to release it. If the device is reset +again within ten minutes of that restart, the next start uses software rendering, which doesn't +depend on the graphics driver. Updating the graphics driver from the GPU maker usually stops the +resets. Laptops that lose the device on waking from sleep (often with an NVIDIA GPU that doesn't +drive the screen) can keep SolidCraft on the integrated GPU instead: Preferences › Graphics. ## A design saved by 0.10 won't open in an older version