diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20a0ccae..f48a21f4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -133,6 +133,8 @@ jobs: mkdir -p dist if [ "${{ matrix.os }}" = windows-latest ]; then 7z a -tzip "dist/${{ matrix.archive }}" "./$dir/devstack.exe" README.md + # The build's symbols, apart: they read a dump of a hung hub (#398). + 7z a -tzip "dist/devstack-${{ matrix.target }}-pdb.zip" "./$dir/devstack.pdb" else tar -czf "dist/${{ matrix.archive }}" -C "$dir" devstack -C "$GITHUB_WORKSPACE" README.md fi @@ -144,7 +146,7 @@ jobs: - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: bin-${{ matrix.target }} - path: dist/${{ matrix.archive }} + path: dist/devstack-${{ matrix.target }}* publish: needs: [build, advisories] diff --git a/CHANGELOG.md b/CHANGELOG.md index c58a31f9..de3bfdec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,13 @@ and listed with their assets on the `up` tool takes `shift`; `POST /hub/stacks/{id}/up` takes `{shift}`; the stack page shows "shifted +1000" and, for a stack that collides, a *Start with shift* button. +- **The hub watches itself** (#398): a thread outside its async runtime + asks the hub's own port every 30 s. After two minutes without an answer + the hub writes a dump of itself next to its log (`hub-stuck-.dmp`, + Windows; the newest three are kept), logs `hub_stuck`, starts the next + hub and exits, so the dashboard comes back by itself. The release has + the Windows build's symbols as an asset + (`devstack-x86_64-pc-windows-msvc-pdb.zip`) to read such a dump. ## [0.33.0] - 2026-10-31 diff --git a/Cargo.toml b/Cargo.toml index 06d826a7..51d37ba5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,9 +111,12 @@ windows-sys = { version = "0.61", features = [ "Win32_System_SystemInformation", "Win32_System_Com", "Win32_System_Console", + "Win32_System_Diagnostics_Debug", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", + "Win32_System_Kernel", "Win32_System_LibraryLoader", + "Win32_System_Memory", "Win32_System_Threading", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_Shell", diff --git a/crates/devstack-hub/src/lib.rs b/crates/devstack-hub/src/lib.rs index 717c955a..adf27820 100644 --- a/crates/devstack-hub/src/lib.rs +++ b/crates/devstack-hub/src/lib.rs @@ -54,6 +54,7 @@ mod stdio; mod summary; pub mod tray; pub mod upstreams; +mod watchdog; mod worktrees; mod ws; @@ -472,6 +473,7 @@ pub fn serve() -> Result<(), String> { .await; }); } + watchdog::start(port, dir.clone()); let hub_json = dir.join("hub.json"); diag::info( "hub_start", diff --git a/crates/devstack-hub/src/watchdog.rs b/crates/devstack-hub/src/watchdog.rs new file mode 100644 index 00000000..e704d811 --- /dev/null +++ b/crates/devstack-hub/src/watchdog.rs @@ -0,0 +1,197 @@ +//! The hub watches itself (#398): a thread outside the async runtime asks +//! the hub's own port for `/hub/ping` every [`EVERY`]. A hub whose runtime +//! is stuck still has the port, and the operating system takes connections +//! for it until its queue is full, so a refused or unanswered request is +//! the only sign. After [`STUCK_AFTER`] without an answer it writes a dump +//! of itself (Windows), says so in its log, starts the next hub and exits: +//! the dashboard comes back by itself, and the dump says where it hung. + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use devstack_daemon::diag; + +/// How often the hub asks itself. +const EVERY: Duration = Duration::from_secs(30); +/// How long it waits for its own answer. +const WITHIN: Duration = Duration::from_secs(10); +/// How long without an answer before it counts as stuck. +const STUCK_AFTER: Duration = Duration::from_secs(120); +/// Dumps kept in the hub's directory, the newest. +const KEEP: usize = 3; + +pub fn start(port: u16, dir: PathBuf) { + let _ = std::thread::Builder::new() + .name("hub-watchdog".into()) + .spawn(move || watch(port, &dir)); +} + +fn watch(port: u16, dir: &Path) { + let mut failed = 0u32; + loop { + std::thread::sleep(EVERY); + if answers(port) { + failed = 0; + continue; + } + failed += 1; + if !stuck(failed) { + continue; + } + let dump = dir.join(format!("hub-stuck-{}.dmp", crate::now_ms())); + let written = devstack_proc::write_dump(&dump); + diag::warn( + "hub_stuck", + serde_json::json!({ + "port": port, + "silent_s": u64::from(failed) * EVERY.as_secs(), + "dump": written.as_ref().ok().map(|()| dump.display().to_string()), + "dump_error": written.as_ref().err().map(ToString::to_string), + }), + ); + prune(dir); + successor(); + let _ = std::fs::remove_file(dir.join("hub.json")); + std::process::exit(1); + } +} + +/// Whether `failed` checks in a row without an answer make the hub stuck. +fn stuck(failed: u32) -> bool { + u64::from(failed) * EVERY.as_secs() >= STUCK_AFTER.as_secs() +} + +/// Whether the hub answers an HTTP request on `port` at all (any status): +/// a connection the operating system took for it is not enough. +fn answers(port: u16) -> bool { + let Ok(mut s) = TcpStream::connect_timeout(&SocketAddr::from(([127, 0, 0, 1], port)), WITHIN) + else { + return false; + }; + let _ = s.set_read_timeout(Some(WITHIN)); + let _ = s.set_write_timeout(Some(WITHIN)); + if s.write_all(b"GET /hub/ping HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n") + .is_err() + { + return false; + } + let mut head = [0u8; 12]; + matches!(s.read(&mut head), Ok(n) if is_http_answer(&head[..n])) +} + +fn is_http_answer(head: &[u8]) -> bool { + head.starts_with(b"HTTP/1.") +} + +/// Start the next hub before this one exits: it waits for the lock (up to +/// 15 s), which is free once this process is gone. +fn successor() { + let Ok(exe) = crate::devstack_exe() else { + return; + }; + let mut cmd = std::process::Command::new(exe); + cmd.arg("hub") + .current_dir(crate::state_dir()) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + let _ = devstack_proc::spawn_detached(&mut cmd); +} + +/// Keep the newest [`KEEP`] dumps. +fn prune(dir: &Path) { + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + let mut dumps: Vec = rd + .flatten() + .map(|e| e.path()) + .filter(|p| is_dump(p)) + .collect(); + // Named by the time they were written: the oldest sort first. + dumps.sort(); + let extra = dumps.len().saturating_sub(KEEP); + for p in dumps.into_iter().take(extra) { + let _ = std::fs::remove_file(p); + } +} + +fn is_dump(p: &Path) -> bool { + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("hub-stuck-") && n.ends_with(".dmp")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stuck_after_two_minutes_of_silence() { + assert!(!stuck(1)); + assert!(!stuck(3)); + assert!(stuck(4)); + } + + #[test] + fn only_an_http_status_line_is_an_answer() { + assert!(is_http_answer(b"HTTP/1.1 200")); + assert!(!is_http_answer(b"")); + assert!(!is_http_answer(b"SSH-2.0-Op")); + } + + #[test] + fn a_silent_port_does_not_answer() { + // Takes the connection (the operating system's queue) and says + // nothing: what a hub with a stuck runtime looks like from outside. + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = l.local_addr().unwrap().port(); + let t = std::thread::spawn(move || { + let (_s, _) = l.accept().unwrap(); + std::thread::sleep(WITHIN + Duration::from_secs(1)); + }); + assert!(!answers(port)); + t.join().unwrap(); + } + + #[test] + fn the_newest_dumps_are_kept() { + let d = tempfile::tempdir().unwrap(); + for t in [1, 2, 3, 4, 5] { + std::fs::write(d.path().join(format!("hub-stuck-{t}.dmp")), b"").unwrap(); + } + std::fs::write(d.path().join("hub.log"), b"").unwrap(); + prune(d.path()); + let mut left: Vec = std::fs::read_dir(d.path()) + .unwrap() + .flatten() + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect(); + left.sort(); + assert_eq!( + left, + [ + "hub-stuck-3.dmp", + "hub-stuck-4.dmp", + "hub-stuck-5.dmp", + "hub.log" + ] + ); + } + + #[cfg(windows)] + #[test] + fn a_dump_of_this_process_is_written() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("hub-stuck-1.dmp"); + devstack_proc::write_dump(&p).unwrap(); + let mut sig = [0u8; 4]; + std::fs::File::open(&p) + .unwrap() + .read_exact(&mut sig) + .unwrap(); + assert_eq!(&sig, b"MDMP"); + } +} diff --git a/crates/devstack-proc/src/lib.rs b/crates/devstack-proc/src/lib.rs index 6bc957ad..7eec7c4d 100644 --- a/crates/devstack-proc/src/lib.rs +++ b/crates/devstack-proc/src/lib.rs @@ -92,6 +92,46 @@ pub fn pid_alive(pid: u32) -> bool { backend::pid_alive(pid) } +/// Write a minidump of this process to `path`: its threads' stacks and +/// what they point at, for reading a hang afterwards with the build's PDB +/// (#398). Windows only; elsewhere `Unsupported`. +pub fn write_dump(path: &std::path::Path) -> std::io::Result<()> { + #[cfg(windows)] + { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::Diagnostics::Debug::{ + MiniDumpWithIndirectlyReferencedMemory, MiniDumpWithThreadInfo, MiniDumpWriteDump, + }; + use windows_sys::Win32::System::Threading::{GetCurrentProcess, GetCurrentProcessId}; + let file = std::fs::File::create(path)?; + // SAFETY: a pseudo handle to this process and a file handle we own + // for the duration of the call; no optional parameters. + let ok = unsafe { + MiniDumpWriteDump( + GetCurrentProcess(), + GetCurrentProcessId(), + file.as_raw_handle(), + MiniDumpWithThreadInfo | MiniDumpWithIndirectlyReferencedMemory, + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + ) + }; + if ok == 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + } + #[cfg(not(windows))] + { + let _ = path; + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "a minidump is written on Windows only", + )) + } +} + /// End the process `pid`, one devstack did not start (a daemon or a hub /// that does not answer): Windows terminates it, which closes its job /// objects and with them every process tree it owns (ADR-0009); elsewhere diff --git a/docs/releasing.md b/docs/releasing.md index 510c3156..8bce17a0 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -84,7 +84,9 @@ The release workflow checks that the tag matches `Cargo.toml` and - builds Windows (x86_64), Linux (x86_64, arm64; static) and macOS (Apple Silicon, Intel); -- publishes the GitHub release: the five archives, `install.sh`, +- publishes the GitHub release: the five archives, the Windows build's + symbols (`devstack-x86_64-pc-windows-msvc-pdb.zip`, for reading a hung + hub's dump, #398), `install.sh`, `install.ps1` and `SHA256SUMS`, each with a build provenance attestation, and notes generated from the merged pull requests; - submits the Windows build to winget (no second pull request while the