feat(hub): the hub watches itself and leaves a dump when stuck (#398)

.github/workflows/release.yml

@@ -133,6 +133,8 @@ jobs:
           mkdir -p dist
           if [ "${{ matrix.os }}" = windows-latest ]; then
             7z a -tzip "dist/${{ matrix.archive }}" "./$dir/devstack.exe" README.md
+            # The build's symbols, apart: they read a dump of a hung hub (#398).
+            7z a -tzip "dist/devstack-${{ matrix.target }}-pdb.zip" "./$dir/devstack.pdb"
           else
             tar -czf "dist/${{ matrix.archive }}" -C "$dir" devstack -C "$GITHUB_WORKSPACE" README.md
           fi
@@ -144,7 +146,7 @@ jobs:
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: bin-${{ matrix.target }}
-          path: dist/${{ matrix.archive }}
+          path: dist/devstack-${{ matrix.target }}*
 
   publish:
     needs: [build, advisories]

CHANGELOG.md

@@ -25,6 +25,13 @@ and listed with their assets on the
   `up` tool takes `shift`; `POST /hub/stacks/{id}/up` takes `{shift}`; the
   stack page shows "shifted +1000" and, for a stack that collides, a
   *Start with shift* button.
+- **The hub watches itself** (#398): a thread outside its async runtime
+  asks the hub's own port every 30 s. After two minutes without an answer
+  the hub writes a dump of itself next to its log (`hub-stuck-<ms>.dmp`,
+  Windows; the newest three are kept), logs `hub_stuck`, starts the next
+  hub and exits, so the dashboard comes back by itself. The release has
+  the Windows build's symbols as an asset
+  (`devstack-x86_64-pc-windows-msvc-pdb.zip`) to read such a dump.
 
 ## [0.33.0] - 2026-10-31
 

Cargo.toml

@@ -111,9 +111,12 @@ windows-sys = { version = "0.61", features = [
     "Win32_System_SystemInformation",
     "Win32_System_Com",
     "Win32_System_Console",
+    "Win32_System_Diagnostics_Debug",
     "Win32_System_Diagnostics_ToolHelp",
     "Win32_System_JobObjects",
+    "Win32_System_Kernel",
     "Win32_System_LibraryLoader",
+    "Win32_System_Memory",
     "Win32_System_Threading",
     "Win32_UI_Input_KeyboardAndMouse",
     "Win32_UI_Shell",

crates/devstack-hub/src/lib.rs

@@ -54,6 +54,7 @@ mod stdio;
 mod summary;
 pub mod tray;
 pub mod upstreams;
+mod watchdog;
 mod worktrees;
 mod ws;
 
@@ -472,6 +473,7 @@ pub fn serve() -> Result<(), String> {
                     .await;
             });
         }
+        watchdog::start(port, dir.clone());
         let hub_json = dir.join("hub.json");
         diag::info(
             "hub_start",

crates/devstack-hub/src/watchdog.rs

@@ -0,0 +1,197 @@
+//! The hub watches itself (#398): a thread outside the async runtime asks
+//! the hub's own port for `/hub/ping` every [`EVERY`]. A hub whose runtime
+//! is stuck still has the port, and the operating system takes connections
+//! for it until its queue is full, so a refused or unanswered request is
+//! the only sign. After [`STUCK_AFTER`] without an answer it writes a dump
+//! of itself (Windows), says so in its log, starts the next hub and exits:
+//! the dashboard comes back by itself, and the dump says where it hung.
+
+use std::io::{Read, Write};
+use std::net::{SocketAddr, TcpStream};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use devstack_daemon::diag;
+
+/// How often the hub asks itself.
+const EVERY: Duration = Duration::from_secs(30);
+/// How long it waits for its own answer.
+const WITHIN: Duration = Duration::from_secs(10);
+/// How long without an answer before it counts as stuck.
+const STUCK_AFTER: Duration = Duration::from_secs(120);
+/// Dumps kept in the hub's directory, the newest.
+const KEEP: usize = 3;
+
+pub fn start(port: u16, dir: PathBuf) {
+    let _ = std::thread::Builder::new()
+        .name("hub-watchdog".into())
+        .spawn(move || watch(port, &dir));
+}
+
+fn watch(port: u16, dir: &Path) {
+    let mut failed = 0u32;
+    loop {
+        std::thread::sleep(EVERY);
+        if answers(port) {
+            failed = 0;
+            continue;
+        }
+        failed += 1;
+        if !stuck(failed) {
+            continue;
+        }
+        let dump = dir.join(format!("hub-stuck-{}.dmp", crate::now_ms()));
+        let written = devstack_proc::write_dump(&dump);
+        diag::warn(
+            "hub_stuck",
+            serde_json::json!({
+                "port": port,
+                "silent_s": u64::from(failed) * EVERY.as_secs(),
+                "dump": written.as_ref().ok().map(|()| dump.display().to_string()),
+                "dump_error": written.as_ref().err().map(ToString::to_string),
+            }),
+        );
+        prune(dir);
+        successor();
+        let _ = std::fs::remove_file(dir.join("hub.json"));
+        std::process::exit(1);
+    }
+}
+
+/// Whether `failed` checks in a row without an answer make the hub stuck.
+fn stuck(failed: u32) -> bool {
+    u64::from(failed) * EVERY.as_secs() >= STUCK_AFTER.as_secs()
+}
+
+/// Whether the hub answers an HTTP request on `port` at all (any status):
+/// a connection the operating system took for it is not enough.
+fn answers(port: u16) -> bool {
+    let Ok(mut s) = TcpStream::connect_timeout(&SocketAddr::from(([127, 0, 0, 1], port)), WITHIN)
+    else {
+        return false;
+    };
+    let _ = s.set_read_timeout(Some(WITHIN));
+    let _ = s.set_write_timeout(Some(WITHIN));
+    if s.write_all(b"GET /hub/ping HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n")
+        .is_err()
+    {
+        return false;
+    }
+    let mut head = [0u8; 12];
+    matches!(s.read(&mut head), Ok(n) if is_http_answer(&head[..n]))
+}
+
+fn is_http_answer(head: &[u8]) -> bool {
+    head.starts_with(b"HTTP/1.")
+}
+
+/// Start the next hub before this one exits: it waits for the lock (up to
+/// 15 s), which is free once this process is gone.
+fn successor() {
+    let Ok(exe) = crate::devstack_exe() else {
+        return;
+    };
+    let mut cmd = std::process::Command::new(exe);
+    cmd.arg("hub")
+        .current_dir(crate::state_dir())
+        .stdin(std::process::Stdio::null())
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null());
+    let _ = devstack_proc::spawn_detached(&mut cmd);
+}
+
+/// Keep the newest [`KEEP`] dumps.
+fn prune(dir: &Path) {
+    let Ok(rd) = std::fs::read_dir(dir) else {
+        return;
+    };
+    let mut dumps: Vec<PathBuf> = rd
+        .flatten()
+        .map(|e| e.path())
+        .filter(|p| is_dump(p))
+        .collect();
+    // Named by the time they were written: the oldest sort first.
+    dumps.sort();
+    let extra = dumps.len().saturating_sub(KEEP);
+    for p in dumps.into_iter().take(extra) {
+        let _ = std::fs::remove_file(p);
+    }
+}
+
+fn is_dump(p: &Path) -> bool {
+    p.file_name()
+        .and_then(|n| n.to_str())
+        .is_some_and(|n| n.starts_with("hub-stuck-") && n.ends_with(".dmp"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn stuck_after_two_minutes_of_silence() {
+        assert!(!stuck(1));
+        assert!(!stuck(3));
+        assert!(stuck(4));
+    }
+
+    #[test]
+    fn only_an_http_status_line_is_an_answer() {
+        assert!(is_http_answer(b"HTTP/1.1 200"));
+        assert!(!is_http_answer(b""));
+        assert!(!is_http_answer(b"SSH-2.0-Op"));
+    }
+
+    #[test]
+    fn a_silent_port_does_not_answer() {
+        // Takes the connection (the operating system's queue) and says
+        // nothing: what a hub with a stuck runtime looks like from outside.
+        let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let port = l.local_addr().unwrap().port();
+        let t = std::thread::spawn(move || {
+            let (_s, _) = l.accept().unwrap();
+            std::thread::sleep(WITHIN + Duration::from_secs(1));
+        });
+        assert!(!answers(port));
+        t.join().unwrap();
+    }
+
+    #[test]
+    fn the_newest_dumps_are_kept() {
+        let d = tempfile::tempdir().unwrap();
+        for t in [1, 2, 3, 4, 5] {
+            std::fs::write(d.path().join(format!("hub-stuck-{t}.dmp")), b"").unwrap();
+        }
+        std::fs::write(d.path().join("hub.log"), b"").unwrap();
+        prune(d.path());
+        let mut left: Vec<String> = std::fs::read_dir(d.path())
+            .unwrap()
+            .flatten()
+            .map(|e| e.file_name().to_string_lossy().into_owned())
+            .collect();
+        left.sort();
+        assert_eq!(
+            left,
+            [
+                "hub-stuck-3.dmp",
+                "hub-stuck-4.dmp",
+                "hub-stuck-5.dmp",
+                "hub.log"
+            ]
+        );
+    }
+
+    #[cfg(windows)]
+    #[test]
+    fn a_dump_of_this_process_is_written() {
+        let d = tempfile::tempdir().unwrap();
+        let p = d.path().join("hub-stuck-1.dmp");
+        devstack_proc::write_dump(&p).unwrap();
+        let mut sig = [0u8; 4];
+        std::fs::File::open(&p)
+            .unwrap()
+            .read_exact(&mut sig)
+            .unwrap();
+        assert_eq!(&sig, b"MDMP");
+    }
+}

crates/devstack-proc/src/lib.rs

@@ -92,6 +92,46 @@ pub fn pid_alive(pid: u32) -> bool {
     backend::pid_alive(pid)
 }
 
+/// Write a minidump of this process to `path`: its threads' stacks and
+/// what they point at, for reading a hang afterwards with the build's PDB
+/// (#398). Windows only; elsewhere `Unsupported`.
+pub fn write_dump(path: &std::path::Path) -> std::io::Result<()> {
+    #[cfg(windows)]
+    {
+        use std::os::windows::io::AsRawHandle;
+        use windows_sys::Win32::System::Diagnostics::Debug::{
+            MiniDumpWithIndirectlyReferencedMemory, MiniDumpWithThreadInfo, MiniDumpWriteDump,
+        };
+        use windows_sys::Win32::System::Threading::{GetCurrentProcess, GetCurrentProcessId};
+        let file = std::fs::File::create(path)?;
+        // SAFETY: a pseudo handle to this process and a file handle we own
+        // for the duration of the call; no optional parameters.
+        let ok = unsafe {
+            MiniDumpWriteDump(
+                GetCurrentProcess(),
+                GetCurrentProcessId(),
+                file.as_raw_handle(),
+                MiniDumpWithThreadInfo | MiniDumpWithIndirectlyReferencedMemory,
+                std::ptr::null(),
+                std::ptr::null(),
+                std::ptr::null(),
+            )
+        };
+        if ok == 0 {
+            return Err(std::io::Error::last_os_error());
+        }
+        Ok(())
+    }
+    #[cfg(not(windows))]
+    {
+        let _ = path;
+        Err(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            "a minidump is written on Windows only",
+        ))
+    }
+}
+
 /// End the process `pid`, one devstack did not start (a daemon or a hub
 /// that does not answer): Windows terminates it, which closes its job
 /// objects and with them every process tree it owns (ADR-0009); elsewhere

docs/releasing.md

@@ -84,7 +84,9 @@ The release workflow checks that the tag matches `Cargo.toml` and
 
 - builds Windows (x86_64), Linux (x86_64, arm64; static) and macOS (Apple
   Silicon, Intel);
-- publishes the GitHub release: the five archives, `install.sh`,
+- publishes the GitHub release: the five archives, the Windows build's
+  symbols (`devstack-x86_64-pc-windows-msvc-pdb.zip`, for reading a hung
+  hub's dump, #398), `install.sh`,
   `install.ps1` and `SHA256SUMS`, each with a build provenance
   attestation, and notes generated from the merged pull requests;
 - submits the Windows build to winget (no second pull request while the