From f77b0951c609e949a3a40b5f8d146eac46c025b0 Mon Sep 17 00:00:00 2001 From: Sebastian Gieseler Date: Fri, 30 Oct 2026 23:31:17 +0100 Subject: [PATCH] fix(node): remove the Compose projects of a lane whose slot is gone (#394) A slot directory removed without its purge (one without `slot.json` at the agent's start, or by an agent of an earlier release) left the lane's Compose projects and its groups' fixed-name networks running for good. On a shared host that used up Docker's address pools, and no lane could get a network. Every purge round (ten minutes) the agent now removes the projects of its lanes whose slot no lease holds, has no directory and is not being purged, found by name (`-`) and by where Compose ran them (under its slots, so another pool's agent keeps its own); their volumes stay. A lane whose network cannot be made because the pools are used up says so, and docs/pool.md says how to give Docker more. Closes #394 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++ crates/devstack-node/src/lanerun.rs | 56 ++++++++++++++- crates/devstack-node/src/node.rs | 105 ++++++++++++++++++++++++++++ docs/pool.md | 18 +++++ 4 files changed, 189 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 178f17e0..f49bd488 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,17 @@ and listed with their assets on the - **`ServiceView.singleton`** (ADR-0080, ADR-0081): whether a service runs once per machine, in the main working tree only. +### Fixed + +- **A node no longer leaves a gone lane's containers running** (#394): a + slot removed without its purge (one without `slot.json`, or by an agent + of an earlier release) left its Compose projects and networks running + for good, until the engine had no network left. Every ten minutes the + agent now removes the Compose projects of its lanes whose slot is gone + (their volumes stay), only its own (they ran under its slots). A lane + that cannot get a network because Docker's address pools are used up + says so; `docs/pool.md` says how to give Docker more. + ## [0.32.0] - 2026-10-30 A pool's project skills stay in their repository: a skill bound with diff --git a/crates/devstack-node/src/lanerun.rs b/crates/devstack-node/src/lanerun.rs index c5302a4f..e6639665 100644 --- a/crates/devstack-node/src/lanerun.rs +++ b/crates/devstack-node/src/lanerun.rs @@ -730,7 +730,18 @@ impl LaneRun { } Docker::clean() .ensure_network(&self.dir, name, &flags) - .map_err(RunError::from) + .map_err(|e| { + let text = e.to_string(); + // Docker's address pools are used up (#394): the node's + // engine, not the lane, is the trouble. + if text.contains("address pools") { + RunError::Docker(format!( + "this node's Docker has no network address left (its address pools are used up), so the lane cannot get a network; its admin can give Docker more (`default-address-pools`, see docs/pool.md): {text}" + )) + } else { + RunError::from(e) + } + }) } /// Start the lane: its infrastructure checked, its images and containers @@ -1259,6 +1270,49 @@ fn own_user() -> Option { Some(format!("{}:{}", field("Uid:")?, field("Gid:")?)) } +/// Take down Compose project `project` of a lane whose slot is gone (#394): +/// its containers and networks, as `compose down` does; its volumes stay, +/// as a kept lane's data does (ADR-0038). `agent`: the agent's own +/// container, which leaves the project's networks first. +pub(crate) fn remove_orphan_project(project: &str, agent: Option<&str>) { + if let Some(me) = agent { + for n in project_networks(project, &[]) { + let _ = docker(&["network", "disconnect", "-f", &n, me]); + } + } + let p = Project { + project: project.to_string(), + networks: Vec::new(), + }; + remove_project(&p, false); +} + +/// Remove the networks a lane of slot `slot` made with a fixed name of its +/// group's, renamed `-` (ADR-0030 point 2, `devstack.lane=1`), +/// once its projects are gone: nothing else is on them. +pub(crate) fn remove_slot_networks(slot: &str, agent: Option<&str>) { + let suffix = format!("-{slot}"); + let names = docker(&[ + "network", + "ls", + "--filter", + "label=devstack.lane=1", + "--format", + "{{.Name}}", + ]) + .unwrap_or_default(); + for n in names + .lines() + .map(str::trim) + .filter(|n| n.ends_with(&suffix)) + { + if let Some(me) = agent { + let _ = docker(&["network", "disconnect", "-f", n, me]); + } + let _ = docker(&["network", "rm", n]); + } +} + /// The networks of a group's Compose project (by Compose's project label), /// and the group's own fixed-name networks: what a lane container joins. fn project_networks(project: &str, own: &[String]) -> Vec { diff --git a/crates/devstack-node/src/node.rs b/crates/devstack-node/src/node.rs index 1a144624..61d8d596 100644 --- a/crates/devstack-node/src/node.rs +++ b/crates/devstack-node/src/node.rs @@ -838,6 +838,43 @@ impl Inner { }); } + /// The Compose projects of lanes whose slot is gone (#394): no lease + /// holds the slot, it has no directory and is not being purged. A slot + /// directory removed without its purge (one without `slot.json`, or an + /// agent of an earlier release) left them running for good. Only this + /// agent's, by where Compose ran them (under its slots): another pool's + /// agent on the same engine keeps its own (ADR-0034). Their volumes stay. + fn sweep_orphan_projects(&self) { + let slots = self.slots(); + let known = |slot: &str| { + slots.join(slot).exists() + || self + .leases + .lock() + .unwrap() + .values() + .any(|l| l.slot.as_deref() == Some(slot)) + || self.slot_busy.lock().unwrap().contains_key(slot) + || self.slot_purging.lock().unwrap().contains(slot) + }; + let agent = self.container.as_deref(); + let mut gone = HashSet::new(); + for (project, slot) in orphan_projects(&compose_projects(), &slots, &known) { + // Asked again right before: a lane of the slot may start now. + if known(&slot) { + continue; + } + eprintln!( + "Compose project `{project}` of a lane that is gone: removed (its volumes stay)" + ); + crate::lanerun::remove_orphan_project(&project, agent); + gone.insert(slot); + } + for slot in gone { + crate::lanerun::remove_slot_networks(&slot, agent); + } + } + fn update_state(&self, at: u64, state: &str, target: Option, error: Option) { *self.update.lock().unwrap() = Some(crate::coordinator::UpdateReport { at, @@ -3442,6 +3479,50 @@ fn remove_lane_images(owner: &str, gone: impl Fn(&str) -> bool) { remove_images(&ids); } +/// Every Compose project on the engine with the directory Compose ran it +/// in: `\t` a line, once per container. +fn compose_projects() -> String { + std::process::Command::new("docker") + .args([ + "ps", + "-a", + "--filter", + "label=com.docker.compose.project", + "--format", + "{{.Label \"com.docker.compose.project\"}}\t{{.Label \"com.docker.compose.project.working_dir\"}}", + ]) + .output() + .map(|o| String::from_utf8_lossy(&o.stdout).into_owned()) + .unwrap_or_default() +} + +/// The projects in `listing` ([`compose_projects`]) of a lane of this +/// agent's whose slot `known` does not know, each once with its slot: named +/// `-` and run from under `slots/`. +fn orphan_projects( + listing: &str, + slots: &Path, + known: &dyn Fn(&str) -> bool, +) -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = Vec::new(); + for line in listing.lines() { + let Some((project, dir)) = line.trim_end_matches('\r').split_once('\t') else { + continue; + }; + let Some(slot) = slot_of_project(project) else { + continue; + }; + if !Path::new(dir).starts_with(slots.join(slot)) + || known(slot) + || out.iter().any(|(p, _)| p == project) + { + continue; + } + out.push((project.to_string(), slot.to_string())); + } + out +} + /// A lane container on the engine, any agent's. struct LaneContainer { lease: String, @@ -3601,6 +3682,7 @@ async fn background(s: Shared) { } s2.purge_gone_members(); s2.sweep_slots(); + s2.sweep_orphan_projects(); }) .await; } @@ -4121,6 +4203,29 @@ pub fn serve_blocking(config: NodeConfig, listen: std::net::SocketAddr) -> Resul mod tests { use super::*; + #[test] + fn a_gone_lanes_projects_are_found_by_slot_and_place() { + let gone = "d4713a1a05c412cb60a5755a"; + let held = "591f02c55bcd63c909ffd5a3"; + let slots = Path::new("/srv/node/slots"); + let listing = format!( + "app-dev-{gone}\t/srv/node/slots/{gone}/src/deploy\n\ + app-dev-{gone}\t/srv/node/slots/{gone}/src/deploy\r\n\ + app-dev-search-{gone}\t/srv/node/slots/{gone}/src/deploy/es\n\ + app-dev-{held}\t/srv/node/slots/{held}/src/deploy\n\ + other-{gone}\t/srv/another-pool/slots/{gone}/src\n\ + website\t/opt/website\n\n" + ); + let found = orphan_projects(&listing, slots, &|s| s == held); + assert_eq!( + found, + [ + (format!("app-dev-{gone}"), gone.to_string()), + (format!("app-dev-search-{gone}"), gone.to_string()), + ] + ); + } + #[test] fn lane_containers_tell_their_agent() { let c = parse_lane_containers( diff --git a/docs/pool.md b/docs/pool.md index 441d601d..9fa197f9 100644 --- a/docs/pool.md +++ b/docs/pool.md @@ -129,6 +129,24 @@ why. A member that only shares from the dashboard is no node. again when it starts; for as long, a requester that never comes back holds the lane's CPU, memory and disk on the node. Ending the lease in the pool area frees them at once. +- Docker's networks: each lane takes one or more (its own, one per Compose + project, and its groups' fixed-name ones). With Docker's default address + pools an engine has about 30 bridge networks in all, which a host that + runs other services too fills quickly; a lane then fails with "this + node's Docker has no network address left". Give the engine smaller + networks in `/etc/docker/daemon.json` and restart it (every container on + the host restarts with it): + + ```json + { "default-address-pools": [{ "base": "10.200.0.0/16", "size": 24 }] } + ``` + + That is 256 networks of 254 addresses each; choose a `base` no network + of yours uses. +- The agent removes what a lane left behind: lane containers no lease + holds when it starts, and every ten minutes the Compose projects of a + lane whose slot is gone (its volumes stay), only its own (they ran under + its slots). - It reads each lane's Compose model in a throwaway `docker:cli` container (`DEVSTACK_NODE_COMPOSE_IMAGE` names another image with `docker compose`), pulled once.