fix(node): remove the Compose projects of a lane whose slot is gone (#394)

CHANGELOG.md

@@ -51,6 +51,17 @@ and listed with their assets on the
 - **`ServiceView.singleton`** (ADR-0080, ADR-0081): whether a service runs
   once per machine, in the main working tree only.
 
+### Fixed
+
+- **A node no longer leaves a gone lane's containers running** (#394): a
+  slot removed without its purge (one without `slot.json`, or by an agent
+  of an earlier release) left its Compose projects and networks running
+  for good, until the engine had no network left. Every ten minutes the
+  agent now removes the Compose projects of its lanes whose slot is gone
+  (their volumes stay), only its own (they ran under its slots). A lane
+  that cannot get a network because Docker's address pools are used up
+  says so; `docs/pool.md` says how to give Docker more.
+
 ## [0.32.0] - 2026-10-30
 
 A pool's project skills stay in their repository: a skill bound with

crates/devstack-node/src/lanerun.rs

@@ -730,7 +730,18 @@ impl LaneRun {
         }
         Docker::clean()
             .ensure_network(&self.dir, name, &flags)
-            .map_err(RunError::from)
+            .map_err(|e| {
+                let text = e.to_string();
+                // Docker's address pools are used up (#394): the node's
+                // engine, not the lane, is the trouble.
+                if text.contains("address pools") {
+                    RunError::Docker(format!(
+                        "this node's Docker has no network address left (its address pools are used up), so the lane cannot get a network; its admin can give Docker more (`default-address-pools`, see docs/pool.md): {text}"
+                    ))
+                } else {
+                    RunError::from(e)
+                }
+            })
     }
 
     /// Start the lane: its infrastructure checked, its images and containers
@@ -1259,6 +1270,49 @@ fn own_user() -> Option<String> {
     Some(format!("{}:{}", field("Uid:")?, field("Gid:")?))
 }
 
+/// Take down Compose project `project` of a lane whose slot is gone (#394):
+/// its containers and networks, as `compose down` does; its volumes stay,
+/// as a kept lane's data does (ADR-0038). `agent`: the agent's own
+/// container, which leaves the project's networks first.
+pub(crate) fn remove_orphan_project(project: &str, agent: Option<&str>) {
+    if let Some(me) = agent {
+        for n in project_networks(project, &[]) {
+            let _ = docker(&["network", "disconnect", "-f", &n, me]);
+        }
+    }
+    let p = Project {
+        project: project.to_string(),
+        networks: Vec::new(),
+    };
+    remove_project(&p, false);
+}
+
+/// Remove the networks a lane of slot `slot` made with a fixed name of its
+/// group's, renamed `<name>-<slot>` (ADR-0030 point 2, `devstack.lane=1`),
+/// once its projects are gone: nothing else is on them.
+pub(crate) fn remove_slot_networks(slot: &str, agent: Option<&str>) {
+    let suffix = format!("-{slot}");
+    let names = docker(&[
+        "network",
+        "ls",
+        "--filter",
+        "label=devstack.lane=1",
+        "--format",
+        "{{.Name}}",
+    ])
+    .unwrap_or_default();
+    for n in names
+        .lines()
+        .map(str::trim)
+        .filter(|n| n.ends_with(&suffix))
+    {
+        if let Some(me) = agent {
+            let _ = docker(&["network", "disconnect", "-f", n, me]);
+        }
+        let _ = docker(&["network", "rm", n]);
+    }
+}
+
 /// The networks of a group's Compose project (by Compose's project label),
 /// and the group's own fixed-name networks: what a lane container joins.
 fn project_networks(project: &str, own: &[String]) -> Vec<String> {

crates/devstack-node/src/node.rs

@@ -838,6 +838,43 @@ impl Inner {
         });
     }
 
+    /// The Compose projects of lanes whose slot is gone (#394): no lease
+    /// holds the slot, it has no directory and is not being purged. A slot
+    /// directory removed without its purge (one without `slot.json`, or an
+    /// agent of an earlier release) left them running for good. Only this
+    /// agent's, by where Compose ran them (under its slots): another pool's
+    /// agent on the same engine keeps its own (ADR-0034). Their volumes stay.
+    fn sweep_orphan_projects(&self) {
+        let slots = self.slots();
+        let known = |slot: &str| {
+            slots.join(slot).exists()
+                || self
+                    .leases
+                    .lock()
+                    .unwrap()
+                    .values()
+                    .any(|l| l.slot.as_deref() == Some(slot))
+                || self.slot_busy.lock().unwrap().contains_key(slot)
+                || self.slot_purging.lock().unwrap().contains(slot)
+        };
+        let agent = self.container.as_deref();
+        let mut gone = HashSet::new();
+        for (project, slot) in orphan_projects(&compose_projects(), &slots, &known) {
+            // Asked again right before: a lane of the slot may start now.
+            if known(&slot) {
+                continue;
+            }
+            eprintln!(
+                "Compose project `{project}` of a lane that is gone: removed (its volumes stay)"
+            );
+            crate::lanerun::remove_orphan_project(&project, agent);
+            gone.insert(slot);
+        }
+        for slot in gone {
+            crate::lanerun::remove_slot_networks(&slot, agent);
+        }
+    }
+
     fn update_state(&self, at: u64, state: &str, target: Option<String>, error: Option<String>) {
         *self.update.lock().unwrap() = Some(crate::coordinator::UpdateReport {
             at,
@@ -3442,6 +3479,50 @@ fn remove_lane_images(owner: &str, gone: impl Fn(&str) -> bool) {
     remove_images(&ids);
 }
 
+/// Every Compose project on the engine with the directory Compose ran it
+/// in: `<project>\t<working_dir>` a line, once per container.
+fn compose_projects() -> String {
+    std::process::Command::new("docker")
+        .args([
+            "ps",
+            "-a",
+            "--filter",
+            "label=com.docker.compose.project",
+            "--format",
+            "{{.Label \"com.docker.compose.project\"}}\t{{.Label \"com.docker.compose.project.working_dir\"}}",
+        ])
+        .output()
+        .map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
+        .unwrap_or_default()
+}
+
+/// The projects in `listing` ([`compose_projects`]) of a lane of this
+/// agent's whose slot `known` does not know, each once with its slot: named
+/// `<project>-<slot>` and run from under `slots/<slot>`.
+fn orphan_projects(
+    listing: &str,
+    slots: &Path,
+    known: &dyn Fn(&str) -> bool,
+) -> Vec<(String, String)> {
+    let mut out: Vec<(String, String)> = Vec::new();
+    for line in listing.lines() {
+        let Some((project, dir)) = line.trim_end_matches('\r').split_once('\t') else {
+            continue;
+        };
+        let Some(slot) = slot_of_project(project) else {
+            continue;
+        };
+        if !Path::new(dir).starts_with(slots.join(slot))
+            || known(slot)
+            || out.iter().any(|(p, _)| p == project)
+        {
+            continue;
+        }
+        out.push((project.to_string(), slot.to_string()));
+    }
+    out
+}
+
 /// A lane container on the engine, any agent's.
 struct LaneContainer {
     lease: String,
@@ -3601,6 +3682,7 @@ async fn background(s: Shared) {
                 }
                 s2.purge_gone_members();
                 s2.sweep_slots();
+                s2.sweep_orphan_projects();
             })
             .await;
         }
@@ -4121,6 +4203,29 @@ pub fn serve_blocking(config: NodeConfig, listen: std::net::SocketAddr) -> Resul
 mod tests {
     use super::*;
 
+    #[test]
+    fn a_gone_lanes_projects_are_found_by_slot_and_place() {
+        let gone = "d4713a1a05c412cb60a5755a";
+        let held = "591f02c55bcd63c909ffd5a3";
+        let slots = Path::new("/srv/node/slots");
+        let listing = format!(
+            "app-dev-{gone}\t/srv/node/slots/{gone}/src/deploy\n\
+             app-dev-{gone}\t/srv/node/slots/{gone}/src/deploy\r\n\
+             app-dev-search-{gone}\t/srv/node/slots/{gone}/src/deploy/es\n\
+             app-dev-{held}\t/srv/node/slots/{held}/src/deploy\n\
+             other-{gone}\t/srv/another-pool/slots/{gone}/src\n\
+             website\t/opt/website\n\n"
+        );
+        let found = orphan_projects(&listing, slots, &|s| s == held);
+        assert_eq!(
+            found,
+            [
+                (format!("app-dev-{gone}"), gone.to_string()),
+                (format!("app-dev-search-{gone}"), gone.to_string()),
+            ]
+        );
+    }
+
     #[test]
     fn lane_containers_tell_their_agent() {
         let c = parse_lane_containers(

docs/pool.md

@@ -129,6 +129,24 @@ why. A member that only shares from the dashboard is no node.
   again when it starts; for as long, a requester that never comes back
   holds the lane's CPU, memory and disk on the node. Ending the lease in
   the pool area frees them at once.
+- Docker's networks: each lane takes one or more (its own, one per Compose
+  project, and its groups' fixed-name ones). With Docker's default address
+  pools an engine has about 30 bridge networks in all, which a host that
+  runs other services too fills quickly; a lane then fails with "this
+  node's Docker has no network address left". Give the engine smaller
+  networks in `/etc/docker/daemon.json` and restart it (every container on
+  the host restarts with it):
+
+  ```json
+  { "default-address-pools": [{ "base": "10.200.0.0/16", "size": 24 }] }
+  ```
+
+  That is 256 networks of 254 addresses each; choose a `base` no network
+  of yours uses.
+- The agent removes what a lane left behind: lane containers no lease
+  holds when it starts, and every ten minutes the Compose projects of a
+  lane whose slot is gone (its volumes stay), only its own (they ran under
+  its slots).
 - It reads each lane's Compose model in a throwaway `docker:cli` container
   (`DEVSTACK_NODE_COMPOSE_IMAGE` names another image with `docker
   compose`), pulled once.