diff --git a/dev2/README.md b/dev2/README.md index f8ee3f1..5bf5652 100644 --- a/dev2/README.md +++ b/dev2/README.md @@ -134,6 +134,22 @@ icon inlined as a data: URI when it is under 48 KB, else a text wordmark of the domain. `sites.d` `"maintenance": {"name": ..., "color": ..., "icon": "/logo.svg" | false}` overrides. If the site does not answer at render time an existing page is kept. +The page says the site is down for maintenance, not "coming soon" or "updating". + +### No maintenance page at all: a standby port + +`sites.d` `"standby_port": ` (convention: the app port + 10000) makes deploys +seamless. `vhost` adds `server 127.0.0.1: backup;` to the app's upstream and +`provision` writes `STANDBY_PORT` into `deploy.env`. `deploy-app.sh` then starts the new +build, app service only, as its own compose project (`-standby`) on that port, +waits for it to answer, and only then recreates the main container. While the main one +restarts, nginx's refused connection goes on to the backup (a POST that was never sent +is retried too). After the main container is healthy it waits 12 s (nginx's 10 s +`fail_timeout`) and removes the standby. An app that needs its stack's siblings (redis, +a db service) will not answer there; the deploy then carries on without it, maintenance +page and all. On by hand for nixamp.com (2026-10-21): 37/37 requests answered 200 +through a full redeploy, none got the page. + ## Per-site overrides `sites.d/.json` is merged over the registry entry at load time. One diff --git a/dev2/dev2-site b/dev2/dev2-site index 5d8c58f..f1eddbe 100755 --- a/dev2/dev2-site +++ b/dev2/dev2-site @@ -1189,6 +1189,7 @@ def cmd_provision(args): key_alias = f"github.com-{s['repo'].replace('/', '__')}" repo_url = f"git@{key_alias}:{s['repo']}.git" if private else f"https://github.com/{s['repo']}.git" deploy_env = f"SITE={s['site']}\nREPO={repo_url}\nBRANCH={branch}\nAPP_PORT={s['port']}\nBUILD_SERVICES=\"{build_services}\"\nHEALTH_PATH={hp}\nIMAGE_ONLY={'1' if image_only else '0'}\n" + if s.get("standby_port"): deploy_env += f"STANDBY_PORT={s['standby_port']}\n" # Secrets come from the logicsrc team vault on every deploy (deploy-app.sh # sync_env_from_vault): sites.d "vault" names it, else --prod; "vault": "" # turns it off. env_remove keys are never taken from the vault. @@ -1545,7 +1546,7 @@ def render_vhost(s): blk = blk.replace(anchor, p["locations"].rstrip("\n") + "\n\n" + anchor) conf += "\n# " + p["host"] + " -> 127.0.0.1:" + str(p["port"]) + "\n" + blk conf += "\nserver {\n listen 80;\n listen [::]:80;\n server_name " + p["host"] + ";\n return 301 https://$host$request_uri;\n}\n" - return keepalive_upstreams(conf, s["site"]), domains + return keepalive_upstreams(conf, s["site"], {str(s["port"]): s["standby_port"]} if s.get("port") and s.get("standby_port") else {}), domains # nginx -> app keepalive. Without an upstream block nginx opens a new TCP connection to # the app for every request, and the box-wide $connection_upgrade map sends @@ -1558,13 +1559,21 @@ KEEPALIVE_MAP = "/etc/nginx/conf.d/profullstack-keepalive.conf" KEEPALIVE_MAP_BODY = (f"# managed by cli-tools/dev2/dev2-site vhost: see keepalive_upstreams\n" "map $http_upgrade $connection_upgrade_keepalive {\n default upgrade;\n '' '';\n}\n") -def keepalive_upstreams(conf, site_name): +# A site with "standby_port" in sites.d gets that port as the `backup` server of its +# app's upstream. deploy-app.sh starts the new build there before it recreates the main +# container, so while the main one restarts nginx's refused connection goes on to the +# standby (proxy_next_upstream's default `error` retries even a POST that was never +# sent) and nobody sees the maintenance page. With no standby running, the backup +# refuses too and the maintenance page answers as before. +def keepalive_upstreams(conf, site_name, standby=None): key = re.sub(r"[^A-Za-z0-9]", "_", site_name) ports = sorted(set(re.findall(r"proxy_pass http://127\.0\.0\.1:(\d+);", conf)), key=int) if not ports: return conf conf = re.sub(r"proxy_pass http://127\.0\.0\.1:(\d+);", lambda m: f"proxy_pass http://dev2_{key}_{m.group(1)};", conf) conf = conf.replace("proxy_set_header Connection $connection_upgrade;", "proxy_set_header Connection $connection_upgrade_keepalive;") - ups = "".join(f"upstream dev2_{key}_{p} {{\n server 127.0.0.1:{p};\n keepalive 16;\n keepalive_timeout 4s;\n}}\n\n" for p in ports) + standby = standby or {} + backup = lambda p: f" server 127.0.0.1:{standby[p]} backup;\n" if standby.get(p) else "" + ups = "".join(f"upstream dev2_{key}_{p} {{\n server 127.0.0.1:{p};\n{backup(p)} keepalive 16;\n keepalive_timeout 4s;\n}}\n\n" for p in ports) head, sep, rest = conf.partition("\nserver {") return head + "\n" + ups.rstrip("\n") + "\n" + sep + rest diff --git a/dev2/sites.d/nixamp.com.json b/dev2/sites.d/nixamp.com.json index 3a5da83..5a59848 100644 --- a/dev2/sites.d/nixamp.com.json +++ b/dev2/sites.d/nixamp.com.json @@ -1,4 +1,6 @@ { "mem_limit": "2g", - "mem_note": "2026-10-01: Bun, 540 MB steady, 690 MB peak" + "mem_note": "2026-10-01: Bun, 540 MB steady, 690 MB peak", + "standby_port": 13216, + "standby_note": "2026-10-21: deploys start the new build here first (nginx backup), so nixamp.com never shows the maintenance page" } diff --git a/dev2/templates/deploy-app.sh b/dev2/templates/deploy-app.sh index 743a1ef..325c402 100755 --- a/dev2/templates/deploy-app.sh +++ b/dev2/templates/deploy-app.sh @@ -164,10 +164,10 @@ on_err() { } health() { - local i + local i port=${1:-$APP_PORT} for i in $(seq 1 "$((HEALTH_TIMEOUT / 5))"); do # Any HTTP answer counts: a 3xx redirect or a 401 from the app is alive. - if curl -sS -m 5 -o /dev/null -w '%{http_code}' "http://127.0.0.1:${APP_PORT}${HEALTH_PATH}" | grep -Eq '^[1-4]'; then return 0; fi + if curl -sS -m 5 -o /dev/null -w '%{http_code}' "http://127.0.0.1:${port}${HEALTH_PATH}" | grep -Eq '^[1-4]'; then return 0; fi sleep 5 done return 1 @@ -270,6 +270,28 @@ if [ -n "$WANT_SUBNET" ] && [ -n "$APP_CID" ]; then fi fi +# No maintenance page during a deploy (STANDBY_PORT in deploy.env, from sites.d +# "standby_port"): the new build answers on the standby port first, which nginx has as +# the `backup` server of the app's upstream, so while the main container is recreated +# requests go there. Only the app service runs there, in a project of its own; one that +# needs its stack's siblings (redis, db) will not come up healthy, and the deploy goes +# on without it, maintenance page and all, as before. +STANDBY_ON=0 +STANDBY_PROJECT="$(basename "$ROOT" | tr 'A-Z' 'a-z' | tr -c 'a-z0-9\n' '-')-standby" +standby() { (cd "$ROOT" && APP_PORT="$STANDBY_PORT" docker compose -p "$STANDBY_PROJECT" -f docker-compose.app.yml --env-file "$ROOT/deploy.env" "$@"); } +standby_down() { [ "$STANDBY_ON" = 1 ] || return 0; STANDBY_ON=0; standby down --remove-orphans >/dev/null 2>&1 || true; log "Standby removed"; } +if [ -n "${STANDBY_PORT:-}" ] && [ "$RECREATE" = 0 ]; then + log "Standby: the new build on 127.0.0.1:$STANDBY_PORT while the main container restarts" + standby down --remove-orphans >/dev/null 2>&1 || true + if standby up -d --no-build --no-deps app && HEALTH_TIMEOUT=120 health "$STANDBY_PORT"; then + STANDBY_ON=1 + trap standby_down EXIT + else + log "Standby did not answer; deploying without it" + standby down --remove-orphans >/dev/null 2>&1 || true + fi +fi + # From here on the running stack is touched: any failure rolls back to the images # it was running. snapshot @@ -306,6 +328,9 @@ if health; then echo "DEPLOYED_AT=$(date -u +%FT%TZ)" } > "$STATE" compose ps + # nginx marks a refused server down for fail_timeout (10 s): give it time to come + # back to the main container before the standby goes. + if [ "$STANDBY_ON" = 1 ]; then sleep 12; standby_down; fi else trap - ERR log "UNHEALTHY after ${HEALTH_TIMEOUT}s - last 60 lines:" diff --git a/dev2/templates/maintenance.html b/dev2/templates/maintenance.html index 64681f6..fb0c72b 100644 --- a/dev2/templates/maintenance.html +++ b/dev2/templates/maintenance.html @@ -8,7 +8,7 @@ -__NAME__ is updating +__NAME__ is down for maintenance __FAVICON__