From 49cc478c52f6ed98eb057b041b98ee79ca613b84 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Thu, 8 Oct 2026 13:03:03 -0700
Subject: [PATCH 01/20] Refactor testbench

---
 .github/scripts/deploy-instance.sh           | 30 +++++++----
 .github/scripts/disable-testbench-relay.sh   | 27 ++++++++++
 .github/scripts/enable-testbench-relay.sh    | 54 ++++++++++++++++++++
 .github/templates/testbench-relay.yaml       | 14 +++++
 .github/workflows/deploy-pm4.yml             | 39 +++++++++++++-
 .github/workflows/nightly-testbench-arch.yml | 39 +++++++++++++-
 6 files changed, 190 insertions(+), 13 deletions(-)
 create mode 100755 .github/scripts/disable-testbench-relay.sh
 create mode 100755 .github/scripts/enable-testbench-relay.sh
 create mode 100644 .github/templates/testbench-relay.yaml

diff --git a/.github/scripts/deploy-instance.sh b/.github/scripts/deploy-instance.sh
index ccf185c..9d72a27 100644
--- a/.github/scripts/deploy-instance.sh
+++ b/.github/scripts/deploy-instance.sh
@@ -4,6 +4,20 @@ current_datetime=$(echo -n ${CURRENT_DATE} | md5sum | head -c 10)
 echo "NAMESPACE : ci-{{INSTANCE}}-ns-pm4"
 helm repo add processmaker ${HELM_REPO} --username ${HELM_USERNAME} --password ${HELM_PASSWORD} && helm repo update
 
+helm_common_sets() {
+  echo \
+    --set deploy.web.octane.enable=${OCTANE} \
+    --set deploy.pmai.openaiApiKey=${OPENAI_API_KEY} \
+    --set analytics.awsAccessKey=${ANALYTICS_AWS_ACCESS_KEY} \
+    --set analytics.awsSecretKey=${ANALYTICS_AWS_SECRET_KEY} \
+    --set dockerRegistry.password=${REGISTRY_PASSWORD} \
+    --set dockerRegistry.url=${REGISTRY_HOST} \
+    --set dockerRegistry.username=${REGISTRY_USERNAME} \
+    --set twilio.sid=${TWILIO_SID} \
+    --set twilio.token=${TWILIO_TOKEN} \
+    --set appVersion=${APP_VERSION}
+}
+
 if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then
     echo "New instance. Creating Namespace"
     kubectl create namespace ci-{{INSTANCE}}-ns-pm4
@@ -34,20 +48,14 @@ if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then
     APP_VERSION=$(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g" | sed "s/refs-heads-//g")
 
     helm install --timeout 75m -f .github/templates/instance.yaml ci-{{INSTANCE}} processmaker/enterprise \
-        --set deploy.web.octane.enable=${OCTANE} \
-        --set deploy.pmai.openaiApiKey=${OPENAI_API_KEY} \
-        --set analytics.awsAccessKey=${ANALYTICS_AWS_ACCESS_KEY} \
-        --set analytics.awsSecretKey=${ANALYTICS_AWS_SECRET_KEY} \
-        --set dockerRegistry.password=${REGISTRY_PASSWORD} \
-        --set dockerRegistry.url=${REGISTRY_HOST} \
-        --set dockerRegistry.username=${REGISTRY_USERNAME} \
-        --set twilio.sid=${TWILIO_SID} \
-        --set twilio.token=${TWILIO_TOKEN} \
-        --set appVersion=${APP_VERSION} \
+        $(helm_common_sets) \
         --version ${versionHelm}
 else
     echo "Instance exists. Running upgrade and bouncing pods"
-    helm upgrade --timeout 60m ci-{{INSTANCE}} processmaker/enterprise --version ${versionHelm}
+    APP_VERSION=$(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g" | sed "s/refs-heads-//g")
+    helm upgrade --timeout 60m -f .github/templates/instance.yaml ci-{{INSTANCE}} processmaker/enterprise \
+        $(helm_common_sets) \
+        --version ${versionHelm}
     
     #Bounce pods
     webPod=$(kubectl get pods -n ci-{{INSTANCE}}-ns-pm4|grep web|awk '{print $1}')
diff --git a/.github/scripts/disable-testbench-relay.sh b/.github/scripts/disable-testbench-relay.sh
new file mode 100755
index 0000000..a203c1c
--- /dev/null
+++ b/.github/scripts/disable-testbench-relay.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required}"
+RELEASE="ci-${INSTANCE}"
+
+if [ -z "${versionHelm:-}" ]; then
+  echo "versionHelm is required"
+  exit 1
+fi
+
+if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then
+  echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required"
+  exit 1
+fi
+
+helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true
+helm repo update
+
+echo "Disabling testbench runner relay for release ${RELEASE}..."
+helm upgrade --timeout 10m "${RELEASE}" processmaker/enterprise \
+  --reuse-values \
+  --set testbenchRunnerRelay.enable=false \
+  --set testbenchRunnerRelay.tailscale.authKey= \
+  --version "${versionHelm}"
+
+echo "Relay disabled."
diff --git a/.github/scripts/enable-testbench-relay.sh b/.github/scripts/enable-testbench-relay.sh
new file mode 100755
index 0000000..621fff4
--- /dev/null
+++ b/.github/scripts/enable-testbench-relay.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}"
+RELEASE="ci-${INSTANCE}"
+NAMESPACE="${RELEASE}-ns-pm4"
+DEPLOYMENT="${RELEASE}-testbench-runner-relay"
+
+if [ -z "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
+  echo "TAILSCALE_AUTH_KEY_CI_RELAY is required"
+  exit 1
+fi
+
+if [ -z "${versionHelm:-}" ]; then
+  echo "versionHelm is required"
+  exit 1
+fi
+
+if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then
+  echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required"
+  exit 1
+fi
+
+RELAY_VALUES="${RELAY_VALUES_FILE:-.github/templates/testbench-relay.yaml}"
+if [ ! -f "${RELAY_VALUES}" ]; then
+  echo "Relay values file not found: ${RELAY_VALUES}"
+  exit 1
+fi
+
+RELAY_VALUES_RENDERED=$(mktemp)
+sed "s#{{INSTANCE}}#${INSTANCE}#g" "${RELAY_VALUES}" > "${RELAY_VALUES_RENDERED}"
+trap 'rm -f "${RELAY_VALUES_RENDERED}"' EXIT
+
+helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true
+helm repo update
+
+echo "Enabling testbench runner relay for release ${RELEASE} (upstream tb-ci-${INSTANCE})..."
+helm upgrade --timeout 15m "${RELEASE}" processmaker/enterprise \
+  --reuse-values \
+  -f "${RELAY_VALUES_RENDERED}" \
+  --set testbenchRunnerRelay.enable=true \
+  --set "testbenchRunnerRelay.upstreamHost=tb-ci-${INSTANCE}" \
+  --set "testbenchRunnerRelay.tailscale.authKey=${TAILSCALE_AUTH_KEY_CI_RELAY}" \
+  --version "${versionHelm}"
+
+if kubectl get deployment "${DEPLOYMENT}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  kubectl rollout status "deployment/${DEPLOYMENT}" -n "${NAMESPACE}" --timeout=180s
+else
+  echo "Warning: deployment ${DEPLOYMENT} not found after upgrade"
+  kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true
+  exit 1
+fi
+
+echo "Testbench runner relay is ready."
diff --git a/.github/templates/testbench-relay.yaml b/.github/templates/testbench-relay.yaml
new file mode 100644
index 0000000..e7a5fc9
--- /dev/null
+++ b/.github/templates/testbench-relay.yaml
@@ -0,0 +1,14 @@
+# Overlay for testbench jobs only (helm upgrade --reuse-values -f this file).
+# Do not enable relay on default CI deploy.
+testbenchRunnerRelay:
+  enable: true
+  upstreamHost: tb-ci-{{INSTANCE}}
+  forwards:
+    - name: smtp
+      listen: 587
+      upstream: 587
+      protocol: tcp
+    - name: imaps
+      listen: 993
+      upstream: 993
+      protocol: tcp
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index c8d9655..d231cc5 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -364,6 +364,40 @@ jobs:
       contains(github.event.pull_request.body, 'ci:run-testbench')
     runs-on: ${{ vars.RUNNER }}
     steps:
+      - name: Checkout dot-github
+        uses: actions/checkout@v4
+        with:
+          repository: processmaker/.github
+          path: dot-github
+          token: ${{ secrets.GIT_TOKEN }}
+
+      - name: Common
+        uses: ./dot-github/.github/actions/common
+        with:
+          token: ${{ secrets.GIT_TOKEN }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Authenticate with Amazon EKS
+        run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng
+
+      - name: Enable testbench runner relay
+        working-directory: dot-github
+        env:
+          HELM_REPO: ${{ secrets.HELM_REPO }}
+          HELM_USERNAME: ${{ secrets.HELM_USERNAME }}
+          HELM_PASSWORD: ${{ secrets.HELM_PASSWORD }}
+          TAILSCALE_AUTH_KEY_CI_RELAY: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RELAY }}
+        run: |
+          instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
+          chmod +x .github/scripts/enable-testbench-relay.sh
+          .github/scripts/enable-testbench-relay.sh "$instance"
+
       - name: Checkout testbench
         uses: actions/checkout@v4
         with:
@@ -377,9 +411,12 @@ jobs:
           SITE_NAME: CI
           INSTANCE_PASSWORD: ${{ secrets.CI_INSTANCE_PASSWORD }}
           CONTEXT_PATH: ./context
-          TCP_TUNNELS: "mailserver:587 mailserver:993"
+          USE_RUNNER_RELAY: "true"
+          USE_TUNNEL: "false"
+          TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
         run: |
           instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
+          export INSTANCE="$instance"
           echo "INSTANCE: $instance"
           if [ "${{ env.MULTITENANCY }}" = "true" ]; then
             export URL="https://tenant-1.ci-$instance.engk8s.processmaker.net"
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index 427f832..4da7808 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -195,6 +195,40 @@ jobs:
       matrix:
         tenant: ${{ fromJSON(inputs.tenants_json) }}
     steps:
+      - name: Checkout dot-github
+        uses: actions/checkout@v4
+        with:
+          repository: processmaker/.github
+          path: dot-github
+          token: ${{ secrets.GIT_TOKEN }}
+
+      - name: Common
+        uses: ./dot-github/.github/actions/common
+        with:
+          token: ${{ secrets.GIT_TOKEN }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Authenticate with Amazon EKS
+        run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng
+
+      - name: Enable testbench runner relay
+        working-directory: dot-github
+        env:
+          HELM_REPO: ${{ secrets.HELM_REPO }}
+          HELM_USERNAME: ${{ secrets.HELM_USERNAME }}
+          HELM_PASSWORD: ${{ secrets.HELM_PASSWORD }}
+          TAILSCALE_AUTH_KEY_CI_RELAY: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RELAY }}
+        run: |
+          instance="${{ needs.deploy.outputs.instance }}"
+          chmod +x .github/scripts/enable-testbench-relay.sh
+          .github/scripts/enable-testbench-relay.sh "$instance"
+
       - name: Checkout testbench
         uses: actions/checkout@v4
         with:
@@ -208,9 +242,12 @@ jobs:
           SITE_NAME: CI
           INSTANCE_PASSWORD: ${{ secrets.CI_INSTANCE_PASSWORD }}
           CONTEXT_PATH: ./context
-          TCP_TUNNELS: "mailserver:587 mailserver:993"
+          USE_RUNNER_RELAY: "true"
+          USE_TUNNEL: "false"
+          TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
         run: |
           instance="${{ needs.deploy.outputs.instance }}"
+          export INSTANCE="$instance"
           tenant="${{ matrix.tenant }}"
           echo "INSTANCE: $instance"
           echo "TENANT: $tenant"

From 06c858f900f034636d3c00134ad1c72cf7244510 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Thu, 8 Oct 2026 16:29:13 -0700
Subject: [PATCH 02/20] Add ref

---
 .github/workflows/deploy-pm4.yml             | 3 +++
 .github/workflows/nightly-testbench-arch.yml | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index d231cc5..368dc64 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -368,6 +368,9 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: processmaker/.github
+          # Pin to feature branch until relay scripts land on main.
+          # uses: @sha only loads this workflow file; checkout still defaults to main.
+          ref: task/FOUR-33523-B
           path: dot-github
           token: ${{ secrets.GIT_TOKEN }}
 
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index 4da7808..bd9a9d0 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -199,6 +199,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: processmaker/.github
+          # Pin to feature branch until relay scripts land on main.
+          ref: task/FOUR-33523-B
           path: dot-github
           token: ${{ secrets.GIT_TOKEN }}
 

From 4be82883cf5c1cbb2d371bbf717665ef25d26616 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Thu, 8 Oct 2026 16:54:40 -0700
Subject: [PATCH 03/20] Update

---
 .github/scripts/enable-testbench-relay.sh | 53 ++++++++++++++++-------
 1 file changed, 37 insertions(+), 16 deletions(-)

diff --git a/.github/scripts/enable-testbench-relay.sh b/.github/scripts/enable-testbench-relay.sh
index 621fff4..85cbff8 100755
--- a/.github/scripts/enable-testbench-relay.sh
+++ b/.github/scripts/enable-testbench-relay.sh
@@ -11,16 +11,6 @@ if [ -z "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
   exit 1
 fi
 
-if [ -z "${versionHelm:-}" ]; then
-  echo "versionHelm is required"
-  exit 1
-fi
-
-if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then
-  echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required"
-  exit 1
-fi
-
 RELAY_VALUES="${RELAY_VALUES_FILE:-.github/templates/testbench-relay.yaml}"
 if [ ! -f "${RELAY_VALUES}" ]; then
   echo "Relay values file not found: ${RELAY_VALUES}"
@@ -31,20 +21,51 @@ RELAY_VALUES_RENDERED=$(mktemp)
 sed "s#{{INSTANCE}}#${INSTANCE}#g" "${RELAY_VALUES}" > "${RELAY_VALUES_RENDERED}"
 trap 'rm -f "${RELAY_VALUES_RENDERED}"' EXIT
 
-helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true
-helm repo update
+# Prefer the chart cloned by the Common action (ci:k8s-branch) so unpublished
+# template fixes are picked up without waiting on a Harbor chart publish.
+CHART_REF="${PM4_K8S_CHART:-}"
+if [ -z "${CHART_REF}" ] && [ -f ../pm4-k8s-distribution/charts/enterprise/Chart.yaml ]; then
+  CHART_REF="../pm4-k8s-distribution/charts/enterprise"
+fi
+if [ -z "${CHART_REF}" ] && [ -f pm4-k8s-distribution/charts/enterprise/Chart.yaml ]; then
+  CHART_REF="pm4-k8s-distribution/charts/enterprise"
+fi
+
+HELM_CHART_ARGS=()
+if [ -n "${CHART_REF}" ]; then
+  echo "Using local enterprise chart: ${CHART_REF}"
+  HELM_CHART_ARGS=("${CHART_REF}")
+else
+  if [ -z "${versionHelm:-}" ]; then
+    echo "versionHelm is required when no local chart is available"
+    exit 1
+  fi
+  if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then
+    echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required when no local chart is available"
+    exit 1
+  fi
+  helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true
+  helm repo update
+  HELM_CHART_ARGS=(processmaker/enterprise --version "${versionHelm}")
+fi
 
 echo "Enabling testbench runner relay for release ${RELEASE} (upstream tb-ci-${INSTANCE})..."
-helm upgrade --timeout 15m "${RELEASE}" processmaker/enterprise \
+helm upgrade --timeout 15m "${RELEASE}" "${HELM_CHART_ARGS[@]}" \
   --reuse-values \
   -f "${RELAY_VALUES_RENDERED}" \
   --set testbenchRunnerRelay.enable=true \
   --set "testbenchRunnerRelay.upstreamHost=tb-ci-${INSTANCE}" \
-  --set "testbenchRunnerRelay.tailscale.authKey=${TAILSCALE_AUTH_KEY_CI_RELAY}" \
-  --version "${versionHelm}"
+  --set "testbenchRunnerRelay.tailscale.authKey=${TAILSCALE_AUTH_KEY_CI_RELAY}"
 
 if kubectl get deployment "${DEPLOYMENT}" -n "${NAMESPACE}" >/dev/null 2>&1; then
-  kubectl rollout status "deployment/${DEPLOYMENT}" -n "${NAMESPACE}" --timeout=180s
+  if ! kubectl rollout status "deployment/${DEPLOYMENT}" -n "${NAMESPACE}" --timeout=180s; then
+    echo "Relay rollout failed; collecting diagnostics..."
+    kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true
+    kubectl describe pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true
+    kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=200 || true
+    kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --previous --tail=200 || true
+    exit 1
+  fi
 else
   echo "Warning: deployment ${DEPLOYMENT} not found after upgrade"
   kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true

From 78a3a8c0cb234c0190d775bd8d826a966ba6bdbc Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Thu, 8 Oct 2026 17:35:52 -0700
Subject: [PATCH 04/20] Pin branch

---
 .github/workflows/deploy-pm4.yml             | 2 ++
 .github/workflows/nightly-testbench-arch.yml | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 368dc64..2275112 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -405,6 +405,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ProcessMaker/testbench
+          # Pin to feature branch until relay-capable start.sh lands on main.
+          ref: task/FOUR-33523-B
           path: testbench
           token: ${{ secrets.GIT_TOKEN }}
 
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index bd9a9d0..e2d5b39 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -235,6 +235,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ProcessMaker/testbench
+          # Pin to feature branch until relay-capable start.sh lands on main.
+          ref: task/FOUR-33523-B
           path: testbench
           token: ${{ secrets.GIT_TOKEN }}
 

From 320f0240a618534a3551c6ead5cabda3c8ce39e6 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Thu, 8 Oct 2026 22:24:53 -0700
Subject: [PATCH 05/20] Add fail-fast verify for testbench runner relay path.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Probe web→relay Service and relay→tb-ci-*:587/993 before Playwright so Tailscale ACL/path failures surface with diagnostics instead of empty IMAP.
---
 .github/scripts/verify-testbench-relay.sh    | 127 +++++++++++++++++++
 .github/workflows/deploy-pm4.yml             |   1 +
 .github/workflows/nightly-testbench-arch.yml |   1 +
 3 files changed, 129 insertions(+)
 create mode 100755 .github/scripts/verify-testbench-relay.sh

diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh
new file mode 100755
index 0000000..4988c28
--- /dev/null
+++ b/.github/scripts/verify-testbench-relay.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+# Fail-fast: prove PM → in-cluster relay → Tailscale → runner mail ports.
+# Run after the runner has joined the tailnet as tb-ci-<INSTANCE> (start.sh).
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}"
+RELEASE="ci-${INSTANCE}"
+NAMESPACE="${RELEASE}-ns-pm4"
+RELAY_DEPLOY="deploy/${RELEASE}-testbench-runner-relay"
+WEB_DEPLOY="deploy/${RELEASE}-processmaker-web"
+UPSTREAM_HOST="tb-ci-${INSTANCE}"
+RELAY_SVC="${RELEASE}-testbench-relay"
+NC_TIMEOUT_SEC="${RELAY_NC_TIMEOUT_SEC:-5}"
+# Allow MagicDNS / DERP / HAProxy health to settle after runner joins.
+WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-90}"
+POLL_SEC="${RELAY_VERIFY_POLL_SEC:-5}"
+
+if ! command -v kubectl >/dev/null 2>&1; then
+  echo "kubectl is required to verify the testbench runner relay"
+  exit 1
+fi
+
+dump_diagnostics() {
+  echo ""
+  echo "======== testbench runner relay diagnostics ========"
+  echo "--- pods ---"
+  kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true
+  echo ""
+  echo "--- relay deploy / svc ---"
+  kubectl get deploy,svc -n "${NAMESPACE}" -l service=testbench-runner-relay 2>/dev/null || \
+    kubectl get deploy "${RELEASE}-testbench-runner-relay" -n "${NAMESPACE}" 2>/dev/null || true
+  kubectl get svc "${RELAY_SVC}" -n "${NAMESPACE}" 2>/dev/null || true
+  echo ""
+  echo "--- relay: tailscale status ---"
+  kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- tailscale status 2>&1 || true
+  echo ""
+  echo "--- relay: peer ${UPSTREAM_HOST} (json) ---"
+  kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- tailscale status --json 2>&1 \
+    | python3 -c "
+import json, sys
+try:
+    d = json.load(sys.stdin)
+except Exception as e:
+    print('failed to parse status json:', e)
+    sys.exit(0)
+self = d.get('Self') or {}
+print('Self:', self.get('HostName'), 'Tags:', self.get('Tags'), 'Online:', self.get('Online'))
+for p in (d.get('Peer') or {}).values():
+    host = p.get('HostName') or ''
+    if host == '${UPSTREAM_HOST}' or '${INSTANCE}' in host:
+        print(
+            'Peer:', host,
+            'Tags:', p.get('Tags'),
+            'Online:', p.get('Online'),
+            'RxBytes:', p.get('RxBytes'),
+            'TxBytes:', p.get('TxBytes'),
+            'LastSeen:', p.get('LastSeen'),
+            'Relay:', p.get('Relay'),
+            'CurAddr:', p.get('CurAddr'),
+        )
+" 2>&1 || true
+  echo ""
+  echo "--- relay logs (tail 100) ---"
+  kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=100 2>&1 || true
+  echo ""
+  echo "--- previous relay logs (if restarted) ---"
+  kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --previous --tail=50 2>&1 || true
+  echo ""
+  echo "Hints:"
+  echo "  - Confirm Tailscale ACL allows tag:ci-relay -> tag:ci-runner:587 and :993"
+  echo "  - Confirm runner auth key has tag:ci-runner and relay key has tag:ci-relay"
+  echo "  - Confirm runner hostname is ${UPSTREAM_HOST} (TS_HOSTNAME / start.sh)"
+  echo "  - Exit 137 on relay often means OOM; check memory limits"
+  echo "===================================================="
+}
+
+echo "Verifying testbench runner relay path for ${RELEASE} (upstream ${UPSTREAM_HOST})..."
+
+if ! kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  echo "Relay deployment not found: ${RELAY_DEPLOY} in ${NAMESPACE}"
+  dump_diagnostics
+  exit 1
+fi
+
+echo "--- web -> ${RELAY_SVC}:587 ---"
+if ! kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
+  sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 587"; then
+  echo "FAIL: web pod cannot reach relay Service :587"
+  dump_diagnostics
+  exit 1
+fi
+
+echo "--- web -> ${RELAY_SVC}:993 ---"
+if ! kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
+  sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 993"; then
+  echo "FAIL: web pod cannot reach relay Service :993"
+  dump_diagnostics
+  exit 1
+fi
+
+deadline=$((SECONDS + WAIT_SEC))
+attempt=0
+while true; do
+  attempt=$((attempt + 1))
+  echo "--- relay -> ${UPSTREAM_HOST}:587 (attempt ${attempt}, ${NC_TIMEOUT_SEC}s) ---"
+  if kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- \
+    sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${UPSTREAM_HOST} 587"; then
+    echo "--- relay -> ${UPSTREAM_HOST}:993 ---"
+    if kubectl exec -n "${NAMESPACE}" "${RELAY_DEPLOY}" -- \
+      sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${UPSTREAM_HOST} 993"; then
+      echo "OK: cluster relay path is reachable (web -> relay svc -> ${UPSTREAM_HOST}:587/993)."
+      exit 0
+    fi
+    echo "SMTP OK but IMAP :993 failed; will retry if time remains..."
+  else
+    echo "SMTP :587 not reachable yet..."
+  fi
+
+  if [ "${SECONDS}" -ge "${deadline}" ]; then
+    break
+  fi
+  sleep "${POLL_SEC}"
+done
+
+echo "FAIL: relay pod cannot reach ${UPSTREAM_HOST}:587/993 over Tailscale within ${WAIT_SEC}s"
+dump_diagnostics
+exit 1
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 2275112..ef648c2 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -419,6 +419,7 @@ jobs:
           USE_RUNNER_RELAY: "true"
           USE_TUNNEL: "false"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
+          VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
         run: |
           instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
           export INSTANCE="$instance"
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index e2d5b39..fe0cfaf 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -249,6 +249,7 @@ jobs:
           USE_RUNNER_RELAY: "true"
           USE_TUNNEL: "false"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
+          VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
         run: |
           instance="${{ needs.deploy.outputs.instance }}"
           export INSTANCE="$instance"

From eeabdc26079839308a0ebc815c74c2063d24b438 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 9 Oct 2026 05:50:53 -0700
Subject: [PATCH 06/20] Restart testbench relay after runner joins Tailscale.

HAProxy can keep a stale MagicDNS IP for tb-ci-* from a prior ephemeral
node; restarting the relay before verify forces a fresh upstream resolve.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/scripts/verify-testbench-relay.sh | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh
index 4988c28..dd2ae72 100755
--- a/.github/scripts/verify-testbench-relay.sh
+++ b/.github/scripts/verify-testbench-relay.sh
@@ -1,6 +1,10 @@
 #!/bin/bash
 # Fail-fast: prove PM → in-cluster relay → Tailscale → runner mail ports.
 # Run after the runner has joined the tailnet as tb-ci-<INSTANCE> (start.sh).
+#
+# Restarts the relay Deployment first so HAProxy/MagicDNS pick up the current
+# ephemeral runner IP (same hostname tb-ci-* can point at a dead peer from a
+# previous testbench run if the relay pod was left running).
 set -euo pipefail
 
 INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}"
@@ -11,9 +15,10 @@ WEB_DEPLOY="deploy/${RELEASE}-processmaker-web"
 UPSTREAM_HOST="tb-ci-${INSTANCE}"
 RELAY_SVC="${RELEASE}-testbench-relay"
 NC_TIMEOUT_SEC="${RELAY_NC_TIMEOUT_SEC:-5}"
-# Allow MagicDNS / DERP / HAProxy health to settle after runner joins.
+# Allow MagicDNS / DERP / HAProxy health to settle after runner joins / relay restart.
 WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-90}"
 POLL_SEC="${RELAY_VERIFY_POLL_SEC:-5}"
+ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-180}"
 
 if ! command -v kubectl >/dev/null 2>&1; then
   echo "kubectl is required to verify the testbench runner relay"
@@ -70,6 +75,7 @@ for p in (d.get('Peer') or {}).values():
   echo "  - Confirm Tailscale ACL allows tag:ci-relay -> tag:ci-runner:587 and :993"
   echo "  - Confirm runner auth key has tag:ci-runner and relay key has tag:ci-relay"
   echo "  - Confirm runner hostname is ${UPSTREAM_HOST} (TS_HOSTNAME / start.sh)"
+  echo "  - HAProxy 'no server available' / stale IP: relay must restart after runner joins"
   echo "  - Exit 137 on relay often means OOM; check memory limits"
   echo "===================================================="
 }
@@ -82,6 +88,14 @@ if ! kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" >/dev/null 2>&1; then
   exit 1
 fi
 
+echo "--- restart relay (refresh HAProxy upstream for ${UPSTREAM_HOST}) ---"
+kubectl rollout restart "${RELAY_DEPLOY}" -n "${NAMESPACE}"
+if ! kubectl rollout status "${RELAY_DEPLOY}" -n "${NAMESPACE}" --timeout="${ROLLOUT_TIMEOUT_SEC}s"; then
+  echo "FAIL: relay rollout after restart did not complete"
+  dump_diagnostics
+  exit 1
+fi
+
 echo "--- web -> ${RELAY_SVC}:587 ---"
 if ! kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
   sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 587"; then

From 7d55200485a2d5bc685ee43a56bcd5659d5c31b0 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 9 Oct 2026 06:09:26 -0700
Subject: [PATCH 07/20] Run Tailscale CI node cleanup before enabling the
 relay.

Use TAILSCALE_API_KEY_CI so stale tb-ci/relay-ci devices are removed before hostnames are claimed again.
---
 .github/scripts/verify-testbench-relay.sh    |  2 +-
 .github/workflows/deploy-pm4.yml             | 33 ++++++++++++++------
 .github/workflows/nightly-testbench-arch.yml | 33 ++++++++++++++------
 3 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh
index dd2ae72..355fec5 100755
--- a/.github/scripts/verify-testbench-relay.sh
+++ b/.github/scripts/verify-testbench-relay.sh
@@ -74,7 +74,7 @@ for p in (d.get('Peer') or {}).values():
   echo "Hints:"
   echo "  - Confirm Tailscale ACL allows tag:ci-relay -> tag:ci-runner:587 and :993"
   echo "  - Confirm runner auth key has tag:ci-runner and relay key has tag:ci-relay"
-  echo "  - Confirm runner hostname is ${UPSTREAM_HOST} (TS_HOSTNAME / start.sh)"
+  echo "  - Confirm runner hostname is ${UPSTREAM_HOST} (not ${UPSTREAM_HOST}-1); use TAILSCALE_API_KEY_CI cleanup"
   echo "  - HAProxy 'no server available' / stale IP: relay must restart after runner joins"
   echo "  - Exit 137 on relay often means OOM; check memory limits"
   echo "===================================================="
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index ef648c2..8fc41a5 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -389,6 +389,29 @@ jobs:
       - name: Authenticate with Amazon EKS
         run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng
 
+      - name: Checkout testbench
+        uses: actions/checkout@v4
+        with:
+          repository: ProcessMaker/testbench
+          # Pin to feature branch until relay-capable start.sh lands on main.
+          ref: task/FOUR-33523-B
+          path: testbench
+          token: ${{ secrets.GIT_TOKEN }}
+
+      # Must run before enable-testbench-relay so new relay/runner reclaim stable hostnames.
+      - name: Cleanup stale Tailscale CI nodes
+        working-directory: testbench
+        env:
+          TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
+        run: |
+          instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
+          if [ -z "${TAILSCALE_API_KEY_CI}" ]; then
+            echo "ERROR: org secret TAILSCALE_API_KEY_CI is required (Tailscale API access token tskey-api-…)"
+            exit 1
+          fi
+          npm ci
+          npx tsx src/index.ts cleanup-tailscale --instance "$instance" --targets all
+
       - name: Enable testbench runner relay
         working-directory: dot-github
         env:
@@ -401,15 +424,6 @@ jobs:
           chmod +x .github/scripts/enable-testbench-relay.sh
           .github/scripts/enable-testbench-relay.sh "$instance"
 
-      - name: Checkout testbench
-        uses: actions/checkout@v4
-        with:
-          repository: ProcessMaker/testbench
-          # Pin to feature branch until relay-capable start.sh lands on main.
-          ref: task/FOUR-33523-B
-          path: testbench
-          token: ${{ secrets.GIT_TOKEN }}
-
       - name: Run testbench
         working-directory: testbench
         env:
@@ -419,6 +433,7 @@ jobs:
           USE_RUNNER_RELAY: "true"
           USE_TUNNEL: "false"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
+          TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
           VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
         run: |
           instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index fe0cfaf..94dbe20 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -219,6 +219,29 @@ jobs:
       - name: Authenticate with Amazon EKS
         run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng
 
+      - name: Checkout testbench
+        uses: actions/checkout@v4
+        with:
+          repository: ProcessMaker/testbench
+          # Pin to feature branch until relay-capable start.sh lands on main.
+          ref: task/FOUR-33523-B
+          path: testbench
+          token: ${{ secrets.GIT_TOKEN }}
+
+      # Must run before enable-testbench-relay so new relay/runner reclaim stable hostnames.
+      - name: Cleanup stale Tailscale CI nodes
+        working-directory: testbench
+        env:
+          TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
+        run: |
+          instance="${{ needs.deploy.outputs.instance }}"
+          if [ -z "${TAILSCALE_API_KEY_CI}" ]; then
+            echo "ERROR: org secret TAILSCALE_API_KEY_CI is required (Tailscale API access token tskey-api-…)"
+            exit 1
+          fi
+          npm ci
+          npx tsx src/index.ts cleanup-tailscale --instance "$instance" --targets all
+
       - name: Enable testbench runner relay
         working-directory: dot-github
         env:
@@ -231,15 +254,6 @@ jobs:
           chmod +x .github/scripts/enable-testbench-relay.sh
           .github/scripts/enable-testbench-relay.sh "$instance"
 
-      - name: Checkout testbench
-        uses: actions/checkout@v4
-        with:
-          repository: ProcessMaker/testbench
-          # Pin to feature branch until relay-capable start.sh lands on main.
-          ref: task/FOUR-33523-B
-          path: testbench
-          token: ${{ secrets.GIT_TOKEN }}
-
       - name: Run testbench
         working-directory: testbench
         env:
@@ -249,6 +263,7 @@ jobs:
           USE_RUNNER_RELAY: "true"
           USE_TUNNEL: "false"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
+          TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
           VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
         run: |
           instance="${{ needs.deploy.outputs.instance }}"

From 2bc6ba164075365e53b0bb114baba18309aadc93 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 9 Oct 2026 06:17:18 -0700
Subject: [PATCH 08/20] Skip Helm upgrade hooks when enabling or disabling the
 testbench relay.

The instance is already ready; waiting on post-upgrade update-pm4 adds minutes for a relay-only values change.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/scripts/disable-testbench-relay.sh | 2 ++
 .github/scripts/enable-testbench-relay.sh  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/scripts/disable-testbench-relay.sh b/.github/scripts/disable-testbench-relay.sh
index a203c1c..10799f0 100755
--- a/.github/scripts/disable-testbench-relay.sh
+++ b/.github/scripts/disable-testbench-relay.sh
@@ -18,8 +18,10 @@ helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --passwo
 helm repo update
 
 echo "Disabling testbench runner relay for release ${RELEASE}..."
+# --no-hooks: relay-only change; skip post-upgrade update-pm4 (and other hooks).
 helm upgrade --timeout 10m "${RELEASE}" processmaker/enterprise \
   --reuse-values \
+  --no-hooks \
   --set testbenchRunnerRelay.enable=false \
   --set testbenchRunnerRelay.tailscale.authKey= \
   --version "${versionHelm}"
diff --git a/.github/scripts/enable-testbench-relay.sh b/.github/scripts/enable-testbench-relay.sh
index 85cbff8..544313e 100755
--- a/.github/scripts/enable-testbench-relay.sh
+++ b/.github/scripts/enable-testbench-relay.sh
@@ -50,8 +50,10 @@ else
 fi
 
 echo "Enabling testbench runner relay for release ${RELEASE} (upstream tb-ci-${INSTANCE})..."
+# --no-hooks: instance is already ready; skip post-upgrade update-pm4 (and other hooks).
 helm upgrade --timeout 15m "${RELEASE}" "${HELM_CHART_ARGS[@]}" \
   --reuse-values \
+  --no-hooks \
   -f "${RELAY_VALUES_RENDERED}" \
   --set testbenchRunnerRelay.enable=true \
   --set "testbenchRunnerRelay.upstreamHost=tb-ci-${INSTANCE}" \

From 6d4e3dd4566deebc933dd782ba4e944df14da6c6 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 9 Oct 2026 10:49:26 -0700
Subject: [PATCH 09/20] Remove tunnel, using tailscale now

---
 .github/scripts/ensure-testbench-relay.sh    | 72 ++++++++++++++++++++
 .github/scripts/verify-testbench-relay.sh    |  4 +-
 .github/workflows/deploy-pm4.yml             |  1 -
 .github/workflows/nightly-testbench-arch.yml |  1 -
 4 files changed, 74 insertions(+), 4 deletions(-)
 create mode 100755 .github/scripts/ensure-testbench-relay.sh

diff --git a/.github/scripts/ensure-testbench-relay.sh b/.github/scripts/ensure-testbench-relay.sh
new file mode 100755
index 0000000..dbf1826
--- /dev/null
+++ b/.github/scripts/ensure-testbench-relay.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+# Prepare an existing in-cluster testbench relay for a local/CI runner join (no Helm).
+# Run before the runner Tailscale node comes up (start.sh with ENSURE_TESTBENCH_RELAY_SCRIPT).
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}"
+RELEASE="ci-${INSTANCE}"
+NAMESPACE="${RELEASE}-ns-pm4"
+DEPLOYMENT="${RELEASE}-testbench-runner-relay"
+RELAY_DEPLOY="deploy/${DEPLOYMENT}"
+CONFIGMAP="${RELEASE}-testbench-runner-relay"
+SECRET="${RELEASE}-testbench-relay-tailscale"
+UPSTREAM_HOST="tb-ci-${INSTANCE}"
+ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-180}"
+RELAY_HAPROXY_CHECK_INTER="${RELAY_HAPROXY_CHECK_INTER:-60s}"
+
+if ! command -v kubectl >/dev/null 2>&1; then
+  echo "kubectl is required to ensure the testbench runner relay"
+  exit 1
+fi
+
+if ! kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  echo "Relay deployment not found: ${DEPLOYMENT} in namespace ${NAMESPACE}."
+  echo ""
+  echo "The testbench runner relay was never enabled for this CI instance."
+  echo "Run a CI testbench job once (enable-testbench-relay.sh), or use Helm with"
+  echo "enable-testbench-relay.sh when HELM credentials are available."
+  exit 1
+fi
+
+echo "Ensuring testbench runner relay for ${RELEASE} (upstream ${UPSTREAM_HOST})..."
+
+if kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  haproxy_cfg="$(kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" -o jsonpath='{.data.haproxy\.cfg}')"
+  patched_cfg="$(printf '%s' "${haproxy_cfg}" \
+    | sed -E "s/server runner [^:]+:/server runner ${UPSTREAM_HOST}:/g" \
+    | sed -E "s/check inter [0-9]+[smhd]+/check inter ${RELAY_HAPROXY_CHECK_INTER}/g")"
+  if [ "${patched_cfg}" = "${haproxy_cfg}" ]; then
+    echo "ConfigMap upstream (${UPSTREAM_HOST}) and health check interval (${RELAY_HAPROXY_CHECK_INTER}) already set."
+  else
+    echo "Patching ConfigMap ${CONFIGMAP} (upstream ${UPSTREAM_HOST}, check inter ${RELAY_HAPROXY_CHECK_INTER})..."
+    kubectl create configmap "${CONFIGMAP}" -n "${NAMESPACE}" \
+      --from-literal=haproxy.cfg="${patched_cfg}" \
+      --dry-run=client -o yaml \
+      | kubectl apply -f -
+  fi
+else
+  echo "WARNING: ConfigMap ${CONFIGMAP} not found; skipping HAProxy patch"
+fi
+
+if [ -n "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
+  if kubectl get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+    echo "Refreshing relay Tailscale auth key secret ${SECRET}..."
+    kubectl create secret generic "${SECRET}" -n "${NAMESPACE}" \
+      --from-literal=TS_AUTHKEY="${TAILSCALE_AUTH_KEY_CI_RELAY}" \
+      --dry-run=client -o yaml \
+      | kubectl apply -f -
+  else
+    echo "WARNING: Secret ${SECRET} not found; skipping auth key refresh"
+  fi
+fi
+
+echo "Restarting relay deployment (refresh HAProxy / MagicDNS for ${UPSTREAM_HOST})..."
+kubectl rollout restart "${RELAY_DEPLOY}" -n "${NAMESPACE}"
+if ! kubectl rollout status "${RELAY_DEPLOY}" -n "${NAMESPACE}" --timeout="${ROLLOUT_TIMEOUT_SEC}s"; then
+  echo "Relay rollout failed after restart."
+  kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true
+  kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=100 || true
+  exit 1
+fi
+
+echo "Testbench runner relay is ready for runner ${UPSTREAM_HOST}."
diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh
index 355fec5..7379925 100755
--- a/.github/scripts/verify-testbench-relay.sh
+++ b/.github/scripts/verify-testbench-relay.sh
@@ -16,8 +16,8 @@ UPSTREAM_HOST="tb-ci-${INSTANCE}"
 RELAY_SVC="${RELEASE}-testbench-relay"
 NC_TIMEOUT_SEC="${RELAY_NC_TIMEOUT_SEC:-5}"
 # Allow MagicDNS / DERP / HAProxy health to settle after runner joins / relay restart.
-WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-90}"
-POLL_SEC="${RELAY_VERIFY_POLL_SEC:-5}"
+WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-120}"
+POLL_SEC="${RELAY_VERIFY_POLL_SEC:-15}"
 ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-180}"
 
 if ! command -v kubectl >/dev/null 2>&1; then
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 8fc41a5..384051f 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -431,7 +431,6 @@ jobs:
           INSTANCE_PASSWORD: ${{ secrets.CI_INSTANCE_PASSWORD }}
           CONTEXT_PATH: ./context
           USE_RUNNER_RELAY: "true"
-          USE_TUNNEL: "false"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
           TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
           VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index 94dbe20..49923ef 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -261,7 +261,6 @@ jobs:
           INSTANCE_PASSWORD: ${{ secrets.CI_INSTANCE_PASSWORD }}
           CONTEXT_PATH: ./context
           USE_RUNNER_RELAY: "true"
-          USE_TUNNEL: "false"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
           TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
           VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh

From d4c76d5794872a6cceaeb39ce77cc6ffbf6a226a Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Sat, 10 Oct 2026 13:42:27 -0700
Subject: [PATCH 10/20] Update

---
 .github/scripts/ensure-testbench-relay.sh | 63 +++++++++++++++++++----
 .github/scripts/verify-testbench-relay.sh |  6 +--
 2 files changed, 56 insertions(+), 13 deletions(-)

diff --git a/.github/scripts/ensure-testbench-relay.sh b/.github/scripts/ensure-testbench-relay.sh
index dbf1826..6151e3e 100755
--- a/.github/scripts/ensure-testbench-relay.sh
+++ b/.github/scripts/ensure-testbench-relay.sh
@@ -11,7 +11,6 @@ RELAY_DEPLOY="deploy/${DEPLOYMENT}"
 CONFIGMAP="${RELEASE}-testbench-runner-relay"
 SECRET="${RELEASE}-testbench-relay-tailscale"
 UPSTREAM_HOST="tb-ci-${INSTANCE}"
-ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-180}"
 RELAY_HAPROXY_CHECK_INTER="${RELAY_HAPROXY_CHECK_INTER:-60s}"
 
 if ! command -v kubectl >/dev/null 2>&1; then
@@ -35,6 +34,12 @@ if kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" >/dev/null 2>&1; then
   patched_cfg="$(printf '%s' "${haproxy_cfg}" \
     | sed -E "s/server runner [^:]+:/server runner ${UPSTREAM_HOST}:/g" \
     | sed -E "s/check inter [0-9]+[smhd]+/check inter ${RELAY_HAPROXY_CHECK_INTER}/g")"
+  if ! printf '%s' "${patched_cfg}" | grep -q 'hard-stop-after'; then
+    patched_cfg="$(printf '%s' "${patched_cfg}" | awk '
+      /^[[:space:]]*maxconn / { print; print "      hard-stop-after 5s"; next }
+      { print }
+    ')"
+  fi
   if [ "${patched_cfg}" = "${haproxy_cfg}" ]; then
     echo "ConfigMap upstream (${UPSTREAM_HOST}) and health check interval (${RELAY_HAPROXY_CHECK_INTER}) already set."
   else
@@ -60,13 +65,51 @@ if [ -n "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
   fi
 fi
 
-echo "Restarting relay deployment (refresh HAProxy / MagicDNS for ${UPSTREAM_HOST})..."
-kubectl rollout restart "${RELAY_DEPLOY}" -n "${NAMESPACE}"
-if ! kubectl rollout status "${RELAY_DEPLOY}" -n "${NAMESPACE}" --timeout="${ROLLOUT_TIMEOUT_SEC}s"; then
-  echo "Relay rollout failed after restart."
-  kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true
-  kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=100 || true
-  exit 1
-fi
+echo "Applying fast relay rollout settings (Recreate strategy, exec probes, grace period)..."
+# Null tcpSocket so JSON merge patch does not leave a stale port probe that
+# proxies through HAProxy to the runner mailserver on every kubelet tick.
+kubectl patch "${RELAY_DEPLOY}" -n "${NAMESPACE}" --type=merge -p "$(cat <<'EOF'
+{
+  "spec": {
+    "strategy": {"type": "Recreate"},
+    "template": {
+      "spec": {
+        "terminationGracePeriodSeconds": 10,
+        "containers": [{
+          "name": "relay",
+          "readinessProbe": {
+            "tcpSocket": null,
+            "httpGet": null,
+            "exec": {
+              "command": [
+                "/bin/sh",
+                "-c",
+                "tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1"
+              ]
+            },
+            "initialDelaySeconds": 3,
+            "periodSeconds": 5,
+            "failureThreshold": 20
+          },
+          "livenessProbe": {
+            "tcpSocket": null,
+            "httpGet": null,
+            "exec": {
+              "command": [
+                "/bin/sh",
+                "-c",
+                "tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1"
+              ]
+            },
+            "initialDelaySeconds": 15,
+            "periodSeconds": 15
+          }
+        }]
+      }
+    }
+  }
+}
+EOF
+)" || echo "WARNING: could not patch relay deployment rollout settings"
 
-echo "Testbench runner relay is ready for runner ${UPSTREAM_HOST}."
+echo "Relay prepare complete (ConfigMap / deploy settings). verify-testbench-relay.sh restarts after the runner joins."
diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh
index 7379925..cab9da0 100755
--- a/.github/scripts/verify-testbench-relay.sh
+++ b/.github/scripts/verify-testbench-relay.sh
@@ -16,9 +16,9 @@ UPSTREAM_HOST="tb-ci-${INSTANCE}"
 RELAY_SVC="${RELEASE}-testbench-relay"
 NC_TIMEOUT_SEC="${RELAY_NC_TIMEOUT_SEC:-5}"
 # Allow MagicDNS / DERP / HAProxy health to settle after runner joins / relay restart.
-WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-120}"
-POLL_SEC="${RELAY_VERIFY_POLL_SEC:-15}"
-ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-180}"
+WAIT_SEC="${RELAY_VERIFY_WAIT_SEC:-90}"
+POLL_SEC="${RELAY_VERIFY_POLL_SEC:-10}"
+ROLLOUT_TIMEOUT_SEC="${RELAY_ROLLOUT_TIMEOUT_SEC:-90}"
 
 if ! command -v kubectl >/dev/null 2>&1; then
   echo "kubectl is required to verify the testbench runner relay"

From fec72fa6db0f39960aa3ad884cf11933dab6a2c4 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Sat, 10 Oct 2026 15:20:26 -0700
Subject: [PATCH 11/20] Update

---
 .github/workflows/deploy-pm4.yml             | 1 +
 .github/workflows/nightly-testbench-arch.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 384051f..914095f 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -433,6 +433,7 @@ jobs:
           USE_RUNNER_RELAY: "true"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
           TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
+          ENSURE_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/ensure-testbench-relay.sh
           VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
         run: |
           instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index 49923ef..6c8d1ea 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -263,6 +263,7 @@ jobs:
           USE_RUNNER_RELAY: "true"
           TAILSCALE_AUTH_KEY_CI_RUNNER: ${{ secrets.TAILSCALE_AUTH_KEY_CI_RUNNER }}
           TAILSCALE_API_KEY_CI: ${{ secrets.TAILSCALE_API_KEY_CI }}
+          ENSURE_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/ensure-testbench-relay.sh
           VERIFY_TESTBENCH_RELAY_SCRIPT: ${{ github.workspace }}/dot-github/.github/scripts/verify-testbench-relay.sh
         run: |
           instance="${{ needs.deploy.outputs.instance }}"

From bb3f0fe8b1967d810bd1c40a98279ca23630bbf6 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Sat, 10 Oct 2026 21:54:13 -0700
Subject: [PATCH 12/20] Update

---
 .github/scripts/ensure-testbench-relay.sh | 68 +++++++++--------------
 .github/scripts/verify-testbench-relay.sh | 39 +++++++++----
 2 files changed, 53 insertions(+), 54 deletions(-)

diff --git a/.github/scripts/ensure-testbench-relay.sh b/.github/scripts/ensure-testbench-relay.sh
index 6151e3e..806dedc 100755
--- a/.github/scripts/ensure-testbench-relay.sh
+++ b/.github/scripts/ensure-testbench-relay.sh
@@ -66,49 +66,31 @@ if [ -n "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
 fi
 
 echo "Applying fast relay rollout settings (Recreate strategy, exec probes, grace period)..."
-# Null tcpSocket so JSON merge patch does not leave a stale port probe that
-# proxies through HAProxy to the runner mailserver on every kubelet tick.
-kubectl patch "${RELAY_DEPLOY}" -n "${NAMESPACE}" --type=merge -p "$(cat <<'EOF'
-{
-  "spec": {
-    "strategy": {"type": "Recreate"},
-    "template": {
-      "spec": {
-        "terminationGracePeriodSeconds": 10,
-        "containers": [{
-          "name": "relay",
-          "readinessProbe": {
-            "tcpSocket": null,
-            "httpGet": null,
-            "exec": {
-              "command": [
-                "/bin/sh",
-                "-c",
-                "tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1"
-              ]
-            },
-            "initialDelaySeconds": 3,
-            "periodSeconds": 5,
-            "failureThreshold": 20
-          },
-          "livenessProbe": {
-            "tcpSocket": null,
-            "httpGet": null,
-            "exec": {
-              "command": [
-                "/bin/sh",
-                "-c",
-                "tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1"
-              ]
-            },
-            "initialDelaySeconds": 15,
-            "periodSeconds": 15
-          }
-        }]
-      }
-    }
-  }
-}
+# JSON patch (not --type=merge): merge patch replaces the entire containers[]
+# array and drops required fields like image.
+# Replace whole probes so a prior tcpSocket handler cannot remain beside exec.
+TS_PROBE='tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1'
+grace_op="add"
+if kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" \
+  -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}' 2>/dev/null | grep -q '[0-9]'; then
+  grace_op="replace"
+fi
+kubectl patch "${RELAY_DEPLOY}" -n "${NAMESPACE}" --type=json -p "$(cat <<EOF
+[
+  {"op": "replace", "path": "/spec/strategy", "value": {"type": "Recreate"}},
+  {"op": "${grace_op}", "path": "/spec/template/spec/terminationGracePeriodSeconds", "value": 10},
+  {"op": "replace", "path": "/spec/template/spec/containers/0/readinessProbe", "value": {
+    "exec": {"command": ["/bin/sh", "-c", "${TS_PROBE}"]},
+    "initialDelaySeconds": 3,
+    "periodSeconds": 5,
+    "failureThreshold": 20
+  }},
+  {"op": "replace", "path": "/spec/template/spec/containers/0/livenessProbe", "value": {
+    "exec": {"command": ["/bin/sh", "-c", "${TS_PROBE}"]},
+    "initialDelaySeconds": 15,
+    "periodSeconds": 15
+  }}
+]
 EOF
 )" || echo "WARNING: could not patch relay deployment rollout settings"
 
diff --git a/.github/scripts/verify-testbench-relay.sh b/.github/scripts/verify-testbench-relay.sh
index cab9da0..ca9266d 100755
--- a/.github/scripts/verify-testbench-relay.sh
+++ b/.github/scripts/verify-testbench-relay.sh
@@ -96,18 +96,35 @@ if ! kubectl rollout status "${RELAY_DEPLOY}" -n "${NAMESPACE}" --timeout="${ROL
   exit 1
 fi
 
-echo "--- web -> ${RELAY_SVC}:587 ---"
-if ! kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
-  sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 587"; then
-  echo "FAIL: web pod cannot reach relay Service :587"
-  dump_diagnostics
-  exit 1
-fi
+# HAProxy may still be binding after the pod becomes Ready (esp. while Tailscale
+# finishes login). Retry cluster-local probes with the same settle window.
+deadline=$((SECONDS + WAIT_SEC))
+attempt=0
+web_ok=false
+while true; do
+  attempt=$((attempt + 1))
+  echo "--- web -> ${RELAY_SVC}:587 (attempt ${attempt}, ${NC_TIMEOUT_SEC}s) ---"
+  if kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
+    sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 587"; then
+    echo "--- web -> ${RELAY_SVC}:993 ---"
+    if kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
+      sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 993"; then
+      web_ok=true
+      break
+    fi
+    echo "SMTP OK but IMAP :993 failed; will retry if time remains..."
+  else
+    echo "web -> relay :587 not reachable yet..."
+  fi
+
+  if [ "${SECONDS}" -ge "${deadline}" ]; then
+    break
+  fi
+  sleep "${POLL_SEC}"
+done
 
-echo "--- web -> ${RELAY_SVC}:993 ---"
-if ! kubectl exec -n "${NAMESPACE}" "${WEB_DEPLOY}" -c pm4-app-ui -- \
-  sh -c "timeout ${NC_TIMEOUT_SEC} nc -zv ${RELAY_SVC} 993"; then
-  echo "FAIL: web pod cannot reach relay Service :993"
+if [ "${web_ok}" != "true" ]; then
+  echo "FAIL: web pod cannot reach relay Service :587/:993 within ${WAIT_SEC}s"
   dump_diagnostics
   exit 1
 fi

From 90c8c21066faacb29697d5e400c26a65ed094808 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 16 Oct 2026 10:46:57 -0700
Subject: [PATCH 13/20] Add ability to skip phpunit

---
 .github/workflows/deploy-pm4.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 914095f..647a382 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -611,7 +611,8 @@ jobs:
       !cancelled() &&
       needs.imageEKS.result == 'success' &&
       github.event.action != 'closed' &&
-      inputs.delete == ''
+      inputs.delete == '' &&
+      !contains(github.event.pull_request.body, 'ci:skip-phpunit')
     needs: imageEKS
     runs-on: ${{ vars.RUNNER }}
     timeout-minutes: 150

From 623022fe7c595a968ac123bf004e93b2b6ad98da Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 16 Oct 2026 11:40:49 -0700
Subject: [PATCH 14/20] Fix runner

---
 .github/workflows/deploy-pm4.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 647a382..bc2198b 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -452,7 +452,7 @@ jobs:
   deleteEKS:
     name: Delete Instance
     if: github.event.action == 'closed' || inputs.delete != '' || contains(github.event.pull_request.body, 'ci:redeploy')
-    runs-on: self-hosted
+    runs-on: [self-hosted, pm-delete-runner]
     # GITHUB_TOKEN needs write to strip ci:redeploy from the PR body.
     # Using github.token (not GIT_TOKEN) so the edit does not trigger a new workflow run.
     permissions:

From 804be2f6a54e6bff490cc354e31febf736c372c4 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 16 Oct 2026 12:31:05 -0700
Subject: [PATCH 15/20] Remove dbs if previous attempt failed

---
 .github/scripts/delete-instance.sh | 56 +++++++++++++----
 .github/workflows/deploy-pm4.yml   | 99 +++++++-----------------------
 2 files changed, 64 insertions(+), 91 deletions(-)

diff --git a/.github/scripts/delete-instance.sh b/.github/scripts/delete-instance.sh
index 4773852..67a1f1d 100755
--- a/.github/scripts/delete-instance.sh
+++ b/.github/scripts/delete-instance.sh
@@ -29,19 +29,48 @@ fi
 
 set +e
 
-if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
-  echo "Deleting Instance :: ${RELEASE}"
-  helm delete "${RELEASE}" || true
-  kubectl delete namespace "${NAMESPACE}" || true
+delete_k8s() {
+  if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
+    echo "Deleting Instance :: ${RELEASE}"
+    helm delete "${RELEASE}" || true
+    kubectl delete namespace "${NAMESPACE}" || true
+    echo "The instance [https://ci-${INSTANCE}.engk8s.processmaker.net] was removed from the cluster."
+    return 0
+  fi
+
+  echo "Namespace ${NAMESPACE} not found; skipping cluster delete."
+  return 0
+}
+
+delete_rds() {
+  echo "Cleaning RDS for instance ${INSTANCE} (always runs, even when the namespace is already gone)."
 
   set -o pipefail
-  mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -N -e "SHOW DATABASES LIKE '${deploy_db}'" \
-    | xargs -r -I{} mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`{}\`;"
-  if [ $? -ne 0 ]; then
-    echo "WARN: Failed dropping databases matching ${deploy_db}"
+  matching_dbs=$(
+    mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -N -e "SHOW DATABASES LIKE '${deploy_db}'"
+  )
+  list_status=$?
+  set +o pipefail
+
+  if [ "${list_status}" -ne 0 ]; then
+    echo "WARN: Failed listing databases matching ${deploy_db}"
     FAILED=1
+    return 1
+  fi
+
+  if [ -z "${matching_dbs}" ]; then
+    echo "No databases matching ${deploy_db} on RDS (already clean)."
+  else
+    echo "Dropping databases matching ${deploy_db}:"
+    echo "${matching_dbs}"
+    set -o pipefail
+    echo "${matching_dbs}" | xargs -r -I{} mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`{}\`;"
+    if [ $? -ne 0 ]; then
+      echo "WARN: Failed dropping databases matching ${deploy_db}"
+      FAILED=1
+    fi
+    set +o pipefail
   fi
-  set +o pipefail
 
   mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`${deploy_ai}\`;"
   if [ $? -ne 0 ]; then
@@ -61,10 +90,11 @@ if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
     FAILED=1
   fi
 
-  echo "The instance [https://ci-${INSTANCE}.engk8s.processmaker.net] was deleted."
-else
-  echo "Namespace ${NAMESPACE} not found; nothing to delete on the cluster."
-fi
+  return 0
+}
+
+delete_k8s
+delete_rds
 
 if [ "${DELETE_HARBOR}" = "true" ]; then
   if [ -z "${IMAGE_TAG:-}" ]; then
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index bc2198b..4ac4076 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -488,96 +488,39 @@ jobs:
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
 
+      - name: Checkout .github repo
+        uses: actions/checkout@v4
+        with:
+          repository: processmaker/.github
+          ref: main
+
       - name: Delete instance EKS
-        run: |           
+        env:
+          USER_MYSQL_ENG: ${{ secrets.USER_MYSQL_ENG }}
+          PASS_MYSQL_ENG: ${{ secrets.PASS_MYSQL_ENG }}
+          RDS_ENG: ${{ secrets.RDS_ENG }}
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+          REGISTRY_HOST: ${{ secrets.REGISTRY_HOST }}
+        run: |
           set +e
-          FAILED=0
-
-          # If inputs.delete does not equal '', set the IMAGE_TAG to the value of inputs.delete
           if [ "${{ inputs.delete }}" != "" ]; then
             IMAGE_TAG=${{ inputs.delete }}
           else
             IMAGE_TAG=${{ env.IMAGE_TAG }}
           fi
-          INSTANCE=$(echo -n $IMAGE_TAG | md5sum | head -c 10)
+          export INSTANCE=$(echo -n "$IMAGE_TAG" | md5sum | head -c 10)
+          export IMAGE_TAG
 
-          # Delete Harbor image only on PR close / manual delete — not on ci:redeploy
-          DELETE_HARBOR=false
           if [ "${{ github.event.action }}" = "closed" ] || [ "${{ inputs.delete }}" != "" ]; then
-            DELETE_HARBOR=true
-          fi
-
-          if kubectl get namespace/ci-$INSTANCE-ns-pm4 ; then
-            echo "Deleting Instance :: ci-$INSTANCE"
-            helm delete ci-$INSTANCE || true
-            kubectl delete namespace ci-$INSTANCE-ns-pm4 || true
-       
-            #Drop database
-            deploy_db="pm4_ci-${INSTANCE}%"
-            deploy_ai="\`pm4_ci-$INSTANCE_ai\`"
-
-            # check that that string length of $deploy_db is 12 or more as a safety check. If its less than 12, exit now
-            if [ ${#deploy_db} -lt 12 ]; then
-              echo "ERROR: deploy_db safety check failed (length ${#deploy_db})"
-              exit 1
-            fi
-
-            # Drop the main database including any tenant databases.
-            # Continue on failure so a half-built instance still gets cleaned up.
-            set -o pipefail
-            mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -h ${{ secrets.RDS_ENG }} -N -e "SHOW DATABASES LIKE '${deploy_db}'" | xargs -r -I{} mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -h ${{ secrets.RDS_ENG }} -e "DROP DATABASE IF EXISTS \`{}\`;"
-            if [ $? -ne 0 ]; then
-              echo "WARN: Failed dropping databases matching ${deploy_db}"
-              FAILED=1
-            fi
-            set +o pipefail
-
-            mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -e "DROP DATABASE IF EXISTS $deploy_ai" -h ${{ secrets.RDS_ENG }}
-            if [ $? -ne 0 ]; then
-              echo "WARN: Failed dropping AI database $deploy_ai"
-              FAILED=1
-            fi
-
-            mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -e "DROP USER IF EXISTS 'user_ci-$INSTANCE'@'%'" -h ${{ secrets.RDS_ENG }}
-            if [ $? -ne 0 ]; then
-              echo "WARN: Failed dropping user user_ci-$INSTANCE"
-              FAILED=1
-            fi
-
-            mysql -u${{ secrets.USER_MYSQL_ENG }} -p${{ secrets.PASS_MYSQL_ENG }} -e "DROP USER IF EXISTS 'user_ci-$INSTANCE_ai'@'%'" -h ${{ secrets.RDS_ENG }}
-            if [ $? -ne 0 ]; then
-              echo "WARN: Failed dropping user user_ci-$INSTANCE_ai"
-              FAILED=1
-            fi
-
-            echo "The instance [https://ci-$INSTANCE.engk8s.processmaker.net] was deleted!!"
+            export DELETE_HARBOR=true
           else
-            echo "The pull request does not have an instance on K8s [https://ci-$INSTANCE.engk8s.processmaker.net] not found!!"
+            export DELETE_HARBOR=false
           fi
 
-          if [ "$DELETE_HARBOR" = "true" ]; then
-            echo "Deleting image from Harbor: ${IMAGE_TAG}"
-            HTTP_CODE=$(curl -s -o /tmp/harbor_delete_out -w "%{http_code}" -X DELETE \
-              -u ${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }} \
-              "https://${{ secrets.REGISTRY_HOST }}/api/v2.0/projects/processmaker/repositories/enterprise/artifacts/${IMAGE_TAG}")
-            if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
-              echo "Harbor image deleted (HTTP $HTTP_CODE)"
-            elif [ "$HTTP_CODE" = "404" ]; then
-              echo "Harbor image not found (already deleted)"
-            else
-              echo "WARN: Failed deleting Harbor image ${IMAGE_TAG} (HTTP $HTTP_CODE)"
-              cat /tmp/harbor_delete_out || true
-              FAILED=1
-            fi
-          else
-            echo "Skipping Harbor image delete (ci:redeploy)"
-          fi
-
-          if [ "$FAILED" -ne 0 ]; then
-            echo "One or more delete operations failed"
-            exit 1
-          fi
-          echo "Delete completed successfully"
+          echo "INSTANCE=${INSTANCE}"
+          chmod +x .github/scripts/delete-instance.sh
+          bash .github/scripts/delete-instance.sh
 
       # Strip ci:redeploy from the PR body so it does not re-trigger on later edits.
       # Uses github.token (default) so this edit does not start a new workflow run.

From 9736a7a19f0d0a54efea646e4e902d920f8dcc73 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Thu, 22 Oct 2026 16:27:10 -0700
Subject: [PATCH 16/20] Save certs when redeploying to avoid lets encrypt
 limits

---
 .github/scripts/delete-instance.sh           | 13 ++++
 .github/scripts/deploy-instance.sh           | 12 ++++
 .github/scripts/export-tls-secret.sh         | 76 ++++++++++++++++++++
 .github/scripts/restore-tls-secret.sh        | 56 +++++++++++++++
 .github/workflows/deploy-pm4.yml             | 47 ++++++++++--
 .github/workflows/nightly-testbench-arch.yml | 28 +++++++-
 6 files changed, 226 insertions(+), 6 deletions(-)
 create mode 100755 .github/scripts/export-tls-secret.sh
 create mode 100755 .github/scripts/restore-tls-secret.sh

diff --git a/.github/scripts/delete-instance.sh b/.github/scripts/delete-instance.sh
index 67a1f1d..9b2885f 100755
--- a/.github/scripts/delete-instance.sh
+++ b/.github/scripts/delete-instance.sh
@@ -7,6 +7,8 @@
 #   DELETE_HARBOR     true to delete the Harbor artifact (default false)
 #   IMAGE_TAG         Harbor tag to delete when DELETE_HARBOR=true
 #   REGISTRY_USERNAME, REGISTRY_PASSWORD, REGISTRY_HOST
+#   PRESERVE_TLS_DIR  if set, export TLS secret here before wiping the namespace
+#                     (for GHA artifact restore on redeploy)
 set -u
 
 FAILED=0
@@ -21,6 +23,7 @@ NAMESPACE="ci-${INSTANCE}-ns-pm4"
 RELEASE="ci-${INSTANCE}"
 deploy_db="pm4_ci-${INSTANCE}%"
 deploy_ai="pm4_ci-${INSTANCE}_ai"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 if [ ${#deploy_db} -lt 12 ]; then
   echo "ERROR: deploy_db safety check failed (length ${#deploy_db})"
@@ -29,6 +32,15 @@ fi
 
 set +e
 
+preserve_tls() {
+  if [ -z "${PRESERVE_TLS_DIR:-}" ]; then
+    return 0
+  fi
+  echo "Preserving TLS secret before delete (PRESERVE_TLS_DIR=${PRESERVE_TLS_DIR})"
+  # shellcheck disable=SC1091
+  bash "${SCRIPT_DIR}/export-tls-secret.sh"
+}
+
 delete_k8s() {
   if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
     echo "Deleting Instance :: ${RELEASE}"
@@ -93,6 +105,7 @@ delete_rds() {
   return 0
 }
 
+preserve_tls
 delete_k8s
 delete_rds
 
diff --git a/.github/scripts/deploy-instance.sh b/.github/scripts/deploy-instance.sh
index 9d72a27..c7024da 100644
--- a/.github/scripts/deploy-instance.sh
+++ b/.github/scripts/deploy-instance.sh
@@ -18,9 +18,21 @@ helm_common_sets() {
     --set appVersion=${APP_VERSION}
 }
 
+# Optional: TLS_SECRET_DIR with a stashed LE cert from a prior delete (GHA artifact).
+# Restore before helm install so cert-manager does not burn a new Let's Encrypt issuance.
+restore_stashed_tls() {
+    if [ -z "${TLS_SECRET_DIR:-}" ]; then
+        return 0
+    fi
+    export INSTANCE="{{INSTANCE}}"
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    bash "${SCRIPT_DIR}/restore-tls-secret.sh"
+}
+
 if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then
     echo "New instance. Creating Namespace"
     kubectl create namespace ci-{{INSTANCE}}-ns-pm4
+    restore_stashed_tls
     echo "Creating DB"
     # Use admin password from secrets
     echo "Update instance yamls"
diff --git a/.github/scripts/export-tls-secret.sh b/.github/scripts/export-tls-secret.sh
new file mode 100755
index 0000000..5ba1be9
--- /dev/null
+++ b/.github/scripts/export-tls-secret.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Export a CI instance TLS secret to a directory for GHA artifact upload.
+# Required env:
+#   INSTANCE       instance id (e.g. 2a0d55eb59 or nt-fpm-st)
+#   PRESERVE_TLS_DIR  directory to write tls.crt / tls.key [/ ca.crt]
+# Exits 0 always. Creates PRESERVE_TLS_DIR/stashed when a usable cert was exported.
+set -u
+
+INSTANCE="${INSTANCE:-}"
+PRESERVE_TLS_DIR="${PRESERVE_TLS_DIR:-}"
+NAMESPACE="ci-${INSTANCE}-ns-pm4"
+SECRET="ci-${INSTANCE}-tls-secret"
+
+if [ -z "${INSTANCE}" ] || [ -z "${PRESERVE_TLS_DIR}" ]; then
+  echo "WARN: INSTANCE and PRESERVE_TLS_DIR are required; skipping TLS export"
+  exit 0
+fi
+
+rm -rf "${PRESERVE_TLS_DIR}"
+mkdir -p "${PRESERVE_TLS_DIR}"
+
+if ! kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
+  echo "Namespace ${NAMESPACE} not found; nothing to stash"
+  exit 0
+fi
+
+if ! kubectl get "secret/${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  echo "Secret ${SECRET} not found in ${NAMESPACE}; nothing to stash"
+  exit 0
+fi
+
+if ! kubectl get "secret/${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.crt}' 2>/dev/null | base64 -d >"${PRESERVE_TLS_DIR}/tls.crt"; then
+  echo "WARN: Failed reading tls.crt from ${SECRET}"
+  rm -rf "${PRESERVE_TLS_DIR}"
+  mkdir -p "${PRESERVE_TLS_DIR}"
+  exit 0
+fi
+
+if ! kubectl get "secret/${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.tls\.key}' 2>/dev/null | base64 -d >"${PRESERVE_TLS_DIR}/tls.key"; then
+  echo "WARN: Failed reading tls.key from ${SECRET}"
+  rm -rf "${PRESERVE_TLS_DIR}"
+  mkdir -p "${PRESERVE_TLS_DIR}"
+  exit 0
+fi
+
+# Optional CA bundle (present for some issuers; harmless if missing).
+kubectl get "secret/${SECRET}" -n "${NAMESPACE}" -o jsonpath='{.data.ca\.crt}' 2>/dev/null | base64 -d >"${PRESERVE_TLS_DIR}/ca.crt" || true
+if [ ! -s "${PRESERVE_TLS_DIR}/ca.crt" ]; then
+  rm -f "${PRESERVE_TLS_DIR}/ca.crt"
+fi
+
+if [ ! -s "${PRESERVE_TLS_DIR}/tls.crt" ] || [ ! -s "${PRESERVE_TLS_DIR}/tls.key" ]; then
+  echo "WARN: Exported tls.crt/tls.key empty; not stashing"
+  rm -rf "${PRESERVE_TLS_DIR}"
+  mkdir -p "${PRESERVE_TLS_DIR}"
+  exit 0
+fi
+
+if command -v openssl >/dev/null 2>&1; then
+  if ! openssl x509 -in "${PRESERVE_TLS_DIR}/tls.crt" -noout -checkend 86400 >/dev/null 2>&1; then
+    echo "WARN: Certificate expired or expires within 24h; not stashing"
+    rm -rf "${PRESERVE_TLS_DIR}"
+    mkdir -p "${PRESERVE_TLS_DIR}"
+    exit 0
+  fi
+  echo "Stashing TLS cert:"
+  openssl x509 -in "${PRESERVE_TLS_DIR}/tls.crt" -noout -subject -issuer -dates 2>/dev/null || true
+else
+  echo "openssl not available; skipping expiry check"
+fi
+
+chmod 600 "${PRESERVE_TLS_DIR}/tls.key"
+# Non-dot marker so actions/upload-artifact reliably includes it.
+touch "${PRESERVE_TLS_DIR}/stashed"
+echo "TLS secret ${SECRET} exported to ${PRESERVE_TLS_DIR}"
+exit 0
diff --git a/.github/scripts/restore-tls-secret.sh b/.github/scripts/restore-tls-secret.sh
new file mode 100755
index 0000000..d72f7ab
--- /dev/null
+++ b/.github/scripts/restore-tls-secret.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Restore a stashed CI TLS secret into the instance namespace before helm install
+# so cert-manager does not request a new Let's Encrypt certificate.
+# Required env:
+#   INSTANCE         instance id
+#   TLS_SECRET_DIR   directory with tls.crt / tls.key from export-tls-secret.sh
+# Creates the namespace if missing. Exits 0 if there is nothing to restore.
+set -u
+
+INSTANCE="${INSTANCE:-}"
+TLS_SECRET_DIR="${TLS_SECRET_DIR:-}"
+NAMESPACE="ci-${INSTANCE}-ns-pm4"
+SECRET="ci-${INSTANCE}-tls-secret"
+
+if [ -z "${INSTANCE}" ] || [ -z "${TLS_SECRET_DIR}" ]; then
+  echo "TLS restore skipped (INSTANCE or TLS_SECRET_DIR unset)"
+  exit 0
+fi
+
+if [ ! -s "${TLS_SECRET_DIR}/tls.crt" ] || [ ! -s "${TLS_SECRET_DIR}/tls.key" ]; then
+  echo "No stashed TLS secret at ${TLS_SECRET_DIR}; cert-manager will issue if needed"
+  exit 0
+fi
+
+if command -v openssl >/dev/null 2>&1; then
+  if ! openssl x509 -in "${TLS_SECRET_DIR}/tls.crt" -noout -checkend 86400 >/dev/null 2>&1; then
+    echo "WARN: Stashed certificate expired or expires within 24h; not restoring"
+    exit 0
+  fi
+fi
+
+if ! kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
+  echo "Creating namespace ${NAMESPACE} for TLS restore"
+  kubectl create namespace "${NAMESPACE}"
+fi
+
+echo "Restoring TLS secret ${SECRET} into ${NAMESPACE}"
+kubectl create secret tls "${SECRET}" \
+  --cert="${TLS_SECRET_DIR}/tls.crt" \
+  --key="${TLS_SECRET_DIR}/tls.key" \
+  -n "${NAMESPACE}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+if [ -s "${TLS_SECRET_DIR}/ca.crt" ]; then
+  # Attach ca.crt when present (create secret tls does not include it).
+  CA_B64=$(base64 <"${TLS_SECRET_DIR}/ca.crt" | tr -d '\n')
+  kubectl patch secret "${SECRET}" -n "${NAMESPACE}" --type merge -p "{\"data\":{\"ca.crt\":\"${CA_B64}\"}}"
+fi
+
+# Help cert-manager adopt the existing secret for the ingress-managed Certificate.
+kubectl annotate secret "${SECRET}" -n "${NAMESPACE}" \
+  "cert-manager.io/certificate-name=${SECRET}" \
+  --overwrite >/dev/null
+
+echo "TLS secret restored; cert-manager should adopt instead of re-issuing"
+exit 0
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 4ac4076..a95ee69 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -207,7 +207,22 @@ jobs:
 
       - name: Authenticate with Amazon EKS
         run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng
-      
+
+      - name: Resolve instance id
+        id: instance
+        run: |
+          instance=$(echo -n "${{ env.IMAGE_TAG }}" | md5sum | head -c 10)
+          echo "instance=${instance}" >> "$GITHUB_OUTPUT"
+          echo "INSTANCE: ${instance}"
+
+      # Restored before helm install so cert-manager does not re-hit Let's Encrypt.
+      # Missing artifact is normal when deleteEKS was skipped or had no usable cert.
+      - name: Download stashed TLS secret
+        continue-on-error: true
+        uses: actions/download-artifact@v4
+        with:
+          name: ci-tls-${{ steps.instance.outputs.instance }}
+          path: ${{ runner.temp }}/ci-tls
       
       - name: Deploy instance EKS
         env:
@@ -235,8 +250,9 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           OCTANE: ${{ env.OCTANE }}
+          TLS_SECRET_DIR: ${{ runner.temp }}/ci-tls
         run: |
-          instance=$(echo -n ${{env.IMAGE_TAG}} | md5sum | head -c 10)
+          instance="${{ steps.instance.outputs.instance }}"
           echo "INSTANCE: $instance"
           echo "IMAGE_TAG: $IMAGE_TAG"
           
@@ -260,7 +276,7 @@ jobs:
           cat .github/templates/instance.yaml
           echo "=== Checking db.yaml after replacements ==="
           cat .github/templates/db.yaml
-          chmod +x .github/scripts/deploy-instance.sh
+          chmod +x .github/scripts/deploy-instance.sh .github/scripts/restore-tls-secret.sh
           bash .github/scripts/deploy-instance.sh
           fi
           if [ "$MULTITENANCY" = "true" ]; then
@@ -495,6 +511,7 @@ jobs:
           ref: main
 
       - name: Delete instance EKS
+        id: delete_instance
         env:
           USER_MYSQL_ENG: ${{ secrets.USER_MYSQL_ENG }}
           PASS_MYSQL_ENG: ${{ secrets.PASS_MYSQL_ENG }}
@@ -511,6 +528,7 @@ jobs:
           fi
           export INSTANCE=$(echo -n "$IMAGE_TAG" | md5sum | head -c 10)
           export IMAGE_TAG
+          echo "instance=${INSTANCE}" >> "$GITHUB_OUTPUT"
 
           if [ "${{ github.event.action }}" = "closed" ] || [ "${{ inputs.delete }}" != "" ]; then
             export DELETE_HARBOR=true
@@ -518,10 +536,31 @@ jobs:
             export DELETE_HARBOR=false
           fi
 
+          # On ci:redeploy, stash the LE TLS secret as a GHA artifact so deploy can
+          # restore it and avoid burning another Let's Encrypt issuance.
+          if [ "${{ github.event.action }}" != "closed" ] && [ "${{ inputs.delete }}" = "" ] && [[ "${{ github.event.pull_request.body }}" == *"ci:redeploy"* ]]; then
+            export PRESERVE_TLS_DIR="${RUNNER_TEMP}/ci-tls"
+          fi
+
           echo "INSTANCE=${INSTANCE}"
-          chmod +x .github/scripts/delete-instance.sh
+          chmod +x .github/scripts/delete-instance.sh .github/scripts/export-tls-secret.sh
           bash .github/scripts/delete-instance.sh
 
+          if [ -n "${PRESERVE_TLS_DIR:-}" ] && [ -f "${PRESERVE_TLS_DIR}/stashed" ]; then
+            echo "tls_stashed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "tls_stashed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Upload stashed TLS secret
+        if: steps.delete_instance.outputs.tls_stashed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-tls-${{ steps.delete_instance.outputs.instance }}
+          path: ${{ runner.temp }}/ci-tls/
+          retention-days: 1
+          if-no-files-found: warn
+
       # Strip ci:redeploy from the PR body so it does not re-trigger on later edits.
       # Uses github.token (default) so this edit does not start a new workflow run.
       # actions/github-script needs no gh binary on the runner.
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index 6c8d1ea..fae411d 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -71,16 +71,32 @@ jobs:
           kubectl cluster-info --request-timeout=15s
 
       - name: Delete instance EKS
+        id: delete_instance
         env:
           INSTANCE: ${{ env.INSTANCE }}
           USER_MYSQL_ENG: ${{ secrets.USER_MYSQL_ENG }}
           PASS_MYSQL_ENG: ${{ secrets.PASS_MYSQL_ENG }}
           RDS_ENG: ${{ secrets.RDS_ENG }}
           DELETE_HARBOR: "false"
+          PRESERVE_TLS_DIR: ${{ runner.temp }}/ci-tls
         run: |
           echo "INSTANCE: $INSTANCE"
-          chmod +x .github/scripts/delete-instance.sh
+          chmod +x .github/scripts/delete-instance.sh .github/scripts/export-tls-secret.sh
           bash .github/scripts/delete-instance.sh
+          if [ -f "${PRESERVE_TLS_DIR}/stashed" ]; then
+            echo "tls_stashed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "tls_stashed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Upload stashed TLS secret
+        if: steps.delete_instance.outputs.tls_stashed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-tls-${{ env.INSTANCE }}
+          path: ${{ runner.temp }}/ci-tls/
+          retention-days: 1
+          if-no-files-found: warn
 
   deploy:
     name: deploy-${{ inputs.arch_id }}
@@ -132,6 +148,13 @@ jobs:
       - name: Authenticate with Amazon EKS
         run: aws eks update-kubeconfig --region us-east-1 --name pm4-eng
 
+      - name: Download stashed TLS secret
+        continue-on-error: true
+        uses: actions/download-artifact@v4
+        with:
+          name: ci-tls-${{ env.INSTANCE }}
+          path: ${{ runner.temp }}/ci-tls
+
       - name: Deploy instance EKS
         env:
           IMAGE_TAG: ${{ inputs.image_tag }}
@@ -160,6 +183,7 @@ jobs:
           MULTITENANCY: ${{ env.MULTITENANCY }}
           CI_PROJECT: processmaker
           CI_PACKAGE_BRANCH: develop
+          TLS_SECRET_DIR: ${{ runner.temp }}/ci-tls
         run: |
           instance="${{ steps.instance.outputs.instance }}"
           echo "INSTANCE: $instance"
@@ -181,7 +205,7 @@ jobs:
           sed -i "s#{{PMAI_HOST_WS}}#$PMAI_HOST_WS#g" .github/templates/instance.yaml
           sed -i "s#{{MYSQL_USERNAME}}#$RDS_ADMIN_USERNAME#g" .github/templates/db.yaml
           sed -i "s#{{MYSQL_PASSWORD}}#$RDS_ADMIN_PASSWORD#g" .github/templates/db.yaml
-          chmod +x .github/scripts/deploy-instance.sh
+          chmod +x .github/scripts/deploy-instance.sh .github/scripts/restore-tls-secret.sh
           bash .github/scripts/deploy-instance.sh
 
   testbench:

From f1d76fe81b97bb1b6c20304c287b4be14559de47 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 23 Oct 2026 10:34:43 -0700
Subject: [PATCH 17/20] Use correct companion branch

---
 .github/workflows/deploy-pm4.yml             | 45 ++++++++++++++------
 .github/workflows/js-package.yml             |  5 ++-
 .github/workflows/nightly-testbench-arch.yml | 22 +++++++---
 3 files changed, 52 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index a95ee69..4aa84eb 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -78,7 +78,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
 
       - name: Common 
         uses: ./.github/actions/common
@@ -177,8 +179,9 @@ jobs:
       - name: Checkout .github repo
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
-          ref: main
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
 
       - name: Common 
         uses: ./.github/actions/common
@@ -299,7 +302,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
 
       - name: Common 
         uses: ./.github/actions/common
@@ -380,13 +385,25 @@ jobs:
       contains(github.event.pull_request.body, 'ci:run-testbench')
     runs-on: ${{ vars.RUNNER }}
     steps:
+      - name: Resolve companion branch
+        id: companion
+        shell: bash
+        run: |
+          # Keep testbench on the same branch name as this reusable workflow
+          # (e.g. task/FOUR-33523-B). Falls back to main for tags/SHAs.
+          wr='${{ github.workflow_ref }}'
+          if [[ "$wr" == *@refs/heads/* ]]; then
+            echo "ref=${wr##*@refs/heads/}" >> "$GITHUB_OUTPUT"
+          else
+            echo "ref=main" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Checkout dot-github
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
-          # Pin to feature branch until relay scripts land on main.
-          # uses: @sha only loads this workflow file; checkout still defaults to main.
-          ref: task/FOUR-33523-B
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
           path: dot-github
           token: ${{ secrets.GIT_TOKEN }}
 
@@ -409,8 +426,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ProcessMaker/testbench
-          # Pin to feature branch until relay-capable start.sh lands on main.
-          ref: task/FOUR-33523-B
+          ref: ${{ steps.companion.outputs.ref }}
           path: testbench
           token: ${{ secrets.GIT_TOKEN }}
 
@@ -507,8 +523,9 @@ jobs:
       - name: Checkout .github repo
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
-          ref: main
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
 
       - name: Delete instance EKS
         id: delete_instance
@@ -605,7 +622,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
 
       - name: Common 
         uses: ./.github/actions/common
diff --git a/.github/workflows/js-package.yml b/.github/workflows/js-package.yml
index 28e9f6d..1a250a2 100644
--- a/.github/workflows/js-package.yml
+++ b/.github/workflows/js-package.yml
@@ -54,8 +54,9 @@ jobs:
       - name: Checkout .github
         uses: actions/checkout@v4
         with:
-          repository: ProcessMaker/.github
-          ref: main
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
 
       - uses: actions/setup-node@v4
         with:
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index fae411d..cc1df6e 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -219,12 +219,25 @@ jobs:
       matrix:
         tenant: ${{ fromJSON(inputs.tenants_json) }}
     steps:
+      - name: Resolve companion branch
+        id: companion
+        shell: bash
+        run: |
+          # Keep testbench on the same branch name as this reusable workflow
+          # (e.g. task/FOUR-33523-B). Falls back to main for tags/SHAs.
+          wr='${{ github.workflow_ref }}'
+          if [[ "$wr" == *@refs/heads/* ]]; then
+            echo "ref=${wr##*@refs/heads/}" >> "$GITHUB_OUTPUT"
+          else
+            echo "ref=main" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Checkout dot-github
         uses: actions/checkout@v4
         with:
-          repository: processmaker/.github
-          # Pin to feature branch until relay scripts land on main.
-          ref: task/FOUR-33523-B
+          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
           path: dot-github
           token: ${{ secrets.GIT_TOKEN }}
 
@@ -247,8 +260,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ProcessMaker/testbench
-          # Pin to feature branch until relay-capable start.sh lands on main.
-          ref: task/FOUR-33523-B
+          ref: ${{ steps.companion.outputs.ref }}
           path: testbench
           token: ${{ secrets.GIT_TOKEN }}
 

From 5eb7e45ce193f67c715689fdf09e8cfc82e5d04d Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 23 Oct 2026 13:07:43 -0700
Subject: [PATCH 18/20] Add correct companion

---
 .github/scripts/deploy-instance.sh           |  0
 .github/workflows/deploy-pm4.yml             | 13 ++++++++-----
 .github/workflows/nightly-testbench-arch.yml | 13 ++++++++-----
 3 files changed, 16 insertions(+), 10 deletions(-)
 mode change 100644 => 100755 .github/scripts/deploy-instance.sh

diff --git a/.github/scripts/deploy-instance.sh b/.github/scripts/deploy-instance.sh
old mode 100644
new mode 100755
diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index 4aa84eb..a3501f9 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -389,14 +389,17 @@ jobs:
         id: companion
         shell: bash
         run: |
-          # Keep testbench on the same branch name as this reusable workflow
-          # (e.g. task/FOUR-33523-B). Falls back to main for tags/SHAs.
-          wr='${{ github.workflow_ref }}'
+          # job_workflow_ref is the *called* reusable workflow (e.g. .github@task/FOUR-33523-B).
+          # github.workflow_ref is the *caller* (often refs/pull/N/merge) and must not be used here.
+          wr='${{ github.job_workflow_ref }}'
+          echo "job_workflow_ref=$wr"
           if [[ "$wr" == *@refs/heads/* ]]; then
-            echo "ref=${wr##*@refs/heads/}" >> "$GITHUB_OUTPUT"
+            ref="${wr##*@refs/heads/}"
           else
-            echo "ref=main" >> "$GITHUB_OUTPUT"
+            ref=main
           fi
+          echo "companion_ref=$ref"
+          echo "ref=$ref" >> "$GITHUB_OUTPUT"
 
       - name: Checkout dot-github
         uses: actions/checkout@v4
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index cc1df6e..0e42ac7 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -223,14 +223,17 @@ jobs:
         id: companion
         shell: bash
         run: |
-          # Keep testbench on the same branch name as this reusable workflow
-          # (e.g. task/FOUR-33523-B). Falls back to main for tags/SHAs.
-          wr='${{ github.workflow_ref }}'
+          # job_workflow_ref is the *called* reusable workflow (e.g. .github@task/FOUR-33523-B).
+          # github.workflow_ref is the *caller* (often refs/pull/N/merge) and must not be used here.
+          wr='${{ github.job_workflow_ref }}'
+          echo "job_workflow_ref=$wr"
           if [[ "$wr" == *@refs/heads/* ]]; then
-            echo "ref=${wr##*@refs/heads/}" >> "$GITHUB_OUTPUT"
+            ref="${wr##*@refs/heads/}"
           else
-            echo "ref=main" >> "$GITHUB_OUTPUT"
+            ref=main
           fi
+          echo "companion_ref=$ref"
+          echo "ref=$ref" >> "$GITHUB_OUTPUT"
 
       - name: Checkout dot-github
         uses: actions/checkout@v4

From fea24bd42a3c25a8cd9d66c959e7a76533a2edba Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Fri, 23 Oct 2026 14:52:05 -0700
Subject: [PATCH 19/20] Test 1

---
 .github/workflows/deploy-pm4.yml             | 21 +++++++++++++-------
 .github/workflows/nightly-testbench-arch.yml | 17 +++++++++-------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index a3501f9..b24f38d 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -388,14 +388,21 @@ jobs:
       - name: Resolve companion branch
         id: companion
         shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GIT_TOKEN }}
         run: |
-          # job_workflow_ref is the *called* reusable workflow (e.g. .github@task/FOUR-33523-B).
-          # github.workflow_ref is the *caller* (often refs/pull/N/merge) and must not be used here.
-          wr='${{ github.job_workflow_ref }}'
-          echo "job_workflow_ref=$wr"
-          if [[ "$wr" == *@refs/heads/* ]]; then
-            ref="${wr##*@refs/heads/}"
-          else
+          # Nested reusable workflows leave github.job_workflow_ref empty; job.workflow_sha
+          # is reliable. Map that SHA to a branch tip so testbench tracks the same series
+          # (e.g. task/FOUR-33523-B). Fall back to ci:k8s-branch, then main.
+          sha='${{ job.workflow_sha }}'
+          repo='${{ job.workflow_repository }}'
+          echo "workflow_sha=$sha repo=$repo"
+          ref="$(gh api "repos/${repo}/commits/${sha}/branches-where-head" --jq '.[0].name // empty' || true)"
+          if [ -z "$ref" ] && [[ "${CI_PR_BODY:-}" =~ ci:k8s-branch:([^[:space:]]+) ]]; then
+            ref="${BASH_REMATCH[1]}"
+            echo "branches-where-head empty; using ci:k8s-branch=$ref"
+          fi
+          if [ -z "$ref" ]; then
             ref=main
           fi
           echo "companion_ref=$ref"
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index 0e42ac7..d8c426d 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -222,14 +222,17 @@ jobs:
       - name: Resolve companion branch
         id: companion
         shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GIT_TOKEN }}
         run: |
-          # job_workflow_ref is the *called* reusable workflow (e.g. .github@task/FOUR-33523-B).
-          # github.workflow_ref is the *caller* (often refs/pull/N/merge) and must not be used here.
-          wr='${{ github.job_workflow_ref }}'
-          echo "job_workflow_ref=$wr"
-          if [[ "$wr" == *@refs/heads/* ]]; then
-            ref="${wr##*@refs/heads/}"
-          else
+          # Nested reusable workflows leave github.job_workflow_ref empty; job.workflow_sha
+          # is reliable. Map that SHA to a branch tip so testbench tracks the same series
+          # (e.g. task/FOUR-33523-B). Fall back to main.
+          sha='${{ job.workflow_sha }}'
+          repo='${{ job.workflow_repository }}'
+          echo "workflow_sha=$sha repo=$repo"
+          ref="$(gh api "repos/${repo}/commits/${sha}/branches-where-head" --jq '.[0].name // empty' || true)"
+          if [ -z "$ref" ]; then
             ref=main
           fi
           echo "companion_ref=$ref"

From 731e68e822435738f18b4349486d5d860801bd02 Mon Sep 17 00:00:00 2001
From: Nolan Ehrstrom <github@nolanpro.com>
Date: Mon, 26 Oct 2026 11:41:01 -0700
Subject: [PATCH 20/20] Remove temporary branch name from workflow comments.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/workflows/deploy-pm4.yml             | 14 +++++++-------
 .github/workflows/js-package.yml             |  2 +-
 .github/workflows/nightly-testbench-arch.yml |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml
index b24f38d..e700562 100644
--- a/.github/workflows/deploy-pm4.yml
+++ b/.github/workflows/deploy-pm4.yml
@@ -78,7 +78,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
 
@@ -179,7 +179,7 @@ jobs:
       - name: Checkout .github repo
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
 
@@ -302,7 +302,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
 
@@ -393,7 +393,7 @@ jobs:
         run: |
           # Nested reusable workflows leave github.job_workflow_ref empty; job.workflow_sha
           # is reliable. Map that SHA to a branch tip so testbench tracks the same series
-          # (e.g. task/FOUR-33523-B). Fall back to ci:k8s-branch, then main.
+          # (e.g. the calling feature branch). Fall back to ci:k8s-branch, then main.
           sha='${{ job.workflow_sha }}'
           repo='${{ job.workflow_repository }}'
           echo "workflow_sha=$sha repo=$repo"
@@ -411,7 +411,7 @@ jobs:
       - name: Checkout dot-github
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
           path: dot-github
@@ -533,7 +533,7 @@ jobs:
       - name: Checkout .github repo
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
 
@@ -632,7 +632,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
 
diff --git a/.github/workflows/js-package.yml b/.github/workflows/js-package.yml
index 1a250a2..b134e9e 100644
--- a/.github/workflows/js-package.yml
+++ b/.github/workflows/js-package.yml
@@ -54,7 +54,7 @@ jobs:
       - name: Checkout .github
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
 
diff --git a/.github/workflows/nightly-testbench-arch.yml b/.github/workflows/nightly-testbench-arch.yml
index d8c426d..a12c6b8 100644
--- a/.github/workflows/nightly-testbench-arch.yml
+++ b/.github/workflows/nightly-testbench-arch.yml
@@ -227,7 +227,7 @@ jobs:
         run: |
           # Nested reusable workflows leave github.job_workflow_ref empty; job.workflow_sha
           # is reliable. Map that SHA to a branch tip so testbench tracks the same series
-          # (e.g. task/FOUR-33523-B). Fall back to main.
+          # (e.g. the calling feature branch). Fall back to main.
           sha='${{ job.workflow_sha }}'
           repo='${{ job.workflow_repository }}'
           echo "workflow_sha=$sha repo=$repo"
@@ -241,7 +241,7 @@ jobs:
       - name: Checkout dot-github
         uses: actions/checkout@v4
         with:
-          # Match the reusable-workflow ref (e.g. task/FOUR-33523-B), not main.
+          # Match the reusable-workflow ref (e.g. the calling feature branch), not main.
           repository: ${{ job.workflow_repository }}
           ref: ${{ job.workflow_sha }}
           path: dot-github