Refactor testbench CI for Tailscale relay

.github/scripts/delete-instance.sh

@@ -7,6 +7,8 @@
 #   DELETE_HARBOR     true to delete the Harbor artifact (default false)
 #   IMAGE_TAG         Harbor tag to delete when DELETE_HARBOR=true
 #   REGISTRY_USERNAME, REGISTRY_PASSWORD, REGISTRY_HOST
+#   PRESERVE_TLS_DIR  if set, export TLS secret here before wiping the namespace
+#                     (for GHA artifact restore on redeploy)
 set -u
 
 FAILED=0
@@ -21,6 +23,7 @@ NAMESPACE="ci-${INSTANCE}-ns-pm4"
 RELEASE="ci-${INSTANCE}"
 deploy_db="pm4_ci-${INSTANCE}%"
 deploy_ai="pm4_ci-${INSTANCE}_ai"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 if [ ${#deploy_db} -lt 12 ]; then
   echo "ERROR: deploy_db safety check failed (length ${#deploy_db})"
@@ -29,19 +32,57 @@ fi
 
 set +e
 
-if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
-  echo "Deleting Instance :: ${RELEASE}"
-  helm delete "${RELEASE}" || true
-  kubectl delete namespace "${NAMESPACE}" || true
+preserve_tls() {
+  if [ -z "${PRESERVE_TLS_DIR:-}" ]; then
+    return 0
+  fi
+  echo "Preserving TLS secret before delete (PRESERVE_TLS_DIR=${PRESERVE_TLS_DIR})"
+  # shellcheck disable=SC1091
+  bash "${SCRIPT_DIR}/export-tls-secret.sh"
+}
+
+delete_k8s() {
+  if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
+    echo "Deleting Instance :: ${RELEASE}"
+    helm delete "${RELEASE}" || true
+    kubectl delete namespace "${NAMESPACE}" || true
+    echo "The instance [https://ci-${INSTANCE}.engk8s.processmaker.net] was removed from the cluster."
+    return 0
+  fi
+
+  echo "Namespace ${NAMESPACE} not found; skipping cluster delete."
+  return 0
+}
+
+delete_rds() {
+  echo "Cleaning RDS for instance ${INSTANCE} (always runs, even when the namespace is already gone)."
 
   set -o pipefail
-  mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -N -e "SHOW DATABASES LIKE '${deploy_db}'" \
-    | xargs -r -I{} mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`{}\`;"
-  if [ $? -ne 0 ]; then
-    echo "WARN: Failed dropping databases matching ${deploy_db}"
+  matching_dbs=$(
+    mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -N -e "SHOW DATABASES LIKE '${deploy_db}'"
+  )
+  list_status=$?
+  set +o pipefail
+
+  if [ "${list_status}" -ne 0 ]; then
+    echo "WARN: Failed listing databases matching ${deploy_db}"
     FAILED=1
+    return 1
+  fi
+
+  if [ -z "${matching_dbs}" ]; then
+    echo "No databases matching ${deploy_db} on RDS (already clean)."
+  else
+    echo "Dropping databases matching ${deploy_db}:"
+    echo "${matching_dbs}"
+    set -o pipefail
+    echo "${matching_dbs}" | xargs -r -I{} mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`{}\`;"
+    if [ $? -ne 0 ]; then
+      echo "WARN: Failed dropping databases matching ${deploy_db}"
+      FAILED=1
+    fi
+    set +o pipefail
   fi
-  set +o pipefail
 
   mysql -u"${USER_MYSQL_ENG}" -p"${PASS_MYSQL_ENG}" -h "${RDS_ENG}" -e "DROP DATABASE IF EXISTS \`${deploy_ai}\`;"
   if [ $? -ne 0 ]; then
@@ -61,10 +102,12 @@ if kubectl get "namespace/${NAMESPACE}" >/dev/null 2>&1; then
     FAILED=1
   fi
 
-  echo "The instance [https://ci-${INSTANCE}.engk8s.processmaker.net] was deleted."
-else
-  echo "Namespace ${NAMESPACE} not found; nothing to delete on the cluster."
-fi
+  return 0
+}
+
+preserve_tls
+delete_k8s
+delete_rds
 
 if [ "${DELETE_HARBOR}" = "true" ]; then
   if [ -z "${IMAGE_TAG:-}" ]; then

.github/scripts/deploy-instance.sh

@@ -4,9 +4,35 @@ current_datetime=$(echo -n ${CURRENT_DATE} | md5sum | head -c 10)
 echo "NAMESPACE : ci-{{INSTANCE}}-ns-pm4"
 helm repo add processmaker ${HELM_REPO} --username ${HELM_USERNAME} --password ${HELM_PASSWORD} && helm repo update
 
+helm_common_sets() {
+  echo \
+    --set deploy.web.octane.enable=${OCTANE} \
+    --set deploy.pmai.openaiApiKey=${OPENAI_API_KEY} \
+    --set analytics.awsAccessKey=${ANALYTICS_AWS_ACCESS_KEY} \
+    --set analytics.awsSecretKey=${ANALYTICS_AWS_SECRET_KEY} \
+    --set dockerRegistry.password=${REGISTRY_PASSWORD} \
+    --set dockerRegistry.url=${REGISTRY_HOST} \
+    --set dockerRegistry.username=${REGISTRY_USERNAME} \
+    --set twilio.sid=${TWILIO_SID} \
+    --set twilio.token=${TWILIO_TOKEN} \
+    --set appVersion=${APP_VERSION}
+}
+
+# Optional: TLS_SECRET_DIR with a stashed LE cert from a prior delete (GHA artifact).
+# Restore before helm install so cert-manager does not burn a new Let's Encrypt issuance.
+restore_stashed_tls() {
+    if [ -z "${TLS_SECRET_DIR:-}" ]; then
+        return 0
+    fi
+    export INSTANCE="{{INSTANCE}}"
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    bash "${SCRIPT_DIR}/restore-tls-secret.sh"
+}
+
 if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then
     echo "New instance. Creating Namespace"
     kubectl create namespace ci-{{INSTANCE}}-ns-pm4
+    restore_stashed_tls
     echo "Creating DB"
     # Use admin password from secrets
     echo "Update instance yamls"
@@ -34,20 +60,14 @@ if ! kubectl get namespace/ci-{{INSTANCE}}-ns-pm4 >/dev/null 2>&1; then
     APP_VERSION=$(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g" | sed "s/refs-heads-//g")
 
     helm install --timeout 75m -f .github/templates/instance.yaml ci-{{INSTANCE}} processmaker/enterprise \
-        --set deploy.web.octane.enable=${OCTANE} \
-        --set deploy.pmai.openaiApiKey=${OPENAI_API_KEY} \
-        --set analytics.awsAccessKey=${ANALYTICS_AWS_ACCESS_KEY} \
-        --set analytics.awsSecretKey=${ANALYTICS_AWS_SECRET_KEY} \
-        --set dockerRegistry.password=${REGISTRY_PASSWORD} \
-        --set dockerRegistry.url=${REGISTRY_HOST} \
-        --set dockerRegistry.username=${REGISTRY_USERNAME} \
-        --set twilio.sid=${TWILIO_SID} \
-        --set twilio.token=${TWILIO_TOKEN} \
-        --set appVersion=${APP_VERSION} \
+        $(helm_common_sets) \
         --version ${versionHelm}
 else
     echo "Instance exists. Running upgrade and bouncing pods"
-    helm upgrade --timeout 60m ci-{{INSTANCE}} processmaker/enterprise --version ${versionHelm}
+    APP_VERSION=$(echo "$CI_PROJECT-$CI_PACKAGE_BRANCH" | sed "s;/;-;g" | sed "s/refs-heads-//g")
+    helm upgrade --timeout 60m -f .github/templates/instance.yaml ci-{{INSTANCE}} processmaker/enterprise \
+        $(helm_common_sets) \
+        --version ${versionHelm}
 
     #Bounce pods
     webPod=$(kubectl get pods -n ci-{{INSTANCE}}-ns-pm4|grep web|awk '{print $1}')

.github/scripts/disable-testbench-relay.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required}"
+RELEASE="ci-${INSTANCE}"
+
+if [ -z "${versionHelm:-}" ]; then
+  echo "versionHelm is required"
+  exit 1
+fi
+
+if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then
+  echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required"
+  exit 1
+fi
+
+helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true
+helm repo update
+
+echo "Disabling testbench runner relay for release ${RELEASE}..."
+# --no-hooks: relay-only change; skip post-upgrade update-pm4 (and other hooks).
+helm upgrade --timeout 10m "${RELEASE}" processmaker/enterprise \
+  --reuse-values \
+  --no-hooks \
+  --set testbenchRunnerRelay.enable=false \
+  --set testbenchRunnerRelay.tailscale.authKey= \
+  --version "${versionHelm}"
+
+echo "Relay disabled."

.github/scripts/enable-testbench-relay.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}"
+RELEASE="ci-${INSTANCE}"
+NAMESPACE="${RELEASE}-ns-pm4"
+DEPLOYMENT="${RELEASE}-testbench-runner-relay"
+
+if [ -z "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
+  echo "TAILSCALE_AUTH_KEY_CI_RELAY is required"
+  exit 1
+fi
+
+RELAY_VALUES="${RELAY_VALUES_FILE:-.github/templates/testbench-relay.yaml}"
+if [ ! -f "${RELAY_VALUES}" ]; then
+  echo "Relay values file not found: ${RELAY_VALUES}"
+  exit 1
+fi
+
+RELAY_VALUES_RENDERED=$(mktemp)
+sed "s#{{INSTANCE}}#${INSTANCE}#g" "${RELAY_VALUES}" > "${RELAY_VALUES_RENDERED}"
+trap 'rm -f "${RELAY_VALUES_RENDERED}"' EXIT
+
+# Prefer the chart cloned by the Common action (ci:k8s-branch) so unpublished
+# template fixes are picked up without waiting on a Harbor chart publish.
+CHART_REF="${PM4_K8S_CHART:-}"
+if [ -z "${CHART_REF}" ] && [ -f ../pm4-k8s-distribution/charts/enterprise/Chart.yaml ]; then
+  CHART_REF="../pm4-k8s-distribution/charts/enterprise"
+fi
+if [ -z "${CHART_REF}" ] && [ -f pm4-k8s-distribution/charts/enterprise/Chart.yaml ]; then
+  CHART_REF="pm4-k8s-distribution/charts/enterprise"
+fi
+
+HELM_CHART_ARGS=()
+if [ -n "${CHART_REF}" ]; then
+  echo "Using local enterprise chart: ${CHART_REF}"
+  HELM_CHART_ARGS=("${CHART_REF}")
+else
+  if [ -z "${versionHelm:-}" ]; then
+    echo "versionHelm is required when no local chart is available"
+    exit 1
+  fi
+  if [ -z "${HELM_REPO:-}" ] || [ -z "${HELM_USERNAME:-}" ] || [ -z "${HELM_PASSWORD:-}" ]; then
+    echo "HELM_REPO, HELM_USERNAME, and HELM_PASSWORD are required when no local chart is available"
+    exit 1
+  fi
+  helm repo add processmaker "${HELM_REPO}" --username "${HELM_USERNAME}" --password "${HELM_PASSWORD}" 2>/dev/null || true
+  helm repo update
+  HELM_CHART_ARGS=(processmaker/enterprise --version "${versionHelm}")
+fi
+
+echo "Enabling testbench runner relay for release ${RELEASE} (upstream tb-ci-${INSTANCE})..."
+# --no-hooks: instance is already ready; skip post-upgrade update-pm4 (and other hooks).
+helm upgrade --timeout 15m "${RELEASE}" "${HELM_CHART_ARGS[@]}" \
+  --reuse-values \
+  --no-hooks \
+  -f "${RELAY_VALUES_RENDERED}" \
+  --set testbenchRunnerRelay.enable=true \
+  --set "testbenchRunnerRelay.upstreamHost=tb-ci-${INSTANCE}" \
+  --set "testbenchRunnerRelay.tailscale.authKey=${TAILSCALE_AUTH_KEY_CI_RELAY}"
+
+if kubectl get deployment "${DEPLOYMENT}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  if ! kubectl rollout status "deployment/${DEPLOYMENT}" -n "${NAMESPACE}" --timeout=180s; then
+    echo "Relay rollout failed; collecting diagnostics..."
+    kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay -o wide || true
+    kubectl describe pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true
+    kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --tail=200 || true
+    kubectl logs -n "${NAMESPACE}" -l service=testbench-runner-relay --previous --tail=200 || true
+    exit 1
+  fi
+else
+  echo "Warning: deployment ${DEPLOYMENT} not found after upgrade"
+  kubectl get pods -n "${NAMESPACE}" -l service=testbench-runner-relay || true
+  exit 1
+fi
+
+echo "Testbench runner relay is ready."

.github/scripts/ensure-testbench-relay.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+# Prepare an existing in-cluster testbench relay for a local/CI runner join (no Helm).
+# Run before the runner Tailscale node comes up (start.sh with ENSURE_TESTBENCH_RELAY_SCRIPT).
+set -euo pipefail
+
+INSTANCE="${1:?INSTANCE id required (e.g. nt-amd64 or 10-char md5)}"
+RELEASE="ci-${INSTANCE}"
+NAMESPACE="${RELEASE}-ns-pm4"
+DEPLOYMENT="${RELEASE}-testbench-runner-relay"
+RELAY_DEPLOY="deploy/${DEPLOYMENT}"
+CONFIGMAP="${RELEASE}-testbench-runner-relay"
+SECRET="${RELEASE}-testbench-relay-tailscale"
+UPSTREAM_HOST="tb-ci-${INSTANCE}"
+RELAY_HAPROXY_CHECK_INTER="${RELAY_HAPROXY_CHECK_INTER:-60s}"
+
+if ! command -v kubectl >/dev/null 2>&1; then
+  echo "kubectl is required to ensure the testbench runner relay"
+  exit 1
+fi
+
+if ! kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  echo "Relay deployment not found: ${DEPLOYMENT} in namespace ${NAMESPACE}."
+  echo ""
+  echo "The testbench runner relay was never enabled for this CI instance."
+  echo "Run a CI testbench job once (enable-testbench-relay.sh), or use Helm with"
+  echo "enable-testbench-relay.sh when HELM credentials are available."
+  exit 1
+fi
+
+echo "Ensuring testbench runner relay for ${RELEASE} (upstream ${UPSTREAM_HOST})..."
+
+if kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+  haproxy_cfg="$(kubectl get configmap "${CONFIGMAP}" -n "${NAMESPACE}" -o jsonpath='{.data.haproxy\.cfg}')"
+  patched_cfg="$(printf '%s' "${haproxy_cfg}" \
+    | sed -E "s/server runner [^:]+:/server runner ${UPSTREAM_HOST}:/g" \
+    | sed -E "s/check inter [0-9]+[smhd]+/check inter ${RELAY_HAPROXY_CHECK_INTER}/g")"
+  if ! printf '%s' "${patched_cfg}" | grep -q 'hard-stop-after'; then
+    patched_cfg="$(printf '%s' "${patched_cfg}" | awk '
+      /^[[:space:]]*maxconn / { print; print "      hard-stop-after 5s"; next }
+      { print }
+    ')"
+  fi
+  if [ "${patched_cfg}" = "${haproxy_cfg}" ]; then
+    echo "ConfigMap upstream (${UPSTREAM_HOST}) and health check interval (${RELAY_HAPROXY_CHECK_INTER}) already set."
+  else
+    echo "Patching ConfigMap ${CONFIGMAP} (upstream ${UPSTREAM_HOST}, check inter ${RELAY_HAPROXY_CHECK_INTER})..."
+    kubectl create configmap "${CONFIGMAP}" -n "${NAMESPACE}" \
+      --from-literal=haproxy.cfg="${patched_cfg}" \
+      --dry-run=client -o yaml \
+      | kubectl apply -f -
+  fi
+else
+  echo "WARNING: ConfigMap ${CONFIGMAP} not found; skipping HAProxy patch"
+fi
+
+if [ -n "${TAILSCALE_AUTH_KEY_CI_RELAY:-}" ]; then
+  if kubectl get secret "${SECRET}" -n "${NAMESPACE}" >/dev/null 2>&1; then
+    echo "Refreshing relay Tailscale auth key secret ${SECRET}..."
+    kubectl create secret generic "${SECRET}" -n "${NAMESPACE}" \
+      --from-literal=TS_AUTHKEY="${TAILSCALE_AUTH_KEY_CI_RELAY}" \
+      --dry-run=client -o yaml \
+      | kubectl apply -f -
+  else
+    echo "WARNING: Secret ${SECRET} not found; skipping auth key refresh"
+  fi
+fi
+
+echo "Applying fast relay rollout settings (Recreate strategy, exec probes, grace period)..."
+# JSON patch (not --type=merge): merge patch replaces the entire containers[]
+# array and drops required fields like image.
+# Replace whole probes so a prior tcpSocket handler cannot remain beside exec.
+TS_PROBE='tailscale --socket=/var/run/tailscale/tailscaled.sock status >/dev/null 2>&1'
+grace_op="add"
+if kubectl get "${RELAY_DEPLOY}" -n "${NAMESPACE}" \
+  -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}' 2>/dev/null | grep -q '[0-9]'; then
+  grace_op="replace"
+fi
+kubectl patch "${RELAY_DEPLOY}" -n "${NAMESPACE}" --type=json -p "$(cat <<EOF
+[
+  {"op": "replace", "path": "/spec/strategy", "value": {"type": "Recreate"}},
+  {"op": "${grace_op}", "path": "/spec/template/spec/terminationGracePeriodSeconds", "value": 10},
+  {"op": "replace", "path": "/spec/template/spec/containers/0/readinessProbe", "value": {
+    "exec": {"command": ["/bin/sh", "-c", "${TS_PROBE}"]},
+    "initialDelaySeconds": 3,
+    "periodSeconds": 5,
+    "failureThreshold": 20
+  }},
+  {"op": "replace", "path": "/spec/template/spec/containers/0/livenessProbe", "value": {
+    "exec": {"command": ["/bin/sh", "-c", "${TS_PROBE}"]},
+    "initialDelaySeconds": 15,
+    "periodSeconds": 15
+  }}
+]
+EOF
+)" || echo "WARNING: could not patch relay deployment rollout settings"
+
+echo "Relay prepare complete (ConfigMap / deploy settings). verify-testbench-relay.sh restarts after the runner joins."