Check that each release calls its own environment's simulation service

.github/scripts/check_simulation_entrypoint_environment.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+# Check that a deployment environment calls its own Simulation Entrypoint:
+# staging must call the staging service and production the production
+# service. The entrypoint names the Cloud Run revision that served a request
+# in a response header, and a Cloud Run revision name is its service name
+# followed by a five-digit number and a three-character suffix.
+#
+# The old gateway (SIM_ENTRYPOINT=old_gateway_direct) is one shared Modal app
+# that reports no environment, so it is not checked.
+#
+# Required env vars: DEPLOYMENT_ENVIRONMENT (staging or production),
+# SIM_ENTRYPOINT, and the URL variable SIM_ENTRYPOINT selects.
+
+set -euo pipefail
+
+source .github/scripts/simulation_entrypoint_env.sh
+
+deployment_environment="${DEPLOYMENT_ENVIRONMENT:-}"
+case "${deployment_environment}" in
+  staging)
+    expected_service="policyengine-simulation-entry-staging"
+    ;;
+  production)
+    expected_service="policyengine-simulation-entry"
+    ;;
+  *)
+    printf 'DEPLOYMENT_ENVIRONMENT must be staging or production, not %q\n' \
+      "${deployment_environment}" >&2
+    exit 2
+    ;;
+esac
+
+if [[ "${SIM_ENTRYPOINT:-}" == "old_gateway_direct" ]]; then
+  printf 'SIM_ENTRYPOINT=old_gateway_direct: the shared old gateway reports no environment, so there is nothing to check\n'
+  exit 0
+fi
+
+simulation_url="$(simulation_entrypoint_selected_url "${SIM_ENTRYPOINT:-}")"
+simulation_url="${simulation_url%/}"
+
+health_response="$(
+  curl --fail --silent --show-error --include \
+    --connect-timeout 10 --max-time 60 \
+    "${simulation_url}/health"
+)"
+
+served_revision="$(
+  awk '
+    tolower($1) == "x-policyengine-simulation-revision:" {
+      gsub("\r", "", $2)
+      print $2
+    }
+  ' <<<"${health_response}" |
+    tail -n 1
+)"
+
+if [[ -z "${served_revision}" ]]; then
+  printf '%s/health did not name a Simulation Entrypoint revision; the %s deployment must call the stable URL of %s\n' \
+    "${simulation_url}" "${deployment_environment}" "${expected_service}" >&2
+  exit 1
+fi
+
+if ! [[ "${served_revision}" =~ ^(.+)-[0-9]{5}-[a-z0-9]{3}$ ]]; then
+  printf '%s, served by %s, is not a Cloud Run revision name\n' \
+    "${served_revision}" "${simulation_url}" >&2
+  exit 1
+fi
+served_service="${BASH_REMATCH[1]}"
+
+if [[ "${served_service}" != "${expected_service}" ]]; then
+  printf 'The %s deployment calls %s, which revision %s of %s serves; it must call %s\n' \
+    "${deployment_environment}" "${simulation_url}" "${served_revision}" \
+    "${served_service}" "${expected_service}" >&2
+  exit 1
+fi
+
+printf 'The %s deployment calls %s, served by revision %s of %s\n' \
+  "${deployment_environment}" "${simulation_url}" "${served_revision}" \
+  "${served_service}"

.github/workflows/push.yml

@@ -73,11 +73,14 @@ jobs:
       && (github.event.head_commit.message == 'Update PolicyEngine API')
     environment: staging
     env:
+      DEPLOYMENT_ENVIRONMENT: staging
       SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }}
       SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+      - name: Check the simulation service belongs to staging
+        run: bash .github/scripts/check_simulation_entrypoint_environment.sh
       - name: Install jq
         run: sudo apt-get install -y jq
       - name: Set up uv for installed bundle validation
@@ -438,11 +441,14 @@ jobs:
       && (github.event.head_commit.message == 'Update PolicyEngine API')
     environment: production
     env:
+      DEPLOYMENT_ENVIRONMENT: production
       SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }}
       SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+      - name: Check the simulation service belongs to production
+        run: bash .github/scripts/check_simulation_entrypoint_environment.sh
       - name: Install jq
         run: sudo apt-get install -y jq
       - name: Set up uv for installed bundle validation

changelog.d/staging-simulation-entrypoint-check.added.md

@@ -0,0 +1 @@
+Each release now checks, before deploying to staging or production, that the environment's simulation URL is served by that environment's Simulation Entrypoint service, and fails otherwise; staging must call the staging simulation service, not production.

docs/migration/cloud-run-operations.md

@@ -259,6 +259,19 @@ not block a direct-Modal deployment. The legacy `SIMULATION_API_URL` secret is
 intentionally unsupported: its value is opaque, cannot be audited, and must
 not silently choose a deployment upstream.
 
+Each environment calls its own Simulation Entrypoint. The `staging` secret
+holds the stable URL of the `policyengine-simulation-entry-staging` Cloud Run
+service, which calls the staging Modal environment; the `production` secret
+holds `https://simulation.api.policyengine.org`, served by
+`policyengine-simulation-entry`. Before each release deploys to an
+environment, `.github/scripts/check_simulation_entrypoint_environment.sh`
+reads the `x-policyengine-simulation-revision` header from the selected URL's
+`/health` response and fails the release unless the revision belongs to that
+environment's service. Staging's live simulation tests therefore run against
+staging simulation workers, never production ones. The old gateway is one
+shared Modal app that reports no environment, so `old_gateway_direct` is not
+checked.
+
 ## IAM and bootstrap constraints
 
 - The GitHub deploy service account holds `roles/run.developer`: it can deploy to

tests/unit/test_simulation_entrypoint_environment_script.py

@@ -0,0 +1,202 @@
+"""Tests for .github/scripts/check_simulation_entrypoint_environment.sh."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO = Path(__file__).resolve().parents[2]
+SCRIPT = REPO / ".github/scripts/check_simulation_entrypoint_environment.sh"
+PUSH_WORKFLOW = REPO / ".github/workflows/push.yml"
+SIMULATION_URL = "https://simulation.example.test"
+STAGING_REVISION = "policyengine-simulation-entry-staging-00070-abc"
+PRODUCTION_REVISION = "policyengine-simulation-entry-00122-low"
+HEALTHY_BODY = '{"status":"healthy"}'
+# Cloud Run's placeholder image answers with an HTML page and no revision header.
+PLACEHOLDER_RESPONSE = (
+    "HTTP/2 200\r\ncontent-type: text/html; charset=utf-8\r\n\r\n<html></html>"
+)
+
+
+def _entrypoint_response(revision: str) -> str:
+    return (
+        "HTTP/2 200\r\n"
+        "content-type: application/json\r\n"
+        f"x-policyengine-simulation-revision: {revision}\r\n"
+        "\r\n"
+        f"{HEALTHY_BODY}"
+    )
+
+
+def _fake_curl(tmp_path: Path, response: str, *, exit_code: int = 0) -> Path:
+    """Install a fake curl that records its arguments and prints ``response``."""
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    response_file = tmp_path / "response.txt"
+    response_file.write_text(response, encoding="utf-8")
+    calls = tmp_path / "curl-calls.log"
+    curl = bin_dir / "curl"
+    curl.write_text(
+        "#!/usr/bin/env bash\n"
+        f'echo "$*" >> "{calls}"\n'
+        f'cat "{response_file}"\n'
+        f"exit {exit_code}\n",
+        encoding="utf-8",
+    )
+    curl.chmod(0o755)
+    return bin_dir
+
+
+def _run(
+    tmp_path: Path,
+    response: str,
+    *,
+    deployment_environment: str,
+    sim_entrypoint: str = "cloud_run_simulation_entrypoint",
+    curl_exit_code: int = 0,
+) -> tuple[subprocess.CompletedProcess[str], list[str]]:
+    bin_dir = _fake_curl(tmp_path, response, exit_code=curl_exit_code)
+    result = subprocess.run(
+        ["bash", str(SCRIPT)],
+        cwd=REPO,
+        env={
+            "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+            "DEPLOYMENT_ENVIRONMENT": deployment_environment,
+            "SIM_ENTRYPOINT": sim_entrypoint,
+            "SIMULATION_ENTRYPOINT_URL": SIMULATION_URL,
+            "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test",
+        },
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    calls_file = tmp_path / "curl-calls.log"
+    calls = calls_file.read_text().splitlines() if calls_file.exists() else []
+    return result, calls
+
+
+@pytest.mark.parametrize(
+    ("deployment_environment", "revision"),
+    [("staging", STAGING_REVISION), ("production", PRODUCTION_REVISION)],
+)
+def test_accepts_the_simulation_service_of_the_same_environment(
+    tmp_path: Path, deployment_environment: str, revision: str
+) -> None:
+    result, calls = _run(
+        tmp_path,
+        _entrypoint_response(revision),
+        deployment_environment=deployment_environment,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert len(calls) == 1
+    assert calls[0].endswith(f"{SIMULATION_URL}/health")
+    assert revision in result.stdout
+
+
+def test_rejects_staging_calling_the_production_simulation_service(
+    tmp_path: Path,
+) -> None:
+    result, _ = _run(
+        tmp_path,
+        _entrypoint_response(PRODUCTION_REVISION),
+        deployment_environment="staging",
+    )
+
+    assert result.returncode == 1
+    assert "policyengine-simulation-entry-staging" in result.stderr
+    assert PRODUCTION_REVISION in result.stderr
+
+
+def test_rejects_production_calling_the_staging_simulation_service(
+    tmp_path: Path,
+) -> None:
+    result, _ = _run(
+        tmp_path,
+        _entrypoint_response(STAGING_REVISION),
+        deployment_environment="production",
+    )
+
+    assert result.returncode == 1
+    assert STAGING_REVISION in result.stderr
+
+
+def test_rejects_a_url_that_names_no_entrypoint_revision(tmp_path: Path) -> None:
+    result, _ = _run(tmp_path, PLACEHOLDER_RESPONSE, deployment_environment="staging")
+
+    assert result.returncode == 1
+    assert "did not name a Simulation Entrypoint revision" in result.stderr
+
+
+def test_rejects_a_revision_name_without_a_cloud_run_suffix(tmp_path: Path) -> None:
+    result, _ = _run(
+        tmp_path,
+        _entrypoint_response("policyengine-simulation-entry-staging"),
+        deployment_environment="staging",
+    )
+
+    assert result.returncode == 1
+    assert "is not a Cloud Run revision name" in result.stderr
+
+
+def test_fails_when_the_health_request_fails(tmp_path: Path) -> None:
+    result, _ = _run(
+        tmp_path,
+        "",
+        deployment_environment="staging",
+        curl_exit_code=22,
+    )
+
+    assert result.returncode != 0
+
+
+def test_rejects_an_unknown_deployment_environment(tmp_path: Path) -> None:
+    result, calls = _run(
+        tmp_path,
+        _entrypoint_response(STAGING_REVISION),
+        deployment_environment="preview",
+    )
+
+    assert result.returncode == 2
+    assert "DEPLOYMENT_ENVIRONMENT must be staging or production" in result.stderr
+    assert calls == []
+
+
+def test_does_not_check_the_shared_old_gateway(tmp_path: Path) -> None:
+    result, calls = _run(
+        tmp_path,
+        _entrypoint_response(PRODUCTION_REVISION),
+        deployment_environment="staging",
+        sim_entrypoint="old_gateway_direct",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert calls == []
+    assert "old_gateway_direct" in result.stdout
+
+
+@pytest.mark.parametrize(
+    ("job_name", "deployment_environment"),
+    [
+        ("ensure-staging-model-version-aligns-with-sim-api", "staging"),
+        ("ensure-production-model-version-aligns-with-sim-api", "production"),
+    ],
+)
+def test_each_release_checks_its_simulation_service_before_deploying(
+    job_name: str, deployment_environment: str
+) -> None:
+    job = yaml.safe_load(PUSH_WORKFLOW.read_text(encoding="utf-8"))["jobs"][job_name]
+
+    assert job["environment"] == deployment_environment
+    assert job["env"]["DEPLOYMENT_ENVIRONMENT"] == deployment_environment
+    assert job["env"]["SIM_ENTRYPOINT"] == "${{ vars.SIM_ENTRYPOINT }}"
+    assert (
+        job["env"]["SIMULATION_ENTRYPOINT_URL"]
+        == "${{ secrets.SIMULATION_ENTRYPOINT_URL }}"
+    )
+    commands = [step.get("run", "") for step in job["steps"]]
+    assert "bash .github/scripts/check_simulation_entrypoint_environment.sh" in commands