From dd1bc619785e50bc9ecca714e27ccd1a24f32100 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:45:17 +0400 Subject: [PATCH] Check that each release calls its own environment's simulation service The staging and production releases both read SIMULATION_ENTRYPOINT_URL from their GitHub environment, and both secrets point at the production Simulation Entrypoint, so staging's live simulation tests ran on production simulation workers. Before each release deploys, a new script reads the revision header from the selected URL's /health response and fails unless the revision belongs to that environment's service: policyengine-simulation-entry-staging for staging, policyengine-simulation-entry for production. The shared old gateway reports no environment and is not checked. Co-Authored-By: Claude Opus 5.5 --- ...check_simulation_entrypoint_environment.sh | 80 +++++++ .github/workflows/push.yml | 6 + ...aging-simulation-entrypoint-check.added.md | 1 + docs/migration/cloud-run-operations.md | 13 ++ ...imulation_entrypoint_environment_script.py | 202 ++++++++++++++++++ 5 files changed, 302 insertions(+) create mode 100644 .github/scripts/check_simulation_entrypoint_environment.sh create mode 100644 changelog.d/staging-simulation-entrypoint-check.added.md create mode 100644 tests/unit/test_simulation_entrypoint_environment_script.py diff --git a/.github/scripts/check_simulation_entrypoint_environment.sh b/.github/scripts/check_simulation_entrypoint_environment.sh new file mode 100644 index 000000000..45781f74f --- /dev/null +++ b/.github/scripts/check_simulation_entrypoint_environment.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Check that a deployment environment calls its own Simulation Entrypoint: +# staging must call the staging service and production the production +# service. The entrypoint names the Cloud Run revision that served a request +# in a response header, and a Cloud Run revision name is its service name +# followed by a five-digit number and a three-character suffix. +# +# The old gateway (SIM_ENTRYPOINT=old_gateway_direct) is one shared Modal app +# that reports no environment, so it is not checked. +# +# Required env vars: DEPLOYMENT_ENVIRONMENT (staging or production), +# SIM_ENTRYPOINT, and the URL variable SIM_ENTRYPOINT selects. + +set -euo pipefail + +source .github/scripts/simulation_entrypoint_env.sh + +deployment_environment="${DEPLOYMENT_ENVIRONMENT:-}" +case "${deployment_environment}" in + staging) + expected_service="policyengine-simulation-entry-staging" + ;; + production) + expected_service="policyengine-simulation-entry" + ;; + *) + printf 'DEPLOYMENT_ENVIRONMENT must be staging or production, not %q\n' \ + "${deployment_environment}" >&2 + exit 2 + ;; +esac + +if [[ "${SIM_ENTRYPOINT:-}" == "old_gateway_direct" ]]; then + printf 'SIM_ENTRYPOINT=old_gateway_direct: the shared old gateway reports no environment, so there is nothing to check\n' + exit 0 +fi + +simulation_url="$(simulation_entrypoint_selected_url "${SIM_ENTRYPOINT:-}")" +simulation_url="${simulation_url%/}" + +health_response="$( + curl --fail --silent --show-error --include \ + --connect-timeout 10 --max-time 60 \ + "${simulation_url}/health" +)" + +served_revision="$( + awk ' + tolower($1) == "x-policyengine-simulation-revision:" { + gsub("\r", "", $2) + print $2 + } + ' <<<"${health_response}" | + tail -n 1 +)" + +if [[ -z "${served_revision}" ]]; then + printf '%s/health did not name a Simulation Entrypoint revision; the %s deployment must call the stable URL of %s\n' \ + "${simulation_url}" "${deployment_environment}" "${expected_service}" >&2 + exit 1 +fi + +if ! [[ "${served_revision}" =~ ^(.+)-[0-9]{5}-[a-z0-9]{3}$ ]]; then + printf '%s, served by %s, is not a Cloud Run revision name\n' \ + "${served_revision}" "${simulation_url}" >&2 + exit 1 +fi +served_service="${BASH_REMATCH[1]}" + +if [[ "${served_service}" != "${expected_service}" ]]; then + printf 'The %s deployment calls %s, which revision %s of %s serves; it must call %s\n' \ + "${deployment_environment}" "${simulation_url}" "${served_revision}" \ + "${served_service}" "${expected_service}" >&2 + exit 1 +fi + +printf 'The %s deployment calls %s, served by revision %s of %s\n' \ + "${deployment_environment}" "${simulation_url}" "${served_revision}" \ + "${served_service}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7aa6cfa8d..b9bcdf800 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -73,11 +73,14 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: staging env: + DEPLOYMENT_ENVIRONMENT: staging SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Check the simulation service belongs to staging + run: bash .github/scripts/check_simulation_entrypoint_environment.sh - name: Install jq run: sudo apt-get install -y jq - name: Set up uv for installed bundle validation @@ -438,11 +441,14 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: production env: + DEPLOYMENT_ENVIRONMENT: production SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Check the simulation service belongs to production + run: bash .github/scripts/check_simulation_entrypoint_environment.sh - name: Install jq run: sudo apt-get install -y jq - name: Set up uv for installed bundle validation diff --git a/changelog.d/staging-simulation-entrypoint-check.added.md b/changelog.d/staging-simulation-entrypoint-check.added.md new file mode 100644 index 000000000..afeb5900d --- /dev/null +++ b/changelog.d/staging-simulation-entrypoint-check.added.md @@ -0,0 +1 @@ +Each release now checks, before deploying to staging or production, that the environment's simulation URL is served by that environment's Simulation Entrypoint service, and fails otherwise; staging must call the staging simulation service, not production. diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index a69209d24..75d4f8572 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -259,6 +259,19 @@ not block a direct-Modal deployment. The legacy `SIMULATION_API_URL` secret is intentionally unsupported: its value is opaque, cannot be audited, and must not silently choose a deployment upstream. +Each environment calls its own Simulation Entrypoint. The `staging` secret +holds the stable URL of the `policyengine-simulation-entry-staging` Cloud Run +service, which calls the staging Modal environment; the `production` secret +holds `https://simulation.api.policyengine.org`, served by +`policyengine-simulation-entry`. Before each release deploys to an +environment, `.github/scripts/check_simulation_entrypoint_environment.sh` +reads the `x-policyengine-simulation-revision` header from the selected URL's +`/health` response and fails the release unless the revision belongs to that +environment's service. Staging's live simulation tests therefore run against +staging simulation workers, never production ones. The old gateway is one +shared Modal app that reports no environment, so `old_gateway_direct` is not +checked. + ## IAM and bootstrap constraints - The GitHub deploy service account holds `roles/run.developer`: it can deploy to diff --git a/tests/unit/test_simulation_entrypoint_environment_script.py b/tests/unit/test_simulation_entrypoint_environment_script.py new file mode 100644 index 000000000..073334fdf --- /dev/null +++ b/tests/unit/test_simulation_entrypoint_environment_script.py @@ -0,0 +1,202 @@ +"""Tests for .github/scripts/check_simulation_entrypoint_environment.sh.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / ".github/scripts/check_simulation_entrypoint_environment.sh" +PUSH_WORKFLOW = REPO / ".github/workflows/push.yml" +SIMULATION_URL = "https://simulation.example.test" +STAGING_REVISION = "policyengine-simulation-entry-staging-00070-abc" +PRODUCTION_REVISION = "policyengine-simulation-entry-00122-low" +HEALTHY_BODY = '{"status":"healthy"}' +# Cloud Run's placeholder image answers with an HTML page and no revision header. +PLACEHOLDER_RESPONSE = ( + "HTTP/2 200\r\ncontent-type: text/html; charset=utf-8\r\n\r\n" +) + + +def _entrypoint_response(revision: str) -> str: + return ( + "HTTP/2 200\r\n" + "content-type: application/json\r\n" + f"x-policyengine-simulation-revision: {revision}\r\n" + "\r\n" + f"{HEALTHY_BODY}" + ) + + +def _fake_curl(tmp_path: Path, response: str, *, exit_code: int = 0) -> Path: + """Install a fake curl that records its arguments and prints ``response``.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + response_file = tmp_path / "response.txt" + response_file.write_text(response, encoding="utf-8") + calls = tmp_path / "curl-calls.log" + curl = bin_dir / "curl" + curl.write_text( + "#!/usr/bin/env bash\n" + f'echo "$*" >> "{calls}"\n' + f'cat "{response_file}"\n' + f"exit {exit_code}\n", + encoding="utf-8", + ) + curl.chmod(0o755) + return bin_dir + + +def _run( + tmp_path: Path, + response: str, + *, + deployment_environment: str, + sim_entrypoint: str = "cloud_run_simulation_entrypoint", + curl_exit_code: int = 0, +) -> tuple[subprocess.CompletedProcess[str], list[str]]: + bin_dir = _fake_curl(tmp_path, response, exit_code=curl_exit_code) + result = subprocess.run( + ["bash", str(SCRIPT)], + cwd=REPO, + env={ + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "DEPLOYMENT_ENVIRONMENT": deployment_environment, + "SIM_ENTRYPOINT": sim_entrypoint, + "SIMULATION_ENTRYPOINT_URL": SIMULATION_URL, + "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test", + }, + text=True, + capture_output=True, + check=False, + ) + calls_file = tmp_path / "curl-calls.log" + calls = calls_file.read_text().splitlines() if calls_file.exists() else [] + return result, calls + + +@pytest.mark.parametrize( + ("deployment_environment", "revision"), + [("staging", STAGING_REVISION), ("production", PRODUCTION_REVISION)], +) +def test_accepts_the_simulation_service_of_the_same_environment( + tmp_path: Path, deployment_environment: str, revision: str +) -> None: + result, calls = _run( + tmp_path, + _entrypoint_response(revision), + deployment_environment=deployment_environment, + ) + + assert result.returncode == 0, result.stderr + assert len(calls) == 1 + assert calls[0].endswith(f"{SIMULATION_URL}/health") + assert revision in result.stdout + + +def test_rejects_staging_calling_the_production_simulation_service( + tmp_path: Path, +) -> None: + result, _ = _run( + tmp_path, + _entrypoint_response(PRODUCTION_REVISION), + deployment_environment="staging", + ) + + assert result.returncode == 1 + assert "policyengine-simulation-entry-staging" in result.stderr + assert PRODUCTION_REVISION in result.stderr + + +def test_rejects_production_calling_the_staging_simulation_service( + tmp_path: Path, +) -> None: + result, _ = _run( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="production", + ) + + assert result.returncode == 1 + assert STAGING_REVISION in result.stderr + + +def test_rejects_a_url_that_names_no_entrypoint_revision(tmp_path: Path) -> None: + result, _ = _run(tmp_path, PLACEHOLDER_RESPONSE, deployment_environment="staging") + + assert result.returncode == 1 + assert "did not name a Simulation Entrypoint revision" in result.stderr + + +def test_rejects_a_revision_name_without_a_cloud_run_suffix(tmp_path: Path) -> None: + result, _ = _run( + tmp_path, + _entrypoint_response("policyengine-simulation-entry-staging"), + deployment_environment="staging", + ) + + assert result.returncode == 1 + assert "is not a Cloud Run revision name" in result.stderr + + +def test_fails_when_the_health_request_fails(tmp_path: Path) -> None: + result, _ = _run( + tmp_path, + "", + deployment_environment="staging", + curl_exit_code=22, + ) + + assert result.returncode != 0 + + +def test_rejects_an_unknown_deployment_environment(tmp_path: Path) -> None: + result, calls = _run( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="preview", + ) + + assert result.returncode == 2 + assert "DEPLOYMENT_ENVIRONMENT must be staging or production" in result.stderr + assert calls == [] + + +def test_does_not_check_the_shared_old_gateway(tmp_path: Path) -> None: + result, calls = _run( + tmp_path, + _entrypoint_response(PRODUCTION_REVISION), + deployment_environment="staging", + sim_entrypoint="old_gateway_direct", + ) + + assert result.returncode == 0, result.stderr + assert calls == [] + assert "old_gateway_direct" in result.stdout + + +@pytest.mark.parametrize( + ("job_name", "deployment_environment"), + [ + ("ensure-staging-model-version-aligns-with-sim-api", "staging"), + ("ensure-production-model-version-aligns-with-sim-api", "production"), + ], +) +def test_each_release_checks_its_simulation_service_before_deploying( + job_name: str, deployment_environment: str +) -> None: + job = yaml.safe_load(PUSH_WORKFLOW.read_text(encoding="utf-8"))["jobs"][job_name] + + assert job["environment"] == deployment_environment + assert job["env"]["DEPLOYMENT_ENVIRONMENT"] == deployment_environment + assert job["env"]["SIM_ENTRYPOINT"] == "${{ vars.SIM_ENTRYPOINT }}" + assert ( + job["env"]["SIMULATION_ENTRYPOINT_URL"] + == "${{ secrets.SIMULATION_ENTRYPOINT_URL }}" + ) + commands = [step.get("run", "") for step in job["steps"]] + assert "bash .github/scripts/check_simulation_entrypoint_environment.sh" in commands