diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ea9658024..7cecdb294 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -172,10 +172,12 @@ jobs: secrets: inherit seed-v2-staging-database: + # Runs beside migrate-v1-staging-cloud-sql: the seeding job connects only to + # the v2 database and the v1 upgrade only to Cloud SQL, so neither needs the + # other. The staging deploy needs both. name: Seed staging v2 database needs: - publish-git-tag - - migrate-v1-staging-cloud-sql if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -515,10 +517,11 @@ jobs: secrets: inherit seed-v2-production-database: + # Runs beside migrate-v1-production-cloud-sql, as in staging. The + # production deploy needs both. name: Seed production v2 database needs: - ensure-production-model-version-aligns-with-sim-api - - migrate-v1-production-cloud-sql if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -530,7 +533,9 @@ jobs: deploy-cloud-run-candidate: name: Deploy production Cloud Run candidate runs-on: ubuntu-latest - needs: seed-v2-production-database + needs: + - migrate-v1-production-cloud-sql + - seed-v2-production-database if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') diff --git a/changelog.d/parallel-database-steps.changed.md b/changelog.d/parallel-database-steps.changed.md new file mode 100644 index 000000000..332e21ef6 --- /dev/null +++ b/changelog.d/parallel-database-steps.changed.md @@ -0,0 +1 @@ +The release run upgrades the v1 Cloud SQL schema and seeds the v2 database side by side, in staging and in production, instead of one after the other; each Cloud Run candidate deploy still waits for both. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 9bcf0139a..e8cda9a72 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -344,6 +344,13 @@ shared or deployed database. unversioned or invalid history, takes a backup only when revisions are pending, upgrades under the advisory lock, and requires zero post-migration drift. +- In each environment the v1 release migration and the v2 seeding job + (`seed-v2-database.yml`, which upgrades the v2 schema and publishes the + catalog) run side by side: the v1 job connects only to Cloud SQL and the + seeding job only to the v2 database. Each Cloud Run candidate deploy needs + both, so a candidate is never built until both databases are upgraded. When + one of them fails, the release stops with the other already applied; the + serving revision must work with either database upgraded on its own. - Database credentials and derived URLs must never be printed. The application runtime account must not be granted schema-migration privileges. - Production downgrades are never automatic. MySQL DDL is non-transactional, so diff --git a/tests/unit/test_app_engine_decommission.py b/tests/unit/test_app_engine_decommission.py index 926d04727..a078ceef0 100644 --- a/tests/unit/test_app_engine_decommission.py +++ b/tests/unit/test_app_engine_decommission.py @@ -77,7 +77,10 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: docker_publish = _job_block(workflow, "docker") release_tests = _job_block(workflow, "release-tests") - assert "migrate-v1-staging-cloud-sql" in staging_seed + # The v1 schema upgrade and the v2 seeding run side by side, and each + # candidate deploy needs both (tests/unit/v2/test_metadata_deployment.py). + assert "- migrate-v1-staging-cloud-sql" not in staging_seed + assert "- migrate-v1-staging-cloud-sql" in staging_deploy assert "- seed-v2-staging-database" in staging_deploy # The release tests run in their own job, and the staging deploy needs it. assert "- release-tests" in staging_deploy @@ -93,8 +96,9 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: assert "exercise-phase10-staging" not in workflow assert "exercise-phase11-staging" not in workflow assert "needs: promote-cloud-run-staging" in production_check - assert "migrate-v1-production-cloud-sql" in production_seed - assert "needs: seed-v2-production-database" in production_deploy + assert "- migrate-v1-production-cloud-sql" not in production_seed + assert "- migrate-v1-production-cloud-sql" in production_deploy + assert "- seed-v2-production-database" in production_deploy assert "needs: deploy-cloud-run-candidate" in docker_publish diff --git a/tests/unit/test_cloud_run_push_workflow.py b/tests/unit/test_cloud_run_push_workflow.py index e0c41371d..c4ed071b1 100644 --- a/tests/unit/test_cloud_run_push_workflow.py +++ b/tests/unit/test_cloud_run_push_workflow.py @@ -392,7 +392,8 @@ def test_push_workflow_staging_precedes_all_production_deployments(): docker_publish = _workflow_job_block(workflow, "docker") cloud_run_production = _workflow_job_block(workflow, "deploy-cloud-run-candidate") - assert "needs: seed-v2-production-database" in cloud_run_production + assert "- seed-v2-production-database" in cloud_run_production + assert "- migrate-v1-production-cloud-sql" in cloud_run_production assert "needs: deploy-cloud-run-candidate" in docker_publish assert "compute_cloud_run_metadata.sh stage3-prod" in cloud_run_production assert "Build and push Cloud Run image" not in cloud_run_production diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index c752b46c0..eaf4809e7 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -6,8 +6,28 @@ import subprocess from pathlib import Path +from tests.unit.cloud_run_deploy_support import job_needs, push_workflow_jobs + REPO = Path(__file__).resolve().parents[3] +# Per environment: the v1 Cloud SQL schema upgrade, the v2 seeding, the job +# that deploys the Cloud Run candidate, and the job both database jobs start +# after. +DATABASE_JOBS_BY_ENVIRONMENT = { + "staging": { + "v1_upgrade": "migrate-v1-staging-cloud-sql", + "v2_seeding": "seed-v2-staging-database", + "candidate_deploy": "deploy-cloud-run-staging", + "start_after": "publish-git-tag", + }, + "production": { + "v1_upgrade": "migrate-v1-production-cloud-sql", + "v2_seeding": "seed-v2-production-database", + "candidate_deploy": "deploy-cloud-run-candidate", + "start_after": "ensure-production-model-version-aligns-with-sim-api", + }, +} + def _read(relative_path: str) -> str: return (REPO / relative_path).read_text(encoding="utf-8") @@ -142,23 +162,39 @@ def test_privilege_steps_connect_with_the_migration_credential() -> None: def test_seeding_success_is_required_before_candidate_creation() -> None: - workflow = _read(".github/workflows/push.yml") - staging_seed = _job(workflow, "seed-v2-staging-database") - production_seed = _job(workflow, "seed-v2-production-database") - - assert "deployment_environment: staging" in staging_seed - assert "migrate-v1-staging-cloud-sql" in staging_seed - assert "seed-v2-staging-database" in _job( - workflow, - "deploy-cloud-run-staging", - ) - - assert "migrate-v1-production-cloud-sql" in production_seed - assert "deployment_environment: production" in production_seed - assert "needs: seed-v2-production-database" in _job( - workflow, - "deploy-cloud-run-candidate", - ) + jobs = push_workflow_jobs() + + for environment, database_jobs in DATABASE_JOBS_BY_ENVIRONMENT.items(): + seeding = jobs[database_jobs["v2_seeding"]] + candidate_deploy = jobs[database_jobs["candidate_deploy"]] + + assert seeding["with"] == {"deployment_environment": environment} + assert seeding["secrets"] == "inherit" + # The candidate deploy waits for both database jobs. + assert database_jobs["v2_seeding"] in job_needs(candidate_deploy) + assert database_jobs["v1_upgrade"] in job_needs(candidate_deploy) + + +def test_v1_upgrade_and_v2_seeding_run_side_by_side() -> None: + # Neither job reads the other's database: the v1 upgrade connects only to + # Cloud SQL through its proxy, and the seeding job only to the v2 database, + # with no Cloud SQL proxy or v1 credentials. + jobs = push_workflow_jobs() + + for environment, database_jobs in DATABASE_JOBS_BY_ENVIRONMENT.items(): + v1_upgrade = jobs[database_jobs["v1_upgrade"]] + seeding = jobs[database_jobs["v2_seeding"]] + + assert database_jobs["v1_upgrade"] not in job_needs(seeding), environment + assert database_jobs["v2_seeding"] not in job_needs(v1_upgrade), environment + assert database_jobs["start_after"] in job_needs(seeding), environment + assert database_jobs["start_after"] in job_needs(v1_upgrade), environment + # Each keeps the GitHub environment it reads its database target from. + assert v1_upgrade["with"] == { + "deployment_environment": environment, + "github_environment": f"{environment}-database", + } + assert v1_upgrade["secrets"] == "inherit" def test_stage_9_deployment_shell_script_is_syntax_valid() -> None: