From 8b5f355b678eb4d20b0a5fe08a3c99e6b85a500a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:32:30 +0400 Subject: [PATCH 1/2] Test that each live staging test gets its own pytest worker run_live_staging_tests.sh runs the 9 live staging tests in two pytest workers, so the step takes about as long as the tests on the busier worker (11.4 minutes in release #2608) rather than the longest test (about 7.4). These tests ask for one worker per collected test, and for the step name that says so. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_cloud_run_push_workflow.py | 61 +++++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_cloud_run_push_workflow.py b/tests/unit/test_cloud_run_push_workflow.py index c040674d9..e0c41371d 100644 --- a/tests/unit/test_cloud_run_push_workflow.py +++ b/tests/unit/test_cloud_run_push_workflow.py @@ -2,7 +2,10 @@ from __future__ import annotations +import os import re +import subprocess +import sys from tests.unit.cloud_run_deploy_support import ( PRODUCTION_CLOUD_RUN_SERVICE, @@ -25,6 +28,8 @@ "scripts/resolve_cloud_run_candidate_state.sh", "scripts/set_cloud_run_revision.sh", ) +LIVE_TESTS_STEP = "Run live staging tests, one pytest worker per test" +LIVE_TESTS_SCRIPT = REPO / ".github/scripts/run_live_staging_tests.sh" RELEASE_TESTS_JOB = "release-tests" RELEASE_TESTS_STEP = "Run release tests" RELEASE_TESTS_COMMAND = "bash .github/scripts/run_release_tests.sh" @@ -181,9 +186,8 @@ def _assert_live_tests_run_in_parallel_after_readiness( ) < parallel_live_tests.index(live_readiness_command) assert parallel_live_tests.index( live_readiness_command - ) < parallel_live_tests.index("Run live staging tests in two pytest workers") + ) < parallel_live_tests.index(LIVE_TESTS_STEP) assert "bash .github/scripts/run_live_staging_tests.sh" in parallel_live_tests - assert "-n 2" in live_test_script assert "--dist load" in live_test_script assert "--maxschedchunk=1" in live_test_script assert "tests/integration/test_live_budget_window_cache.py" in live_test_script @@ -198,6 +202,59 @@ def _assert_live_tests_run_in_parallel_after_readiness( assert "github.run_attempt" in parallel_live_tests +def _live_test_worker_count(live_test_script: str) -> int: + worker_counts = re.findall(r"^\s*-n (\S+) \\$", live_test_script, re.MULTILINE) + assert len(worker_counts) == 1, "the live test script must pass -n once" + assert worker_counts[0].isdigit(), "the live test worker count must be a number" + return int(worker_counts[0]) + + +def _collected_live_tests(live_test_script: str) -> list[str]: + test_files = re.findall(r"tests/integration/test_live_\w+\.py", live_test_script) + collection_env = { + name: value for name, value in os.environ.items() if name != "API_BASE_URL" + } + collection = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "-p", + "no:cacheprovider", + *test_files, + ], + cwd=REPO, + env=collection_env, + capture_output=True, + text=True, + check=False, + ) + assert collection.returncode == 0, collection.stdout + collection.stderr + return [ + collected_line + for collected_line in collection.stdout.splitlines() + if "::" in collected_line + ] + + +def test_live_staging_tests_get_one_pytest_worker_each(): + live_test_script = LIVE_TESTS_SCRIPT.read_text(encoding="utf-8") + worker_count = _live_test_worker_count(live_test_script) + collected_tests = _collected_live_tests(live_test_script) + + # Each test waits on its own requests, so with one worker each the step + # takes about as long as the longest test. A new live test needs -n raised + # with it, after checking the staging candidate's capacity: one Cloud Run + # instance admits 6 requests at a time (cloud_run_env.sh), each test sends + # one at a time, and further requests wait at the Cloud Run front end. + assert worker_count == len(collected_tests), ( + f"run_live_staging_tests.sh runs {len(collected_tests)} tests in " + f"{worker_count} pytest workers; give each test its own worker" + ) + + def _assert_promotion_verifies_the_tested_candidate( workflow: str, production_check: str, cloud_run_promotion: str ) -> None: From e615bf3b15fc7c329faafeee78dfcfdc14868543 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:33:17 +0400 Subject: [PATCH 2/2] Give each live staging test its own pytest worker run_live_staging_tests.sh passes -n 9 instead of -n 2, one worker per test, so the step takes about as long as the longest test instead of the tests on the busier worker. The step is renamed to say so. The staging candidate is one Cloud Run instance with a concurrency of 6, and each test sends one request at a time, so at most 3 requests wait at the Cloud Run front end for a free slot. Cloud Run holds a request for the longer of 10 seconds and 3.5 times the average instance startup time, and the tests' client allows 90 seconds per request. The script's comment says this, and asks for -n to be raised, after checking the candidate's capacity, whenever a live test is added. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/run_live_staging_tests.sh | 14 ++++++++++++-- .github/workflows/push.yml | 2 +- changelog.d/live-staging-test-workers.changed.md | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 changelog.d/live-staging-test-workers.changed.md diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 6a1217bd8..d2cb5eaa6 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -2,9 +2,19 @@ set -euo pipefail # Run from the environment uv.lock defines (make install), which includes -# pytest-xdist for the two workers. +# pytest-xdist for the workers. +# +# One pytest worker per test: each test waits on its own requests to the +# staging candidate and the simulation API, so the step takes about as long as +# the longest test. tests/unit/test_cloud_run_push_workflow.py checks that -n +# equals the number of tests here. The candidate is one Cloud Run instance that +# admits 6 requests at a time (cloud_run_env.sh). Each test sends one request at +# a time, so at most 3 requests wait at the Cloud Run front end for a free slot: +# Cloud Run holds them for the longer of 10 seconds and 3.5 times the average +# instance startup time, and the tests' client allows each request 90 seconds. +# Before adding a test, check the candidate can take one more. uv run --frozen --extra dev python -m pytest \ - -n 2 \ + -n 9 \ --dist load \ --maxschedchunk=1 \ tests/integration/test_live_budget_window_cache.py \ diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 964b60c6f..ea9658024 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -407,7 +407,7 @@ jobs: run: make install - name: Wait for Cloud Run staging candidate health before live tests run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check" - - name: Run live staging tests in two pytest workers + - name: Run live staging tests, one pytest worker per test run: bash .github/scripts/run_live_staging_tests.sh env: API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} diff --git a/changelog.d/live-staging-test-workers.changed.md b/changelog.d/live-staging-test-workers.changed.md new file mode 100644 index 000000000..7f8a230f1 --- /dev/null +++ b/changelog.d/live-staging-test-workers.changed.md @@ -0,0 +1 @@ +The live staging tests in a release run each get their own pytest worker, so the step takes about as long as the longest test instead of the tests on the busier of two workers.