diff --git a/.github/country_main_tests.py b/.github/country_main_tests.py
new file mode 100644
index 000000000..d2fb15c75
--- /dev/null
+++ b/.github/country_main_tests.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+"""Run policyengine-us main's Core-facing tests against the Core under test.
+
+The country smoke job installs a pinned policyengine-us release and checks
+that its system loads, so it cannot see a policyengine-us test that a Core
+change breaks. On 2026-10-27, 3.33.13 broke policyengine-us main's
+test_social_security_input_state_machine.py within minutes of its release.
+
+The PolicyEngineUSMain job checks out policyengine-us main and runs the files
+in .github/requirements/policyengine-us-main-tests.txt twice, each with a
+JUnit report: first with the Core release on PyPI, then with this checkout.
+
+- `select LIST REPO` prints the listed files that exist in REPO, one per
+ line, and warns about the rest, which policyengine-us renamed or removed.
+- `compare RELEASED CANDIDATE` fails on each test that fails with this Core
+ but did not fail with the release. A test that fails with both is a
+ policyengine-us failure, so it is only a warning: policyengine-us main is
+ sometimes red for its own reasons.
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+from xml.etree import ElementTree
+
+
+def listed_files(list_path):
+ """The paths in a list file, without comments or blank lines."""
+ paths = []
+ for line in Path(list_path).read_text().splitlines():
+ line = line.split("#", 1)[0].strip()
+ if line:
+ paths.append(line)
+ return paths
+
+
+def select(list_path, repo):
+ """The listed files that exist in the checkout, and those that do not."""
+ present, missing = [], []
+ for path in listed_files(list_path):
+ (present if (Path(repo) / path).is_file() else missing).append(path)
+ return present, missing
+
+
+def outcomes(report):
+ """Each test's outcome in a JUnit report: passed, failed or skipped.
+
+ A test can appear twice, as when its teardown errors after it passed;
+ a failure in either entry counts.
+ """
+ results = {}
+ for case in ElementTree.parse(report).getroot().iter("testcase"):
+ test = f"{case.get('classname', '')}::{case.get('name', '')}"
+ tags = {child.tag for child in case}
+ if tags & {"failure", "error"}:
+ outcome = "failed"
+ elif "skipped" in tags:
+ outcome = "skipped"
+ else:
+ outcome = "passed"
+ if results.get(test) != "failed":
+ results[test] = outcome
+ return results
+
+
+def compare(released, candidate):
+ """Tests this Core breaks, and tests that fail with both Cores."""
+ broken = sorted(
+ test
+ for test, outcome in candidate.items()
+ if outcome == "failed" and released.get(test) != "failed"
+ )
+ both = sorted(
+ test
+ for test, outcome in candidate.items()
+ if outcome == "failed" and released.get(test) == "failed"
+ )
+ return broken, both
+
+
+def summary(released, candidate, broken, both):
+ lines = [
+ "### policyengine-us main against this Core",
+ "",
+ f"{len(candidate)} tests ran with this Core and {len(released)} with "
+ "the Core release.",
+ "",
+ ]
+ if broken:
+ lines.append(f"**This Core breaks {len(broken)}:**")
+ lines += [f"- `{test}`" for test in broken]
+ lines.append("")
+ if both:
+ lines.append(f"Failing with both Cores (policyengine-us's own): {len(both)}")
+ lines += [f"- `{test}`" for test in both]
+ if not broken and not both:
+ lines.append("Every test passes with this Core.")
+ return "\n".join(lines) + "\n"
+
+
+def run_compare(released_report, candidate_report, summary_path=None):
+ if not Path(candidate_report).exists():
+ print(
+ "::error::This Core's run wrote no JUnit report: pytest could not "
+ "start or was killed."
+ )
+ return 1
+ if not Path(released_report).exists():
+ print(
+ "::warning::The Core release's run wrote no JUnit report, so "
+ "nothing can be blamed on this Core."
+ )
+ return 0
+ released = outcomes(released_report)
+ candidate = outcomes(candidate_report)
+ broken, both = compare(released, candidate)
+ for test in both:
+ print(f"::warning::Fails with the Core release too: {test}")
+ for test in broken:
+ print(f"::error::This Core breaks {test}")
+ text = summary(released, candidate, broken, both)
+ print(text)
+ if summary_path:
+ with open(summary_path, "a") as output:
+ output.write(text)
+ return 1 if broken else 0
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ commands = parser.add_subparsers(dest="command", required=True)
+ chooser = commands.add_parser("select", help="list the files to run")
+ chooser.add_argument("list")
+ chooser.add_argument("repo")
+ checker = commands.add_parser("compare", help="compare the two runs")
+ checker.add_argument("released")
+ checker.add_argument("candidate")
+ checker.add_argument(
+ "--summary",
+ default=os.environ.get("GITHUB_STEP_SUMMARY"),
+ help="file to append a Markdown summary to",
+ )
+ args = parser.parse_args(argv)
+ if args.command == "select":
+ present, missing = select(args.list, args.repo)
+ for path in missing:
+ print(
+ f"::warning::policyengine-us main has no {path}; update "
+ ".github/requirements/policyengine-us-main-tests.txt.",
+ file=sys.stderr,
+ )
+ if not present:
+ print("::error::None of the listed files exist.", file=sys.stderr)
+ return 1
+ print("\n".join(present))
+ return 0
+ return run_compare(args.released, args.candidate, args.summary)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/.github/requirements/policyengine-us-main-tests.txt b/.github/requirements/policyengine-us-main-tests.txt
new file mode 100644
index 000000000..84eb2e5e5
--- /dev/null
+++ b/.github/requirements/policyengine-us-main-tests.txt
@@ -0,0 +1,26 @@
+# policyengine-us test files the PolicyEngineUSMain job runs from a checkout of
+# policyengine-us main, first with the Core release on PyPI and then with the
+# Core under test (.github/country_main_tests.py). Paths are relative to the
+# policyengine-us repository root.
+#
+# Each file drives Core internals (simulations, holders, data storage, tools,
+# populations, parameters or reforms) and is fast: together they took about
+# 95 seconds in policyengine-us release run 40544965026. The state machine test
+# runs Core's installed source bodies in namespaces it builds; Core 3.33.13
+# broke it on 2026-10-27, which this job would have shown on the PR.
+#
+# A file policyengine-us renames or removes is skipped with a warning, so keep
+# the list in step when that happens.
+policyengine_us/tests/core/test_social_security_input_state_machine.py
+policyengine_us/tests/core/test_social_security_real_core.py
+policyengine_us/tests/core/test_loss_ald_extraction_parity.py
+policyengine_us/tests/core/test_capital_gains_rate_worksheet_properties.py
+policyengine_us/tests/core/test_ald_determinism.py
+policyengine_us/tests/core/test_pandas3_compatibility.py
+policyengine_us/tests/core/test_social_security_lever_properties.py
+policyengine_us/tests/test_locality_sales_tax_rates.py
+policyengine_us/tests/test_parameter_scale_thresholds.py
+policyengine_us/tests/code_health/test_no_group_variables_in_person_adds.py
+policyengine_us/tests/microsimulation/data/test_extend_single_year_dataset.py
+policyengine_us/tests/policy/baseline/parameters/test_uprating_extensions.py
+policyengine_us/tests/policy/baseline/gov/states/or/tax/income/credits/test_or_ctc_sunset.py
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
index c90189cb6..7df05951e 100644
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@@ -4,10 +4,10 @@ on:
branches: [master]
# A new push to a pull request supersedes the run on its previous head. Cancel
-# that run instead of letting its 19 jobs (8 Test, 8 smoke, 3 checks) keep
-# holding org-wide shared runners. From 2026-10-06 to 10-08, 21 of 55 PR runs
-# were superseded while still running, and finishing them used 850
-# runner-minutes (26% of this workflow).
+# that run instead of letting its 20 jobs (8 Test, 8 smoke, 1 policyengine-us
+# main, 3 checks) keep holding org-wide shared runners. From 2026-10-06 to
+# 10-08, 21 of 55 PR runs were superseded while still running, and finishing
+# them used 850 runner-minutes (26% of this workflow).
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
@@ -121,3 +121,57 @@ jobs:
env:
RUN_SMOKE_TESTS: "0"
POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ PolicyEngineUSMain:
+ name: Test policyengine-us main against this Core
+ runs-on: ubuntu-latest
+ needs: Lint
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ env:
+ PYTHONUNBUFFERED: 1
+ steps:
+ - name: Checkout repo
+ uses: actions/checkout@v6
+ with:
+ path: core
+ persist-credentials: false
+ - name: Checkout policyengine-us main
+ uses: actions/checkout@v6
+ with:
+ repository: PolicyEngine/policyengine-us
+ ref: main
+ path: policyengine-us
+ persist-credentials: false
+ - name: Set up Python
+ uses: actions/setup-python@v6
+ with:
+ python-version: "3.14"
+ - name: Install uv
+ uses: astral-sh/setup-uv@v8.1.0
+ # As policyengine-us's own CI does: its lock with the newest Core release.
+ - name: Install policyengine-us main with the Core release
+ working-directory: policyengine-us
+ run: |
+ uv lock --upgrade-package policyengine-core
+ uv sync --locked --extra dev
+ bash ./update_itemization.sh
+ - name: Select the policyengine-us tests
+ run: python core/.github/country_main_tests.py select core/.github/requirements/policyengine-us-main-tests.txt policyengine-us > selected.txt
+ # Both runs draw the same Hypothesis examples, so a difference between
+ # them comes from Core. A failing run still writes its report.
+ - name: Run them with the Core release
+ working-directory: policyengine-us
+ run: |
+ uv pip list | grep -i '^policyengine-core '
+ xargs uv run --no-sync python -m pytest -p no:cacheprovider --hypothesis-seed=0 --junitxml=../released.xml < ../selected.txt || true
+ - name: Install this Core
+ working-directory: policyengine-us
+ run: |
+ uv pip install --python .venv/bin/python --reinstall-package policyengine-core ../core
+ uv run --no-sync python -c "import json; from importlib.metadata import distribution; url = json.loads(distribution('policyengine-core').read_text('direct_url.json'))['url']; print(url); assert url.startswith('file://'), url"
+ - name: Run them with this Core
+ working-directory: policyengine-us
+ run: xargs uv run --no-sync python -m pytest -p no:cacheprovider --hypothesis-seed=0 --junitxml=../candidate.xml < ../selected.txt || true
+ - name: Fail on tests this Core breaks
+ run: python core/.github/country_main_tests.py compare released.xml candidate.xml
diff --git a/changelog.d/test-policyengine-us-main.changed.md b/changelog.d/test-policyengine-us-main.changed.md
new file mode 100644
index 000000000..52ab3be87
--- /dev/null
+++ b/changelog.d/test-policyengine-us-main.changed.md
@@ -0,0 +1 @@
+Add a PR job that runs policyengine-us main's Core-facing tests with the Core release and then with the Core under test, and fails on tests the change breaks.
diff --git a/tests/core/test_country_main_tests.py b/tests/core/test_country_main_tests.py
new file mode 100644
index 000000000..2cf14c6ee
--- /dev/null
+++ b/tests/core/test_country_main_tests.py
@@ -0,0 +1,169 @@
+"""The PolicyEngineUSMain job blames Core only for the tests Core breaks.
+
+It runs policyengine-us main's listed tests with the Core release and then
+with the Core under test (.github/country_main_tests.py). These tests check
+the comparison on synthetic JUnit reports, the file selection, the list, and
+the order of the job's steps.
+"""
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / ".github" / "country_main_tests.py"
+LIST = ROOT / ".github" / "requirements" / "policyengine-us-main-tests.txt"
+WORKFLOW = ROOT / ".github" / "workflows" / "pr.yaml"
+SPEC = importlib.util.spec_from_file_location("country_main_tests", SCRIPT)
+country = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(country)
+
+
+def junit(path, cases):
+ """Write a JUnit report of (classname, name, outcome) cases."""
+ body = ""
+ for classname, name, outcome in cases:
+ child = {
+ "passed": "",
+ "failed": '',
+ "error": '',
+ "skipped": '',
+ }[outcome]
+ body += f'{child}'
+ path.write_text(f"{body}")
+ return path
+
+
+def test_outcomes_count_a_failure_in_any_entry_of_a_test(tmp_path):
+ report = junit(
+ tmp_path / "r.xml",
+ [
+ ("a.test_x", "test_pass", "passed"),
+ ("a.test_x", "test_fail", "failed"),
+ ("a.test_x", "test_skip", "skipped"),
+ ("a.test_x.TestCase", "runTest", "passed"),
+ ("a.test_x.TestCase", "runTest", "error"),
+ ],
+ )
+ assert country.outcomes(report) == {
+ "a.test_x::test_pass": "passed",
+ "a.test_x::test_fail": "failed",
+ "a.test_x::test_skip": "skipped",
+ "a.test_x.TestCase::runTest": "failed",
+ }
+
+
+def test_only_tests_failing_with_this_core_alone_are_blamed_on_it():
+ released = {"t::a": "passed", "t::b": "failed", "t::c": "skipped"}
+ candidate = {
+ "t::a": "failed", # this Core breaks it
+ "t::b": "failed", # policyengine-us's own failure
+ "t::c": "failed", # skipped before, fails now
+ "t::d": "failed", # absent before, such as a new collection error
+ "t::e": "passed",
+ }
+ assert country.compare(released, candidate) == (
+ ["t::a", "t::c", "t::d"],
+ ["t::b"],
+ )
+
+
+@pytest.mark.parametrize(
+ "released_cases, candidate_cases, status",
+ [
+ ([("t", "a", "passed")], [("t", "a", "failed")], 1),
+ ([("t", "a", "failed")], [("t", "a", "failed")], 0),
+ ([("t", "a", "passed")], [("t", "a", "passed")], 0),
+ (None, [("t", "a", "failed")], 0),
+ ([("t", "a", "passed")], None, 1),
+ ],
+ ids=["broken", "fails-with-both", "passes", "no-release-report", "no-report"],
+)
+def test_compare_fails_only_on_breaks_or_a_missing_report(
+ tmp_path, capsys, released_cases, candidate_cases, status
+):
+ released = tmp_path / "released.xml"
+ candidate = tmp_path / "candidate.xml"
+ if released_cases is not None:
+ junit(released, released_cases)
+ if candidate_cases is not None:
+ junit(candidate, candidate_cases)
+ summary = tmp_path / "summary.md"
+ assert (
+ country.main(
+ ["compare", str(released), str(candidate), "--summary", str(summary)]
+ )
+ == status
+ )
+ output = capsys.readouterr().out
+ if status and candidate_cases is not None:
+ assert "::error::This Core breaks t::a" in output
+ assert "**This Core breaks 1:**" in summary.read_text()
+ if released_cases == candidate_cases == [("t", "a", "failed")]:
+ assert "::warning::Fails with the Core release too: t::a" in output
+
+
+def test_select_skips_files_the_checkout_lacks(tmp_path, capsys):
+ repo = tmp_path / "policyengine-us"
+ (repo / "tests").mkdir(parents=True)
+ (repo / "tests" / "test_kept.py").write_text("")
+ listing = tmp_path / "list.txt"
+ listing.write_text(
+ "# comment\n\ntests/test_kept.py # trailing comment\ntests/test_gone.py\n"
+ )
+ assert country.select(listing, repo) == (
+ ["tests/test_kept.py"],
+ ["tests/test_gone.py"],
+ )
+ assert country.main(["select", str(listing), str(repo)]) == 0
+ captured = capsys.readouterr()
+ assert captured.out == "tests/test_kept.py\n"
+ assert "::warning::policyengine-us main has no tests/test_gone.py" in captured.err
+ listing.write_text("tests/test_gone.py\n")
+ assert country.main(["select", str(listing), str(repo)]) == 1
+
+
+def test_the_list_names_distinct_policyengine_us_test_files():
+ paths = country.listed_files(LIST)
+ assert paths, "the list is empty"
+ assert len(paths) == len(set(paths)), "a file is listed twice"
+ for path in paths:
+ assert path.startswith("policyengine_us/tests/"), path
+ assert Path(path).name.startswith("test_") and path.endswith(".py"), path
+
+
+def test_the_job_runs_the_release_then_this_core_and_compares_them():
+ job = yaml.safe_load(WORKFLOW.read_text())["jobs"]["PolicyEngineUSMain"]
+ names = [step["name"] for step in job["steps"]]
+ assert names == [
+ "Checkout repo",
+ "Checkout policyengine-us main",
+ "Set up Python",
+ "Install uv",
+ "Install policyengine-us main with the Core release",
+ "Select the policyengine-us tests",
+ "Run them with the Core release",
+ "Install this Core",
+ "Run them with this Core",
+ "Fail on tests this Core breaks",
+ ]
+ steps = {step["name"]: step for step in job["steps"]}
+ us = steps["Checkout policyengine-us main"]["with"]
+ assert (us["repository"], us["ref"]) == ("PolicyEngine/policyengine-us", "main")
+ released = steps["Run them with the Core release"]["run"]
+ candidate = steps["Run them with this Core"]["run"]
+ # The same tests and Hypothesis seed, with separate reports.
+ assert released.splitlines()[-1].replace("released.xml", "candidate.xml") == (
+ candidate
+ )
+ assert "--hypothesis-seed=0" in candidate
+ # The candidate replaces the release even at the same version number,
+ # and the job checks that it was installed from this checkout.
+ install = steps["Install this Core"]["run"]
+ assert "--reinstall-package policyengine-core ../core" in install
+ assert "assert url.startswith('file://')" in install
+ assert steps["Fail on tests this Core breaks"]["run"].endswith(
+ "compare released.xml candidate.xml"
+ )