Test policyengine-us main against the Core under test on each PR

.github/country_main_tests.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+"""Run policyengine-us main's Core-facing tests against the Core under test.
+
+The country smoke job installs a pinned policyengine-us release and checks
+that its system loads, so it cannot see a policyengine-us test that a Core
+change breaks. On 2026-10-27, 3.33.13 broke policyengine-us main's
+test_social_security_input_state_machine.py within minutes of its release.
+
+The PolicyEngineUSMain job checks out policyengine-us main and runs the files
+in .github/requirements/policyengine-us-main-tests.txt twice, each with a
+JUnit report: first with the Core release on PyPI, then with this checkout.
+
+- `select LIST REPO` prints the listed files that exist in REPO, one per
+  line, and warns about the rest, which policyengine-us renamed or removed.
+- `compare RELEASED CANDIDATE` fails on each test that fails with this Core
+  but did not fail with the release. A test that fails with both is a
+  policyengine-us failure, so it is only a warning: policyengine-us main is
+  sometimes red for its own reasons.
+"""
+
+import argparse
+import os
+import sys
+from pathlib import Path
+from xml.etree import ElementTree
+
+
+def listed_files(list_path):
+    """The paths in a list file, without comments or blank lines."""
+    paths = []
+    for line in Path(list_path).read_text().splitlines():
+        line = line.split("#", 1)[0].strip()
+        if line:
+            paths.append(line)
+    return paths
+
+
+def select(list_path, repo):
+    """The listed files that exist in the checkout, and those that do not."""
+    present, missing = [], []
+    for path in listed_files(list_path):
+        (present if (Path(repo) / path).is_file() else missing).append(path)
+    return present, missing
+
+
+def outcomes(report):
+    """Each test's outcome in a JUnit report: passed, failed or skipped.
+
+    A test can appear twice, as when its teardown errors after it passed;
+    a failure in either entry counts.
+    """
+    results = {}
+    for case in ElementTree.parse(report).getroot().iter("testcase"):
+        test = f"{case.get('classname', '')}::{case.get('name', '')}"
+        tags = {child.tag for child in case}
+        if tags & {"failure", "error"}:
+            outcome = "failed"
+        elif "skipped" in tags:
+            outcome = "skipped"
+        else:
+            outcome = "passed"
+        if results.get(test) != "failed":
+            results[test] = outcome
+    return results
+
+
+def compare(released, candidate):
+    """Tests this Core breaks, and tests that fail with both Cores."""
+    broken = sorted(
+        test
+        for test, outcome in candidate.items()
+        if outcome == "failed" and released.get(test) != "failed"
+    )
+    both = sorted(
+        test
+        for test, outcome in candidate.items()
+        if outcome == "failed" and released.get(test) == "failed"
+    )
+    return broken, both
+
+
+def summary(released, candidate, broken, both):
+    lines = [
+        "### policyengine-us main against this Core",
+        "",
+        f"{len(candidate)} tests ran with this Core and {len(released)} with "
+        "the Core release.",
+        "",
+    ]
+    if broken:
+        lines.append(f"**This Core breaks {len(broken)}:**")
+        lines += [f"- `{test}`" for test in broken]
+        lines.append("")
+    if both:
+        lines.append(f"Failing with both Cores (policyengine-us's own): {len(both)}")
+        lines += [f"- `{test}`" for test in both]
+    if not broken and not both:
+        lines.append("Every test passes with this Core.")
+    return "\n".join(lines) + "\n"
+
+
+def run_compare(released_report, candidate_report, summary_path=None):
+    if not Path(candidate_report).exists():
+        print(
+            "::error::This Core's run wrote no JUnit report: pytest could not "
+            "start or was killed."
+        )
+        return 1
+    if not Path(released_report).exists():
+        print(
+            "::warning::The Core release's run wrote no JUnit report, so "
+            "nothing can be blamed on this Core."
+        )
+        return 0
+    released = outcomes(released_report)
+    candidate = outcomes(candidate_report)
+    broken, both = compare(released, candidate)
+    for test in both:
+        print(f"::warning::Fails with the Core release too: {test}")
+    for test in broken:
+        print(f"::error::This Core breaks {test}")
+    text = summary(released, candidate, broken, both)
+    print(text)
+    if summary_path:
+        with open(summary_path, "a") as output:
+            output.write(text)
+    return 1 if broken else 0
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    commands = parser.add_subparsers(dest="command", required=True)
+    chooser = commands.add_parser("select", help="list the files to run")
+    chooser.add_argument("list")
+    chooser.add_argument("repo")
+    checker = commands.add_parser("compare", help="compare the two runs")
+    checker.add_argument("released")
+    checker.add_argument("candidate")
+    checker.add_argument(
+        "--summary",
+        default=os.environ.get("GITHUB_STEP_SUMMARY"),
+        help="file to append a Markdown summary to",
+    )
+    args = parser.parse_args(argv)
+    if args.command == "select":
+        present, missing = select(args.list, args.repo)
+        for path in missing:
+            print(
+                f"::warning::policyengine-us main has no {path}; update "
+                ".github/requirements/policyengine-us-main-tests.txt.",
+                file=sys.stderr,
+            )
+        if not present:
+            print("::error::None of the listed files exist.", file=sys.stderr)
+            return 1
+        print("\n".join(present))
+        return 0
+    return run_compare(args.released, args.candidate, args.summary)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/requirements/policyengine-us-main-tests.txt

@@ -0,0 +1,26 @@
+# policyengine-us test files the PolicyEngineUSMain job runs from a checkout of
+# policyengine-us main, first with the Core release on PyPI and then with the
+# Core under test (.github/country_main_tests.py). Paths are relative to the
+# policyengine-us repository root.
+#
+# Each file drives Core internals (simulations, holders, data storage, tools,
+# populations, parameters or reforms) and is fast: together they took about
+# 95 seconds in policyengine-us release run 40544965026. The state machine test
+# runs Core's installed source bodies in namespaces it builds; Core 3.33.13
+# broke it on 2026-10-27, which this job would have shown on the PR.
+#
+# A file policyengine-us renames or removes is skipped with a warning, so keep
+# the list in step when that happens.
+policyengine_us/tests/core/test_social_security_input_state_machine.py
+policyengine_us/tests/core/test_social_security_real_core.py
+policyengine_us/tests/core/test_loss_ald_extraction_parity.py
+policyengine_us/tests/core/test_capital_gains_rate_worksheet_properties.py
+policyengine_us/tests/core/test_ald_determinism.py
+policyengine_us/tests/core/test_pandas3_compatibility.py
+policyengine_us/tests/core/test_social_security_lever_properties.py
+policyengine_us/tests/test_locality_sales_tax_rates.py
+policyengine_us/tests/test_parameter_scale_thresholds.py
+policyengine_us/tests/code_health/test_no_group_variables_in_person_adds.py
+policyengine_us/tests/microsimulation/data/test_extend_single_year_dataset.py
+policyengine_us/tests/policy/baseline/parameters/test_uprating_extensions.py
+policyengine_us/tests/policy/baseline/gov/states/or/tax/income/credits/test_or_ctc_sunset.py

.github/workflows/pr.yaml

@@ -4,10 +4,10 @@ on:
     branches: [master]
 
 # A new push to a pull request supersedes the run on its previous head. Cancel
-# that run instead of letting its 19 jobs (8 Test, 8 smoke, 3 checks) keep
-# holding org-wide shared runners. From 2026-10-06 to 10-08, 21 of 55 PR runs
-# were superseded while still running, and finishing them used 850
-# runner-minutes (26% of this workflow).
+# that run instead of letting its 20 jobs (8 Test, 8 smoke, 1 policyengine-us
+# main, 3 checks) keep holding org-wide shared runners. From 2026-10-06 to
+# 10-08, 21 of 55 PR runs were superseded while still running, and finishing
+# them used 850 runner-minutes (26% of this workflow).
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
@@ -121,3 +121,57 @@ jobs:
         env:
           RUN_SMOKE_TESTS: "0"
           POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  PolicyEngineUSMain:
+    name: Test policyengine-us main against this Core
+    runs-on: ubuntu-latest
+    needs: Lint
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    env:
+      PYTHONUNBUFFERED: 1
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+        with:
+          path: core
+          persist-credentials: false
+      - name: Checkout policyengine-us main
+        uses: actions/checkout@v6
+        with:
+          repository: PolicyEngine/policyengine-us
+          ref: main
+          path: policyengine-us
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+      # As policyengine-us's own CI does: its lock with the newest Core release.
+      - name: Install policyengine-us main with the Core release
+        working-directory: policyengine-us
+        run: |
+          uv lock --upgrade-package policyengine-core
+          uv sync --locked --extra dev
+          bash ./update_itemization.sh
+      - name: Select the policyengine-us tests
+        run: python core/.github/country_main_tests.py select core/.github/requirements/policyengine-us-main-tests.txt policyengine-us > selected.txt
+      # Both runs draw the same Hypothesis examples, so a difference between
+      # them comes from Core. A failing run still writes its report.
+      - name: Run them with the Core release
+        working-directory: policyengine-us
+        run: |
+          uv pip list | grep -i '^policyengine-core '
+          xargs uv run --no-sync python -m pytest -p no:cacheprovider --hypothesis-seed=0 --junitxml=../released.xml < ../selected.txt || true
+      - name: Install this Core
+        working-directory: policyengine-us
+        run: |
+          uv pip install --python .venv/bin/python --reinstall-package policyengine-core ../core
+          uv run --no-sync python -c "import json; from importlib.metadata import distribution; url = json.loads(distribution('policyengine-core').read_text('direct_url.json'))['url']; print(url); assert url.startswith('file://'), url"
+      - name: Run them with this Core
+        working-directory: policyengine-us
+        run: xargs uv run --no-sync python -m pytest -p no:cacheprovider --hypothesis-seed=0 --junitxml=../candidate.xml < ../selected.txt || true
+      - name: Fail on tests this Core breaks
+        run: python core/.github/country_main_tests.py compare released.xml candidate.xml

changelog.d/test-policyengine-us-main.changed.md

@@ -0,0 +1 @@
+Add a PR job that runs policyengine-us main's Core-facing tests with the Core release and then with the Core under test, and fails on tests the change breaks.