ci: gate PyAutoHands's own test suite on pull_request

.github/workflows/tests.yml

@@ -0,0 +1,83 @@
+name: Hands Tests
+
+# PyAutoHands's own unit suite — the executor's self-test. Until this existed,
+# PyAutoHands PRs carried ZERO check runs: `python_matrix.yml` is a weekly cron
+# over the five *libraries'* suites, `navigator_check.yml` is `workflow_call`
+# only (invoked by the workspaces), and `release.yml`'s pytest step runs inside
+# `${{ matrix.project.path }}` — a matrix of the five libraries, with
+# PyAutoHands checked out beside them only as a helper. So the ~300 tests
+# covering build_util (script/notebook execution, per-script timeouts,
+# clean-skip exits), env_config (profile discovery, per-script env, JAX
+# marking, workspace precedence), result_collector, check_navigator,
+# clone_seed, the release/Slack notes and the workflow parsers ran in no CI at
+# all, and a Hands PR's only gate was whatever the authoring session ran
+# locally.
+#
+# That gap had already bitten: b038fdc promoted Python 3.14 to a required
+# matrix leg and retired the experimental_python_314 job, but left
+# test_python_matrix_workflow.py asserting the pre-promotion contract. The
+# guard test sat failing with nothing to report it.
+#
+# Deliberately ONLY pytest. It must not invoke `autohands generate` / `run_all`
+# / `pre_build` against live workspaces, and must not reach the network — those
+# need sibling workspace checkouts and belong to release.yml and the scheduled
+# drivers, not to a PR gate. What runs here is stdlib plus four small packages,
+# so it stays fast (~6s) and flake-free.
+#
+# Dependencies are named explicitly rather than `-r requirements.txt`, which
+# would drag in jupyterlab + ipykernel and turn a 6s gate into a slow one:
+#   pytest            the runner
+#   PyYAML            env_config / validate_env_profiles / the workflow parsers
+#   ipynb-py-convert  a CLI binary build_util shells out to (not an import)
+#   Pillow            generate_markdown's PNG optimisation path
+# FOUR tests self-skip here, and they are two different kinds:
+#   - 3 on absent nbformat/jupyter (test_run_notebook_cwd,
+#     test_notebook_skip_exit) — intentional, that is the notebook-execution
+#     path release.yml covers.
+#   - 1 because this checkout has no sibling workspaces:
+#     test_workspace_config_precedence.test_actual_workspace_files_exist walks
+#     `repo_root.parent / <workspace>` asserting each of the six workspaces owns
+#     its config/build/{no_run,profile_smoke,visualise_notebooks}.yaml, and
+#     skips at the first one absent. In a full local workspace it asserts all
+#     six; here it asserts nothing.
+# That second one is a real hole in this gate, accepted deliberately: closing it
+# means six extra checkouts for a repo-layout invariant, which would couple a 6s
+# gate to six other repos. It is a local/developer check, not a PR check. Local
+# baseline is therefore 301 passed / 3 skipped; CI is 300 passed / 4 skipped.
+
+# One run per commit: PR events carry the CI; pushes only build main.
+# Superseded runs are cancelled on PR refs only — a cancelled main run would
+# read as red CI (cancelled is in Heart's FAILURE_CONCLUSIONS).
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: hands-tests-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+permissions:
+  contents: read
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Tracks the required set python_matrix.yml declares supported (3.14
+        # promoted in b038fdc). test_python_matrix_workflow.py asserts the two
+        # stay equal, so this list cannot silently fall behind that policy.
+        python-version: ["3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install (the whole dependency set — see header)
+        run: pip install pytest PyYAML ipynb-py-convert Pillow
+      - name: Run tests
+        run: pytest tests/ -q

tests/test_python_matrix_workflow.py

@@ -6,44 +6,65 @@
 WORKFLOW = (
     Path(__file__).resolve().parents[1] / ".github" / "workflows" / "python_matrix.yml"
 )
+SELF_TEST_WORKFLOW = (
+    Path(__file__).resolve().parents[1] / ".github" / "workflows" / "tests.yml"
+)
 AUTOHANDS = Path(__file__).resolve().parents[1] / "bin" / "autohands"
 
 
 def load_workflow():
     return yaml.safe_load(WORKFLOW.read_text())
 
 
+def test_self_test_gate_tracks_the_supported_python_set():
+    """Hands's own gate (tests.yml) must run the same Python set python_matrix.yml
+    declares required.
+
+    Drift between a version policy and the file guarding it is exactly how this
+    module went stale: b038fdc promoted 3.14 in python_matrix.yml and nothing
+    reported that the guard still asserted the old shape. Tying the two lists
+    together means promoting or dropping a version has to touch both.
+    """
+    required = load_workflow()["jobs"]["unit_tests"]["strategy"]["matrix"][
+        "python-version"
+    ]
+    gate = yaml.safe_load(SELF_TEST_WORKFLOW.read_text())["jobs"]["pytest"][
+        "strategy"
+    ]["matrix"]["python-version"]
+
+    assert gate == required
+
+
 def test_required_matrices_cover_only_supported_python_versions():
     jobs = load_workflow()["jobs"]
 
     assert jobs["unit_tests"]["strategy"]["matrix"]["python-version"] == [
         "3.12",
         "3.13",
+        "3.14",
     ]
     assert jobs["smoke_tests"]["strategy"]["matrix"]["python-version"] == [
         "3.12",
         "3.13",
+        "3.14",
     ]
 
 
-def test_python_314_is_isolated_and_non_required():
+def test_python_314_is_a_required_leg_not_an_isolated_experiment():
+    """3.14 was promoted to a required leg of both matrices (b038fdc, following
+    the PyAutoFit#1439 forkserver fix), which retired the soft
+    `experimental_python_314` job. Guard the promoted shape: 3.14 must sit in
+    the required matrices above, and must not quietly regrow a
+    `continue-on-error` home where its failures stop counting."""
     jobs = load_workflow()["jobs"]
-    experimental = jobs["experimental_python_314"]
-
-    assert experimental["continue-on-error"] is True
-    assert experimental["strategy"]["matrix"]["python-version"] == ["3.14"]
-    assert len(experimental["strategy"]["matrix"]["project"]) == 5
-    assert "experimental_python_314" in jobs["summary"]["needs"]
-    assert "continue-on-error" not in jobs["unit_tests"]
-    assert "continue-on-error" not in jobs["smoke_tests"]
-
-    record_step = next(
-        step for step in experimental["steps"]
-        if step.get("name") == "Record experimental cell result"
-    )
-    assert record_step["if"] == "always()"
-    assert "job.status" in record_step["run"]
-    assert "does not cover workspace scripts" in record_step["run"]
+
+    assert "experimental_python_314" not in jobs
+
+    for name in ("unit_tests", "smoke_tests"):
+        assert "3.14" in jobs[name]["strategy"]["matrix"]["python-version"]
+        assert "continue-on-error" not in jobs[name]
+
+    assert sorted(jobs["summary"]["needs"]) == ["smoke_tests", "unit_tests"]
 
 
 def test_no_below_floor_success_or_banner_contract_remains():