diff --git a/agents/conductors/hygiene/AGENTS.md b/agents/conductors/hygiene/AGENTS.md index d497d39..2abdbb6 100644 --- a/agents/conductors/hygiene/AGENTS.md +++ b/agents/conductors/hygiene/AGENTS.md @@ -26,7 +26,7 @@ kinds, which is what makes its count comparable (or not): - **debris** — finds directly-removable items; a real, rankable count (`tidy`). - **finding** — confirms a source-quality defect; a real, rankable count - (`docstrings`, `refs`). + (`docstrings`, `refs`, `optdeps`, `extras`). - **timing** — measures import cost; a real, rankable count of *slow* imports (`perf`). - **surface** — only *sizes* the audit; the real problems emerge when the delegated skill runs, so the count is **not** a problem count (`deps`, `docs`). @@ -43,6 +43,8 @@ kinds, which is what makes its count comparable (or not): | `crlf` | executable scripts (`.sh` + shebang-`755` `.py`) with CRLF — the shebang breaks on Linux/HPC (**debris**, the ranked count); library `.py` CRLF is reported separately as *cosmetic* (Python reads it fine — don't mass-normalise) | `/refactor` + `.gitattributes eol=lf` | | `docstrings` | consecutive module-level triple-quoted expressions separated only by whitespace in user-facing `*_workspace` and `HowTo*` root `*.py` entry scripts and `scripts/**/*.py` files (**finding**) | `/refactor` (mechanically merge each confirmed boundary) | | `refs` | file/folder references in user-facing `*_workspace` and `HowTo*` prose (`scripts/**/*.py` docstrings + comments, every `scripts/**/README.md` and `config/**/README.md`, and the top-level README) whose target no longer exists — restructure debt no health sweep can see, since the scripts still run (**finding**). Covers the README idioms a `scripts/`-anchored matcher cannot see: structure-list bullets (``- `slam_pipeline`: ``), slash-less relative folder paths (`data_preparation/imaging`), and config YAML names | `/refactor` (re-point each reference; judge the intended target) | +| `optdeps` | smoke-listed workspace scripts that construct an optional-dependency-gated API (`TransformerNUFFT` → `nufftax`) without the house `find_spec` skip guard, so they hard-fail the CI matrices that omit the extras (**finding**). AST-confirmed — prose mentions don't count; scripts outside `smoke_tests.txt` are never flagged | `/refactor` (add the skip guard) | +| `extras` | the complement of `optdeps`: an optional dependency a library **declares** (in the `[optional]` extra `mode=release` installs) that the `workspace-validation.yml` **`mode=smoke`** leg never installs (**finding**). The extras chain only reaches each library's own `[jax]`, never a sibling's `[optional]`, so those need hand-adding and silently drift — the symptom is a script red in smoke and **green in release** | `/bug` (add the install; fix the install set, **never** the script) | | `config` | library `config/*.yaml` keys missing from the matching workspace config — recursive diff (**surface**) | `/refactor` (mirror keys) | | `artifacts` | tracked files that look like leaked run outputs / stray data (under `output/`, or data-ext outside fixtures) (**debris**) | `/repo_cleanup` (gitignore + `git rm --cached`) | | `packaging` | ignored, fully-untracked top-level `*.egg-info/` and `build/` directories in managed library repos (**debris**) | preview then run `PyAutoBrain/bin/clean_slate.sh --packaging`; repo-set, exact-name, root-depth and tracked-file guards apply | @@ -60,6 +62,8 @@ pyauto-brain hygiene docs # API-docs surface → /audit_docs pyauto-brain hygiene crlf # CRLF .py files → /refactor pyauto-brain hygiene docstrings # adjacent top-level documentation → /refactor pyauto-brain hygiene refs # dead internal references in workspace prose → /refactor +pyauto-brain hygiene optdeps # smoke-listed scripts missing an optional-dep skip guard → /refactor +pyauto-brain hygiene extras # optional deps the smoke CI leg never installs → /bug pyauto-brain hygiene config # library→workspace config drift → /refactor pyauto-brain hygiene artifacts # tracked leaked outputs/data → /repo_cleanup pyauto-brain hygiene packaging # ignored root packaging dirs → clean_slate.sh diff --git a/agents/conductors/hygiene/_hygiene_extras.py b/agents/conductors/hygiene/_hygiene_extras.py new file mode 100644 index 0000000..359d900 --- /dev/null +++ b/agents/conductors/hygiene/_hygiene_extras.py @@ -0,0 +1,329 @@ +""" +Hygiene pre-scan: optional dependencies a PyAuto library declares that the +workspace-validation SMOKE leg never installs. + +``workspace-validation.yml`` runs the same script matrix in both modes — only +the install differs. ``mode=release`` installs every library's ``[optional]`` +extra explicitly, so a script needing an optional package works. ``mode=smoke`` +installs the published ``autolens[optional]`` and leans on the extras chain, +but that chain only ever reaches each library's own ``[jax]`` extra, never a +sibling's ``[optional]``:: + + autolens[optional] -> autolens[jax] -> autogalaxy[jax] + -> autofit[jax] -> autonerves[jax] + +So anything declared in a *sibling's* ``[optional]`` — ``autoarray[optional]``, +``autofit[optional]`` — has to be hand-added to the smoke leg, and nothing +enforces that. The gap is invisible until a nightly smoke run goes red against +a script that passes release validation, and the failure looks like a broken +script rather than a missing install (2026-08-03: ``tfp-nightly``, needed by +the JAX Matern-kernel regularization path, was in ``autoarray[optional]`` only; +two scripts red in smoke, both green in release). + +The remedy is always the install set, never the script: a script that passes +release validation is correct, and parking or skip-guarding it destroys the +coverage it exists to provide. This scan therefore reports MISSING INSTALLS, +and is deliberately the complement of ``optdeps``, which reports scripts that +genuinely should carry a skip guard. + +Resolution is modelled the way pip resolves it: from the smoke leg's declared +roots, follow each PyAuto library's base ``dependencies`` and any requested +extra, and collect every third-party distribution reached. Expected coverage is +the union of every library's ``[optional]`` closure — the set ``mode=release`` +guarantees. The difference is the drift. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import tomllib +from pathlib import Path + +# The source libraries whose extras the workspace matrices install. Mirrors +# hygiene.sh's LIB_REPOS. +LIB_REPOS = ("PyAutoNerves", "PyAutoArray", "PyAutoFit", "PyAutoGalaxy", "PyAutoLens") + +# The extra mode=release installs for every library, and therefore the coverage +# mode=smoke is expected to match. +RELEASE_EXTRA = "optional" + +WORKFLOW = Path("PyAutoHeart/.github/workflows/workspace-validation.yml") + +# The install step whose requirement roots define the smoke leg's coverage. +SMOKE_STEP = re.compile(r"^\s*-\s*name:.*\[mode=smoke\]", re.IGNORECASE) +NEXT_STEP = re.compile(r"^\s*-\s*name:") +PIP_INSTALL = re.compile(r"\bpip\s+install\b(?P.*)$") +# A requirement's distribution name and its optional extras: "autoarray[optional]", +# "nufftax>=0.6.1,<0.7.0", "tfp-nightly==0.26.0.dev20260713". +REQUIREMENT = re.compile(r"^(?P[A-Za-z0-9][A-Za-z0-9._-]*)(?:\[(?P[^\]]*)\])?") + + +def canonical(name: str) -> str: + """PEP 503 normalisation — `tfp_nightly`, `TFP.Nightly` and `tfp-nightly` are one.""" + return re.sub(r"[-_.]+", "-", name).strip().lower() + + +def parse_requirement(token: str) -> tuple[str, tuple[str, ...]] | None: + """('autoarray[optional]') -> ('autoarray', ('optional',)); None if not a requirement.""" + match = REQUIREMENT.match(token.strip().strip("\"'")) + if not match: + return None + raw = match.group("extras") or "" + extras = tuple(sorted({canonical(e) for e in raw.split(",") if e.strip()})) + return canonical(match.group("name")), extras + + +def libraries(root: Path) -> dict[str, dict]: + """Map canonical distribution name -> its parsed pyproject, for each checkout.""" + found: dict[str, dict] = {} + for repo in LIB_REPOS: + pyproject = root / repo / "pyproject.toml" + if not pyproject.exists(): + continue + try: + data = tomllib.loads(pyproject.read_text()) + except (tomllib.TOMLDecodeError, UnicodeDecodeError): + continue # a malformed pyproject is the packaging mode's problem + name = data.get("project", {}).get("name") + if name: + found[canonical(name)] = data + return found + + +def smoke_roots(root: Path) -> list[tuple[str, tuple[str, ...]]]: + """Requirement roots the smoke install step passes to pip, in order.""" + workflow = root / WORKFLOW + if not workflow.exists(): + return [] + + lines = workflow.read_text().splitlines() + block: list[str] = [] + inside = False + for line in lines: + if SMOKE_STEP.match(line): + inside = True + continue + if inside and NEXT_STEP.match(line): + inside = False + continue + if inside: + block.append(line) + + # Rejoin backslash continuations so a multi-line `pip install a \\\n b` is + # read as one command (the release leg is written that way). + joined: list[str] = [] + buffer = "" + for line in block: + stripped = line.strip() + if stripped.startswith("#"): + continue + if stripped.endswith("\\"): + buffer += stripped[:-1] + " " + continue + joined.append(buffer + stripped) + buffer = "" + if buffer: + joined.append(buffer) + + roots: list[tuple[str, tuple[str, ...]]] = [] + for line in joined: + match = PIP_INSTALL.search(line) + if not match: + continue + for token in match.group("rest").split(): + if token.startswith("-"): + continue # a flag (--index-url, --no-deps, ...) + requirement = parse_requirement(token) + if requirement: + roots.append(requirement) + return roots + + +def closure( + roots: list[tuple[str, tuple[str, ...]]], libs: dict[str, dict] +) -> set[str]: + """Third-party distributions pip would install from `roots`. + + A PyAuto library contributes its base `dependencies` (always installed) plus + the requirement list of each extra actually requested; everything else is a + third-party distribution and is recorded. + """ + reached: set[str] = set() + based: set[str] = set() + seen_extras: set[tuple[str, str]] = set() + queue = list(roots) + + while queue: + name, extras = queue.pop() + + if name not in libs: + reached.add(name) + continue + + project = libs[name].get("project", {}) + if name not in based: + based.add(name) + for requirement in project.get("dependencies", []) or []: + parsed = parse_requirement(requirement) + if parsed: + queue.append(parsed) + + declared = project.get("optional-dependencies", {}) or {} + # Extra names normalise the same way distribution names do (PEP 685). + by_extra = {canonical(key): value for key, value in declared.items()} + for extra in extras: + if (name, extra) in seen_extras: + continue + seen_extras.add((name, extra)) + for requirement in by_extra.get(extra, []) or []: + parsed = parse_requirement(requirement) + if parsed: + queue.append(parsed) + + return reached + + +def missing(root: Path) -> tuple[list[dict], str | None]: + """Return (findings, skip-reason). A skip-reason means nothing was scannable.""" + libs = libraries(root) + if not libs: + return [], "no library checkouts under the scan root" + + roots = smoke_roots(root) + if not roots: + return [], f"no smoke install step found in {WORKFLOW}" + + reached = closure(roots, libs) + + findings: list[dict] = [] + for repo in LIB_REPOS: + pyproject = root / repo / "pyproject.toml" + if not pyproject.exists(): + continue + # Resolve by the declared project name rather than the folder name. + try: + name = canonical(tomllib.loads(pyproject.read_text())["project"]["name"]) + except (tomllib.TOMLDecodeError, UnicodeDecodeError, KeyError): + continue + data = libs.get(name) + if data is None: + continue + if RELEASE_EXTRA not in { + canonical(key) + for key in (data.get("project", {}).get("optional-dependencies", {}) or {}) + }: + continue + + for dist in sorted(closure([(name, (RELEASE_EXTRA,))], libs) - reached): + findings.append( + { + "dependency": dist, + "declared_by": f"{name}[{RELEASE_EXTRA}]", + "repo": repo, + } + ) + + # A package can be declared optional by more than one library; report the + # missing install once, naming every declarer. + merged: dict[str, dict] = {} + for finding in findings: + entry = merged.setdefault( + finding["dependency"], + {"dependency": finding["dependency"], "declared_by": [], "repos": []}, + ) + entry["declared_by"].append(finding["declared_by"]) + entry["repos"].append(finding["repo"]) + + return [merged[key] for key in sorted(merged)], None + + +def summarise(findings: list[dict], skipped: str | None) -> str: + if skipped: + return f"0|not scannable here: {skipped}" + if not findings: + return ( + "0|clean: the smoke install set reaches every dependency the " + "libraries declare optional" + ) + detail = ", ".join( + f"{f['dependency']} ({'/'.join(f['declared_by'])})" for f in findings + ) + subject = "dependency is" if len(findings) == 1 else "dependencies are" + return ( + f"{len(findings)}|{len(findings)} optional {subject} declared by a library " + f"but never installed by the workspace-validation smoke leg ({detail}) — " + f"red in smoke, green in release" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + output = parser.add_mutually_exclusive_group() + output.add_argument("--json-row", action="store_true") + output.add_argument("--summary", action="store_true") + args = parser.parse_args() + + findings, skipped = missing(args.root) + + if args.summary: + print(summarise(findings, skipped)) + return 0 + + if args.json_row: + # Must carry the same envelope as every other mode's row — the default + # --json scan keys the rows by "mode", so omitting it breaks the whole + # decision document, not just this row. + print( + json.dumps( + { + "count": len(findings), + "delegate": "/bug", + "findings": findings, + "kind": "finding", + "mode": "extras", + "status": "clean" if not findings else "finding", + "summary": summarise(findings, skipped).split("|", 1)[1], + }, + sort_keys=True, + ) + ) + return 0 + + if skipped: + print(f"Optional-dependency exposure not scannable here: {skipped}.") + return 0 + + if not findings: + print( + "No exposure drift: every dependency the libraries declare optional is " + "reachable from the workspace-validation smoke install set." + ) + return 0 + + print( + f"{len(findings)} optional dependency(ies) the smoke leg never installs:\n" + ) + for finding in findings: + print(f" {finding['dependency']}") + print( + f" declared by {', '.join(finding['declared_by'])} " + f"({', '.join(finding['repos'])}); installed in mode=release, " + f"absent in mode=smoke" + ) + print( + f"\nAdd each to the smoke install step in {WORKFLOW}, mirroring the " + "`autofit[optional]` line already there — prefer installing the declaring\n" + "library's [optional] extra over pinning the single package, so a future\n" + "addition to that extra is covered too.\n\n" + "Do NOT skip-guard or park the failing script: it passes mode=release, so " + "the script is correct and the install set is the defect. Route to /bug." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/agents/conductors/hygiene/hygiene.sh b/agents/conductors/hygiene/hygiene.sh index cb0b662..8d077d7 100755 --- a/agents/conductors/hygiene/hygiene.sh +++ b/agents/conductors/hygiene/hygiene.sh @@ -18,6 +18,7 @@ # docstrings -> /refactor (exact findings; Hygiene remains read-only) # refs -> /refactor (dead internal references in workspace prose) # optdeps -> /refactor (smoke-listed scripts missing an optional-dep skip guard) +# extras -> /bug (optional deps a library declares that the smoke CI leg never installs) # The three Heart skills are read-only observation skills — measurement lives in # Heart; hygiene routes and prioritises. perf's import timing runs in a # SUBPROCESS, so the conductor itself never imports the science/JAX stack. @@ -35,6 +36,7 @@ # hygiene.sh docstrings # adjacent top-level script documentation -> /refactor # hygiene.sh refs # dead internal references in workspace prose -> /refactor # hygiene.sh optdeps # smoke-listed scripts w/ a gated API but no skip guard -> /refactor +# hygiene.sh extras # optional deps declared by a library but missing from the smoke CI install -> /bug # hygiene.sh config # library config keys missing downstream -> /refactor # hygiene.sh artifacts # tracked leaked outputs/data -> /repo_cleanup # hygiene.sh packaging # ignored top-level *.egg-info/build dirs -> clean_slate.sh @@ -81,7 +83,7 @@ PERF_PY="${HYGIENE_PYTHON:-python3}" PERF_THRESHOLD="${HYGIENE_PERF_THRESHOLD:-3.0}" read -r -a PERF_LIBS <<< "${HYGIENE_PERF_LIBS:-autoconf autofit autoarray autogalaxy autolens}" -MODE_ORDER=(perf tidy crlf docstrings refs optdeps artifacts packaging noise deps docs config) +MODE_ORDER=(perf tidy crlf docstrings refs optdeps extras artifacts packaging noise deps docs config) declare -A MODE_DELEGATE=( [perf]="/refactor" [tidy]="condemn → condemned.md (async; 'hygiene sweep' voids)" @@ -89,6 +91,7 @@ declare -A MODE_DELEGATE=( [docstrings]="/refactor" [refs]="/refactor" [optdeps]="/refactor" + [extras]="/bug" [artifacts]="/repo_cleanup" [packaging]="PyAutoBrain/bin/clean_slate.sh --packaging" [noise]="/cli_noise_clean" @@ -105,7 +108,7 @@ declare -A MODE_DELEGATE=( # 'finding', and 'timing' counts drive the ranking. declare -A MODE_KIND=( [perf]="timing" [tidy]="debris" [crlf]="debris" [artifacts]="debris" [packaging]="debris" - [docstrings]="finding" [refs]="finding" [optdeps]="finding" + [docstrings]="finding" [refs]="finding" [optdeps]="finding" [extras]="finding" [deps]="surface" [docs]="surface" [config]="surface" [noise]="advisory" ) @@ -205,6 +208,16 @@ prescan_optdeps() { python3 "$HERE/_hygiene_optdeps.py" --root "$ROOT" --summary } +# extras: the complement of optdeps — an optional dependency a library DECLARES +# (in its [optional] extra, which mode=release installs) that the +# workspace-validation mode=smoke leg never installs. The extras chain reaches +# each library's own [jax], never a sibling's [optional], so those have to be +# hand-added and silently drift. The symptom is a script red in smoke and green +# in release; the fix is always the install set, never the script. +prescan_extras() { + python3 "$HERE/_hygiene_extras.py" --root "$ROOT" --summary +} + # refs: file/folder references in user-facing *_workspace and HowTo* prose # (script docstrings/comments + the top-level README) whose target no longer # exists. A restructure moves the target and the prose keeps the old name — the @@ -327,7 +340,7 @@ prescan() { perf) prescan_perf ;; tidy) prescan_tidy ;; deps) prescan_deps ;; docs) prescan_docs ;; noise) prescan_noise ;; crlf) prescan_crlf ;; docstrings) prescan_docstrings ;; refs) prescan_refs ;; - optdeps) prescan_optdeps ;; + optdeps) prescan_optdeps ;; extras) prescan_extras ;; artifacts) prescan_artifacts ;; packaging) prescan_packaging ;; config) prescan_config ;; esac @@ -339,7 +352,7 @@ mode="default"; json=0; profile_script=""; expect_script=0 for arg in "$@"; do if [[ "$expect_script" -eq 1 ]]; then profile_script="$arg"; expect_script=0; continue; fi case "$arg" in - perf|tidy|sweep|noise|deps|docs|crlf|docstrings|refs|optdeps|config|artifacts|packaging) mode="$arg" ;; + perf|tidy|sweep|noise|deps|docs|crlf|docstrings|refs|optdeps|extras|config|artifacts|packaging) mode="$arg" ;; default) mode="default" ;; --json) json=1 ;; --profile) mode="perf"; expect_script=1 ;; @@ -525,6 +538,10 @@ emit_json_row() { # mode python3 "$HERE/_hygiene_optdeps.py" --root "$ROOT" --json-row return fi + if [[ "$m" == "extras" ]]; then + python3 "$HERE/_hygiene_extras.py" --root "$ROOT" --json-row + return + fi local res count summary kind status res="$(prescan "$m")"; count="${res%%|*}"; summary="${res#*|}"; kind="${MODE_KIND[$m]}" if [[ "$kind" == "advisory" || "$count" == "-1" ]]; then status="advisory" @@ -564,6 +581,10 @@ render_delegate_line() { # mode printf ' %-9s %-9s → route exact findings to /refactor; Hygiene never edits source\n' "" "" return fi + if [[ "$m" == "extras" ]]; then + printf ' %-9s %-9s → route the missing installs to /bug; fix the CI install set, never the script\n' "" "" + return + fi if [[ "${MODE_KIND[$m]}" == "timing" ]]; then printf ' %-9s %-9s → route slow items to %s; slow tests/scripts → Heart script_timing/test_run\n' "" "" "${MODE_DELEGATE[$m]}" else @@ -606,12 +627,15 @@ elif [[ "$mode" == "refs" ]]; then elif [[ "$mode" == "optdeps" ]]; then echo "Smoke-listed scripts missing an optional-dependency skip guard (read-only scan):" python3 "$HERE/_hygiene_optdeps.py" --root "$ROOT" +elif [[ "$mode" == "extras" ]]; then + echo "Optional dependencies the workspace-validation smoke leg never installs (read-only scan):" + python3 "$HERE/_hygiene_extras.py" --root "$ROOT" elif [[ "$mode" == "default" ]]; then # 'debris' and 'finding' pre-scans yield directly-actionable counts (perf's # timing is deferred here — too slow for the fast scan). Rank across them and # recommend the mode with the largest confirmed workload. best=""; best_n=0 - for m in tidy crlf docstrings refs optdeps artifacts packaging; do + for m in tidy crlf docstrings refs optdeps extras artifacts packaging; do local_n="$(prescan "$m")"; local_n="${local_n%%|*}" if [[ "$local_n" -gt "$best_n" ]]; then best_n="$local_n"; best="$m"; fi done diff --git a/skills/hygiene/hygiene.md b/skills/hygiene/hygiene.md index aa1d51d..f166304 100644 --- a/skills/hygiene/hygiene.md +++ b/skills/hygiene/hygiene.md @@ -10,7 +10,7 @@ Shared routing context: `PyAutoBrain/skills/COMMANDS.md`. ## Do 1. Run `bin/pyauto-brain hygiene [perf | tidy | noise | deps | docs | crlf | - docstrings | refs | config | artifacts | packaging]` (no arg = pre-scan across modes → a ranked worklist; + docstrings | refs | optdeps | extras | config | artifacts | packaging]` (no arg = pre-scan across modes → a ranked worklist; perf's import timing is deferred there). This is a **dry run** — each mode does a cheap read-only pre-scan and emits a `HygieneDecision` naming the skill to run for the full audit. Nothing is executed or mutated. (`crlf` = @@ -24,7 +24,11 @@ Shared routing context: `PyAutoBrain/skills/COMMANDS.md`. paths quoted in workspace and HowTo prose whose target no longer exists after a restructure, across `scripts/**/*.py`, every nested `scripts/**/README.md` and `config/**/README.md`, and the top-level README, - including folder-structure bullet lists and slash-less relative paths.) + including folder-structure bullet lists and slash-less relative paths; + `optdeps` = smoke-listed scripts that build an optional-dependency-gated API + with no `find_spec` skip guard; `extras` = its complement, an optional + dependency a library declares in the `[optional]` extra that the + workspace-validation **smoke** leg never installs.) 2. Execute the emitted plan: run the named delegate — `/repo_cleanup` (git debris), `/cli_noise_clean`, `/dep_audit`, `/audit_docs` — for the full audit, or for `perf` route slow imports/functions to `/refactor` / `/bug` (JAX-adapt @@ -34,7 +38,13 @@ Shared routing context: `PyAutoBrain/skills/COMMANDS.md`. exact reported findings to `/refactor`; the Hygiene scan remains read-only. A `refs` finding is the reference **as written** — judge the intended target (a moved file, a file that became a directory, a reference meant for a - sibling repo) before re-pointing it. + sibling repo) before re-pointing it. Route `optdeps` findings to `/refactor` + too (add the skip guard), but route `extras` findings to `/bug`: a script + that fails only in `mode=smoke` and **passes `mode=release`** is correct, so + the defect is the CI install set — add the missing install (prefer the + declaring library's whole `[optional]` extra). Never "fix" such a script by + skip-guarding or parking it; both silently delete coverage that release + validation still depends on. Source changes ship via `ship_library` / `ship_workspace`. The Hygiene Agent **reasons; it never edits source and never mutates a repo.** diff --git a/tests/test_hygiene_conductor.py b/tests/test_hygiene_conductor.py index e4e7f09..206833f 100644 --- a/tests/test_hygiene_conductor.py +++ b/tests/test_hygiene_conductor.py @@ -15,7 +15,7 @@ BRAIN = BRAIN_HOME / "bin" / "pyauto-brain" MODES = { "perf", "tidy", "noise", "deps", "docs", "crlf", "config", "artifacts", - "packaging", "docstrings", "refs", "optdeps", + "packaging", "docstrings", "refs", "optdeps", "extras", } _PROFILE_TARGET = """ @@ -65,6 +65,7 @@ def test_default_json_is_a_hygiene_decision_with_all_modes(tmp_path): assert kinds["docstrings"] == "finding" assert kinds["refs"] == "finding" assert kinds["optdeps"] == "finding" + assert kinds["extras"] == "finding" assert kinds["deps"] == "surface" and kinds["docs"] == "surface" assert kinds["config"] == "surface" assert kinds["noise"] == "advisory" @@ -505,6 +506,141 @@ def test_optdeps_findings_reach_the_default_worklist(tmp_path): assert "route exact findings to /refactor" in result.stdout +_SMOKE_WORKFLOW = """\ +jobs: + run_scripts: + steps: + - name: "Install third-party deps (libs run from source) [mode=smoke]" + if: needs.find_scripts.outputs.mode == 'smoke' + run: | + pip install "autolens[optional]" +{extra_installs}\ + - name: "Install TestPyPI wheels [mode=release]" + if: needs.find_scripts.outputs.mode == 'release' + run: | + pip install \\ + "autoarray[optional]==$V" \\ + "autolens[optional]==$V" +""" + +_PYPROJECTS = { + # autonerves is the base layer; its [jax] extra is what the chain reaches. + "PyAutoNerves": """\ +[project] +name = "autonerves" +dependencies = [] +[project.optional-dependencies] +jax = ["jax>=0.7"] +optional = ["autonerves[jax]", "astropy>=5.0"] +""", + # autoarray declares an optional dep NO sibling's chain reaches — the drift. + "PyAutoArray": """\ +[project] +name = "autoarray" +dependencies = ["autonerves"] +[project.optional-dependencies] +jax = ["autonerves[jax]"] +optional = ["autoarray[jax]", "numba", "tfp-nightly==0.26.0.dev1"] +""", + # autolens[optional] chains to autolens[jax] -> autonerves[jax]; it never + # reaches autoarray[optional], which is the whole point of the scan. + "PyAutoLens": """\ +[project] +name = "autolens" +dependencies = ["autoarray", "autonerves"] +[project.optional-dependencies] +jax = ["autonerves[jax]"] +optional = ["autolens[jax]", "numba", "astropy>=5.0"] +""", +} + + +def _write_extras_fixture(root, extra_installs=""): + """Library checkouts + a PyAutoHeart workflow whose smoke leg under-installs. + + `numba` is reachable from `autolens[optional]`. `astropy` is declared + optional by TWO libraries — autonerves (not reached) and autolens (reached) + — so it must NOT be flagged; only a dependency no reached extra supplies is + drift. `tfp-nightly` is declared ONLY by `autoarray[optional]`, which the + chain never reaches -> FLAG. + """ + for repo, body in _PYPROJECTS.items(): + (root / repo).mkdir(parents=True, exist_ok=True) + (root / repo / "pyproject.toml").write_text(body) + + workflow = root / "PyAutoHeart" / ".github" / "workflows" + workflow.mkdir(parents=True, exist_ok=True) + (workflow / "workspace-validation.yml").write_text( + _SMOKE_WORKFLOW.format(extra_installs=extra_installs) + ) + return root + + +def test_extras_flags_an_optional_dep_the_smoke_leg_never_installs(tmp_path): + _write_extras_fixture(tmp_path) + + result = _run(["extras", "--json"], tmp_path) + + assert result.returncode == 0, result.stderr + row = json.loads(result.stdout)["row"] + assert row["count"] == 1 + assert row["mode"] == "extras" and row["kind"] == "finding" + assert row["delegate"] == "/bug" + finding = row["findings"][0] + assert finding["dependency"] == "tfp-nightly" + assert finding["declared_by"] == ["autoarray[optional]"] + + +def test_extras_is_clean_once_the_declaring_extra_is_installed(tmp_path): + # The house fix: install the declaring library's whole [optional] extra. + _write_extras_fixture( + tmp_path, extra_installs=' pip install "autoarray[optional]"\n' + ) + + result = _run(["extras", "--json"], tmp_path) + + assert result.returncode == 0, result.stderr + row = json.loads(result.stdout)["row"] + assert row["count"] == 0 and row["status"] == "clean" + + +def test_extras_is_clean_when_the_single_package_is_pinned_directly(tmp_path): + # Pinning the one package also closes it (it just does not self-heal). + _write_extras_fixture( + tmp_path, + extra_installs=' pip install "tfp-nightly==0.26.0.dev1"\n', + ) + + result = _run(["extras", "--json"], tmp_path) + + row = json.loads(result.stdout)["row"] + assert row["count"] == 0 and row["status"] == "clean" + + +def test_extras_reports_not_scannable_without_the_workflow(tmp_path): + # Library checkouts but no PyAutoHeart workflow: report nothing rather than + # inventing findings — an absent workflow is not exposure drift. + for repo, body in _PYPROJECTS.items(): + (tmp_path / repo).mkdir(parents=True, exist_ok=True) + (tmp_path / repo / "pyproject.toml").write_text(body) + + result = _run(["extras", "--json"], tmp_path) + + row = json.loads(result.stdout)["row"] + assert row["count"] == 0 and row["status"] == "clean" + assert "not scannable" in row["summary"] + + +def test_extras_findings_reach_the_default_worklist(tmp_path): + _write_extras_fixture(tmp_path) + + result = _run([], tmp_path) + + assert result.returncode == 0, result.stderr + assert "extras 1 findings" in result.stdout + assert "route the missing installs to /bug" in result.stdout + + def _write_refs_fixture(root): """A workspace whose READMEs drift from its real tree.