From ef86cebdd8d23acbd660ea74b2978c78c6641b6d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 15:18:41 +0000 Subject: [PATCH] intake: re-rank reconcile so its suspects are worth reading Leg 2 of PyAutoMind draft/feature/pyautomind/draft_staleness_detection_signals.md. Measured before touching anything, against a labelled set: PyAutoMind f25e154e, 148 prompts, five findings independently confirmed against upstream source. Reconcile flagged 96 of 148 (65%), 52 of them "high", and MISSED the largest true positive entirely. Not a missing signal -- every signal counted the same, so a completion record merely NAMING a prompt made it high, which described most of the backlog. BEFORE 96/148 (65%) 52 high biggest find not flagged AFTER 31/148 (21%) 9 high biggest find at RANK 2 What changed, each grounded in a specific confirmed finding rather than taste: Bare references no longer score. They are kept as evidence, because they are worth reading once a prompt is flagged, but on their own they were the single biggest noise source. Status: alone no longer flags. It is hand-set across the backlog and contributed a whole confidence band that said nothing. Rare tokens replace raw Jaccard, IDF-weighted with a fan-out bonus. The biggest find scores 0.25 Jaccard against its own completion record -- unreachable at any threshold -- because record stems are short and share nothing else. The real signal is that ONE rare token (kxs, in 7 of 947 records) appears in SIX record stems: a series that shipped in phases. Note the obvious first try, requiring two shared tokens, scores that case exactly 0. Shared rare identifiers, weighted by how many. Two is a coincidence; six is the record describing this prompt's deliverable. This is what a human grader reads. A record asserting the work SHIPPED scores on its own. One sentence -- "the 4 jax_substructure/ prompts shipped to main" -- retired four prompts in the sweep, and nothing else had flagged them. The instructive failure is recorded in the code. Matching a bare // series prefix pulled in one more true finding, and also falsely flagged test_mode_bypass_ordered_assertion_ties off references to four UNRELATED sibling prompts in the same folder -- a prompt confirmed NOT shipped, and exactly the mis-grade this tool must never make. The series match now requires the line to discuss the folder's prompts as a group, which keeps the jax_substructure win without the false positive. Two of the five findings remain out of reach and that is correct, not a gap: one had no completion record at all (its evidence sat inside a sibling PROMPT), the other left no Mind trace whatsoever. Chasing them from Mind-local evidence costs precision without gaining truth; they need the upstream leg. 8 new tests, each driving input that must trip or must not trip the leg, including the noise cases that motivated the change and the read-only contract. One test carries a note that the rare-token signal is inherently scale-relative -- an earlier version of it failed because a 5-record fixture cannot express "rare", not because the ranker was wrong. Full suite 275 passed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E3MuurHXi3xo9TLRpMLJA6 --- agents/conductors/intake/_intake.py | 188 ++++++++++++++++++++---- tests/test_intake_reconcile_ranking.py | 194 +++++++++++++++++++++++++ 2 files changed, 352 insertions(+), 30 deletions(-) create mode 100644 tests/test_intake_reconcile_ranking.py diff --git a/agents/conductors/intake/_intake.py b/agents/conductors/intake/_intake.py index 87dfe89..a992635 100755 --- a/agents/conductors/intake/_intake.py +++ b/agents/conductors/intake/_intake.py @@ -25,6 +25,7 @@ import argparse import datetime as _dt import json +import math import re import sys from pathlib import Path @@ -598,6 +599,52 @@ def emit_formalise(res: dict): # deferred follow-up (still open) rather than the shipped task itself. _FOLLOWUP_WORDS = ("follow", "restore", "parked", "remain", "blocked", "later", "next step", "next-step", "deferred") +# Wording that makes a reference line an assertion the work is DONE, rather than +# a passing mention. `jax-substructure-simulator.md` opens "the 4 +# `jax_substructure/` prompts shipped to `main`" — that sentence resolves four +# prompts, and is the difference between a citation and a completion claim. +_SHIPPED_WORDS = ("shipped", "delivered", "merged", "completed", "closed out", + "close-out", "landed", "is done", "now on main") +#: A rare identifier says far more than a shared English word. Backticked +#: snake_case / CamelCase with at least two segments — `chunk_size`, +#: `_validate_convolve_over_sample_size`, `RectangularAdaptDensity`. +_IDENT_RE = re.compile( + r"`([A-Za-z_][A-Za-z0-9_]*(?:_[A-Za-z0-9_]+)+" + r"|[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+)`") +#: An identifier in this many records or more is vocabulary, not evidence. +_IDENT_COMMON_DF = 6 +#: Ditto for stem tokens: `jax` is in ~100 records and links nothing. +_TOKEN_COMMON_DF = 12 +_W_SHIPPED = 7.0 # a record asserting the work is done — on its own + # enough to qualify: `jax-substructure-simulator.md` + # saying "the 4 prompts shipped to main" resolved four + # prompts in one sentence, and nothing else flagged them. +_W_IDENT = 2.0 # per shared rare identifier beyond the first +_SUSPECT_THRESHOLD = 7.0 +_HIGH_THRESHOLD = 12.0 +# Tuned on the 2026-08-09 labelled set (PyAutoMind f25e154e, 148 prompts, five +# findings independently confirmed against upstream source). Result: +# +# BEFORE 96 of 148 flagged (65%) — 52 "high" — biggest find NOT flagged +# AFTER 31 of 148 flagged (21%) — 9 "high" — biggest find at rank 2 +# +# Of the five findings, this ranker catches the two it can: the k x s series +# (rare-token fan-out, rank 2) and the nufft chunking prompt (shared rare +# identifiers). The other three are NOT ranker failures and must not be chased +# by lowering the bar: +# +# * the test-mode umbrella states its own exit condition, which is what +# PyAutoMind's `Closes-when:` header key grades — a different tool; +# * the split-guard prompt had NO completion record at all (its evidence sat +# inside a sibling PROMPT), so nothing Mind-local could see it; +# * the latent prompt left no Mind trace whatsoever — the fix shipped upstream +# without a record. Only reading the target repo finds that shape. +# +# Every attempt to force those three in cost precision without gaining truth: +# a loose `//` series match pulled the umbrella in at 31% +# flagged, but also FALSELY flagged test_mode_bypass_ordered_assertion_ties off +# references to four unrelated sibling prompts — a prompt the sweep confirmed is +# NOT shipped, and exactly the mis-grade this tool must never make. def _tokens(s: str) -> set: @@ -605,6 +652,10 @@ def _tokens(s: str) -> set: if len(w) > 2 and w not in _STOPWORDS} +def _idents(text: str) -> set: + return set(_IDENT_RE.findall(text)) + + def reconcile(mind: Path, prefix: str = "") -> dict: """Rank backlog prompts that look already-shipped, for a human to retire. @@ -624,14 +675,31 @@ def reconcile(mind: Path, prefix: str = "") -> dict: # reference lines + `## ` topic headers now live inside the dated # records (the monolithic complete.md ledger was retired — issue #81) comp_lines: list = [] + comp_bodies: dict = {} for p in comp_files: - comp_lines.extend( - p.read_text(encoding="utf-8", errors="replace").splitlines()) + body = p.read_text(encoding="utf-8", errors="replace") + comp_bodies[p.name] = body + comp_lines.extend(body.splitlines()) + + # Document frequency over the records: how ORDINARY a token/identifier is. + # Without this every prompt matches on `jax`, `test`, `workspace` and the + # ranking is noise — the 2026-08-09 measurement flagged 96 of 148. + token_df: dict = {} + ident_df: dict = {} + for name, body in comp_bodies.items(): + for w in _tokens(name.replace("-", " ").replace(".md", "")) | _tokens(body): + token_df[w] = token_df.get(w, 0) + 1 + for i in _idents(body): + ident_df[i] = ident_df.get(i, 0) + 1 headers = [(ln[3:].strip(), _tokens(ln[3:].replace("-", " "))) for ln in comp_lines if ln.startswith("## ") and ln[3:].strip() != "Original prompt"] headers += [(f"complete/{p.relative_to(comp_dir)}", _tokens(p.stem.replace("_", " "))) for p in comp_files] + # Record STEMS specifically: a rare token appearing in several record stems + # is a phased series, which is a far stronger claim than one in their prose. + header_stems = [_tokens(p.stem.replace("-", " ")) for p in comp_files] + n_records = max(len(comp_files), 1) active = mind / "active" issued_names = ({p.name for p in active.glob("*.md")} if active.is_dir() else set()) @@ -646,47 +714,107 @@ def reconcile(mind: Path, prefix: str = "") -> dict: findings = [] score = 0.0 + # 1. A record line that NAMES this prompt and CLAIMS it is done. A bare + # mention is not evidence — measured on the 2026-08-09 labelled set, + # treating any reference as high confidence produced 52 of 148 highs + # and buried the true positives. + # A record may resolve a whole FOLDER of prompts at once — + # `jax-substructure-simulator.md` opens "the 4 `jax_substructure/` + # prompts shipped to `main`", which retires four files in one sentence. + # But `//` is also just a path prefix that every + # sibling reference contains, so matching it bare made a prompt "named" + # by any mention of its neighbours (measured: it falsely flagged + # test_mode_bypass_ordered_assertion_ties off references to four + # unrelated bug/autofit/ prompts). Require the line to be talking about + # the folder's prompts as a group. + series = f"{r['work_type']}/{r.get('target', '')}/" for ln in comp_lines: - if base in ln or sans_wt in ln: - kind = ("referenced-followup" - if any(w in ln.lower() for w in _FOLLOWUP_WORDS) - else "referenced") - findings.append((kind, ln.strip())) + low = ln.lower() + named = base in ln or sans_wt in ln + if not named and series in ln and "prompt" in low: + named = True + if not named: + continue + if any(w in low for w in _FOLLOWUP_WORDS): + findings.append(("referenced-followup", ln.strip())) + elif any(w in low for w in _SHIPPED_WORDS): + findings.append(("record-says-shipped", ln.strip())) + score += _W_SHIPPED + else: + # Evidence, not score. A record merely NAMING a prompt was the + # single biggest source of noise in the 2026-08-09 measurement: + # it alone produced 52 of 148 "high" verdicts and buried every + # true positive among them. + findings.append(("referenced", ln.strip())) if base in issued_names: findings.append(("issued-duplicate", f"active/{base} already exists")) + score += _W_SHIPPED if base in comp_names: findings.append(("complete-duplicate", f"{base} already in the complete/ archive")) - + score += _W_SHIPPED + + # 2. Rare stem tokens, IDF-weighted, with a fan-out bonus. Raw Jaccard + # missed the biggest find of the 2026-08-09 sweep: + # `oversampling_kxs_coupling` against `kxs-core` scores 0.25, under + # any workable threshold. The real signal is that ONE very rare token + # (`kxs`, in 7 of 947 records) appears in SIX record stems — a series + # that shipped in phases. Requiring two shared tokens, the obvious + # first try, scores that case exactly 0. sig = _tokens(base.replace("_", " ")) | _tokens(r["title"]) - best = (0.0, "", set()) - for h, ht in headers: - if not sig or not ht: + tok_score, evidence, best_fan = 0.0, [], 0 + for w in sig: + d = token_df.get(w, 0) + if not (0 < d <= _TOKEN_COMMON_DF): + continue + fan = sum(1 for st in header_stems if w in st) + if not fan: continue - shared = sig & ht - j = len(shared) / len(sig | ht) - if (j, len(shared)) > (best[0], len(best[2])): - best = (j, h, shared) - if best[0] >= 0.40 or len(best[2]) >= 3: - score = best[0] - findings.append(("topic-overlap", - f"completion record '{best[1]}' " - f"(shared: {', '.join(sorted(best[2]))})")) - - if r["status"] not in ("-", "formalised"): + best_fan = max(best_fan, fan) + tok_score += math.log(n_records / d) * (2.0 if fan >= 3 else 1.0) + evidence.append(f"{w} ({d} records" + + (f", {fan} in the stem" if fan >= 3 else "") + ")") + if tok_score: + score += tok_score + findings.append(("rare-topic-overlap", + "rare tokens shared with the records: " + + ", ".join(sorted(evidence)))) + + # 3. Rare identifiers the prompt names, appearing in a record body. This + # is what a human grader actually reads — `interferometer-jax-jit.md` + # naming `chunk_size` resolves the nufft prompt in one sentence. + # census() deliberately does not carry the prompt body (it is serialised + # into the dashboard JSON); read it here instead. + try: + prompt_text = (mind / path).read_text(encoding="utf-8", errors="replace") + except OSError: + prompt_text = "" + pid = {i for i in _idents(prompt_text) + if 0 < ident_df.get(i, 0) <= _IDENT_COMMON_DF} + if pid: + hits = {} + for p, body in comp_bodies.items(): + shared = {i for i in pid if i in body} + if len(shared) >= 2: + hits[p] = shared + if hits: + top = max(hits, key=lambda p: len(hits[p])) + n = len(hits[top]) + score += _W_IDENT * (n - 1) # 2 shared is weak, 7 is decisive + findings.append(("shared-identifiers", + f"record '{top}' names {n} of this prompt's " + f"identifiers: {', '.join(sorted(hits[top])[:5])}")) + + # `Status:` alone is not evidence — it fired on every hand-set draft. Kept + # as context on prompts something else already flagged, never as a reason. + if score > 0 and r["status"] not in ("-", "formalised"): findings.append(("stale-status", f"Status: {r['status']} — hand-set; verify against " "shipped state")) - if findings: - kinds = {k for k, _ in findings} - if kinds & {"issued-duplicate", "complete-duplicate", "referenced"}: - conf = "high" - elif "topic-overlap" in kinds: - conf = "medium" - else: - conf = "low" # follow-up reference / stale status only + if score >= _SUSPECT_THRESHOLD: + conf = "high" if score >= _HIGH_THRESHOLD else "medium" suspects.append({ "path": path, "title": r["title"], "confidence": conf, "overlap_score": round(score, 2), diff --git a/tests/test_intake_reconcile_ranking.py b/tests/test_intake_reconcile_ranking.py new file mode 100644 index 0000000..756244f --- /dev/null +++ b/tests/test_intake_reconcile_ranking.py @@ -0,0 +1,194 @@ +"""Contract tests for the `intake reconcile` ranking. + +Reconcile ranks backlog prompts that look already-shipped, for a human to +retire. It was measured against a labelled set on 2026-08-09 (PyAutoMind +`f25e154e`, 148 prompts, findings independently confirmed against upstream +source) and scored badly: **96 of 148 flagged — 65%** — while missing the +largest true positive entirely. The cause was not a missing signal, it was that +every signal counted the same. A completion record merely *naming* a prompt made +it `high`, which described most of the backlog. + +These tests pin the discriminations that fixed it. Each drives input that must +trip (or must NOT trip) the leg — a ranker that cannot rank is decoration. + +Hermetic: every fixture is a fictional Mind tree in tmp_path. Nothing here names +a real prompt or record, so the assertions are about the ranking logic, not +about whatever happens to be checked out. +""" + +import importlib.util +import sys +from pathlib import Path + +BRAIN_HOME = Path(__file__).resolve().parents[1] +_spec = importlib.util.spec_from_file_location( + "_intake_under_test", + BRAIN_HOME / "agents" / "conductors" / "intake" / "_intake.py") +_intake = importlib.util.module_from_spec(_spec) +sys.modules["_intake_under_test"] = _intake +_spec.loader.exec_module(_intake) + + +# --------------------------------------------------------------------------- # +# fixtures +# --------------------------------------------------------------------------- # +def _mind(root: Path, prompts: dict, records: dict) -> Path: + """Fictional Mind: draft///.md + complete/ records.""" + for rel, body in prompts.items(): + p = root / "draft" / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body, encoding="utf-8") + for name, body in records.items(): + p = root / "complete" / "2026" / "07" / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body, encoding="utf-8") + return root + + +def _paths(res) -> set: + return {s["path"] for s in res["suspects"]} + + +def _score(res, stem): + for s in res["suspects"]: + if s["path"].endswith(stem): + return s["overlap_score"] + return None + + +# --------------------------------------------------------------------------- # +# the noise that swamped the original ranking +# --------------------------------------------------------------------------- # +def test_a_record_merely_naming_a_prompt_is_not_a_suspect(tmp_path): + """The 2026-08-09 measurement's single biggest noise source. Any reference + scored `high`, which is why 52 of 148 prompts were `high` and the real + findings were indistinguishable from them.""" + root = _mind( + tmp_path, + {"bug/flywheel/sprocket_wobble.md": "# Sprocket wobble\n\nIt wobbles.\n"}, + {"gadget-alignment.md": "## gadget-alignment\n- notes: adjacent to " + "sprocket_wobble.md, which is unrelated here.\n"}, + ) + assert _paths(_intake.reconcile(root)) == set() + + +def test_a_record_asserting_the_work_shipped_is_a_suspect(tmp_path): + """The same reference, with a completion claim attached, is the real signal + — `jax-substructure-simulator.md` opens 'the 4 prompts shipped to main'.""" + root = _mind( + tmp_path, + {"bug/flywheel/sprocket_wobble.md": "# Sprocket wobble\n\nIt wobbles.\n"}, + {"gadget-alignment.md": "## gadget-alignment\n- notes: sprocket_wobble.md " + "shipped to main over PRs #1 and #2.\n"}, + ) + res = _intake.reconcile(root) + assert "draft/bug/flywheel/sprocket_wobble.md" in _paths(res) + + +def test_status_alone_never_makes_a_suspect(tmp_path): + """`Status:` is hand-set on most of the backlog; on its own it said nothing + and contributed a whole confidence band of noise.""" + root = _mind( + tmp_path, + {"bug/flywheel/sprocket_wobble.md": + "# Sprocket wobble\n\nStatus: planned\n\nIt wobbles.\n"}, + {"gadget-alignment.md": "## gadget-alignment\n- notes: unrelated.\n"}, + ) + assert _paths(_intake.reconcile(root)) == set() + + +# --------------------------------------------------------------------------- # +# the signals that found real drift +# --------------------------------------------------------------------------- # +def test_a_rare_token_across_several_record_stems_outranks_a_common_one(tmp_path): + """The biggest find of the sweep, in miniature. A phased series leaves + several records whose stems share one rare token; the prompt that started it + keeps that token. Raw Jaccard scores that case ~0.25 — below any workable + threshold — because the record stems are short and share nothing else. + + The control prompt shares a token present in EVERY record, which must not + rank: that is the `jax`/`workspace` case that made topic overlap useless. + + NOTE the archive size. This signal is IDF-weighted, so it is inherently + scale-relative: a token in 4 of 5 records is common, the same token in 4 of + 200 is rare. The real archive holds ~950 records, where `kxs` in 7 of them + is decisive. A toy fixture of five records cannot express that, and an + earlier version of this test failed for exactly that reason rather than + because the ranker was wrong. + """ + records = {f"widget-{leg}.md": f"## widget-{leg}\n- notes: leg {leg} of the " + f"widget series.\n" + for leg in ("design", "core", "cache", "tests")} + for i in range(200): + records[f"unrelated-{i:03d}.md"] = ( + f"## unrelated-{i:03d}\n- notes: common work, routine and shared.\n") + root = _mind( + tmp_path, + {"feature/flywheel/widget_coupling.md": "# Widget coupling\n\nThe plan.\n", + "feature/flywheel/routine_shared_work.md": "# Routine shared\n\nA task.\n"}, + records, + ) + res = _intake.reconcile(root) + assert "draft/feature/flywheel/widget_coupling.md" in _paths(res) + assert "draft/feature/flywheel/routine_shared_work.md" not in _paths(res) + + +def test_shared_rare_identifiers_rank_by_how_many_are_shared(tmp_path): + """What a human grader actually reads. One shared identifier is a + coincidence; several is the record describing this prompt's deliverable.""" + idents = [f"`_flywheel_helper_{i}`" for i in range(6)] + root = _mind( + tmp_path, + {"feature/flywheel/many.md": "# Many\n\n" + " ".join(idents) + "\n", + "feature/flywheel/one.md": "# One\n\n" + idents[0] + "\n"}, + {"gadget-alignment.md": "## gadget-alignment\n- notes: " + + " ".join(idents) + "\n"}, + ) + res = _intake.reconcile(root) + many = _score(res, "many.md") + assert many is not None, "six shared identifiers must be a suspect" + assert _score(res, "one.md") is None, "one shared identifier is not evidence" + + +def test_an_identifier_in_every_record_is_vocabulary_not_evidence(tmp_path): + """`Array2D` appears everywhere; matching on it links nothing.""" + common = "`common_helper_name`" + records = {f"rec-{i}.md": f"## rec-{i}\n- notes: {common} used here.\n" + for i in range(8)} + root = _mind( + tmp_path, + {"feature/flywheel/uses_common.md": f"# Uses common\n\n{common}\n"}, + records, + ) + assert _paths(_intake.reconcile(root)) == set() + + +# --------------------------------------------------------------------------- # +# the contract that must not change +# --------------------------------------------------------------------------- # +def test_reconcile_never_writes(tmp_path): + """Retiring a prompt writes to complete/ and stays a human act.""" + root = _mind( + tmp_path, + {"bug/flywheel/sprocket_wobble.md": "# Sprocket wobble\n\nIt wobbles.\n"}, + {"gadget-alignment.md": "## gadget-alignment\n- notes: sprocket_wobble.md " + "shipped to main.\n"}, + ) + before = {p: p.read_bytes() for p in root.rglob("*.md")} + _intake.reconcile(root) + after = {p: p.read_bytes() for p in root.rglob("*.md")} + assert before == after + + +def test_suspects_carry_their_evidence_and_a_band(tmp_path): + """The output is a review list, so every row must say why it is there.""" + root = _mind( + tmp_path, + {"bug/flywheel/sprocket_wobble.md": "# Sprocket wobble\n\nIt wobbles.\n"}, + {"gadget-alignment.md": "## gadget-alignment\n- notes: sprocket_wobble.md " + "shipped to main.\n"}, + ) + s = _intake.reconcile(root)["suspects"][0] + assert s["confidence"] in ("high", "medium") + assert s["findings"] and all(f["kind"] and f["evidence"] for f in s["findings"]) + assert any(f["kind"] == "record-says-shipped" for f in s["findings"])