fix(contract): make the contract lanes report what they ran

.github/workflows/hermes-drift.yml

@@ -43,6 +43,10 @@ jobs:
           path: .hermes-agent-upstream
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Record the upstream revision under test
         id: upstream
         run: |

.github/workflows/release.yml

@@ -55,6 +55,10 @@ jobs:
           ref: ${{ github.sha }}
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Attach the exact trigger to local main
         env:
           EXPECTED_SHA: ${{ github.sha }}
@@ -172,6 +176,10 @@ jobs:
           ref: ${{ github.sha }}
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Download the immutable release source
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
@@ -221,6 +229,10 @@ jobs:
           ref: ${{ github.sha }}
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Download the immutable release source
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
         with:
@@ -298,6 +310,10 @@ jobs:
           ref: ${{ github.sha }}
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Install ORAS
         uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d  # v2.0.1
         with:

.github/workflows/test.yml

@@ -16,6 +16,10 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Run unittest suite
         run: just test
 
@@ -25,6 +29,10 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       - name: Build and validate package metadata
         run: |
           just build
@@ -44,16 +52,20 @@ jobs:
   hermes-deployed-contract:
     runs-on: ubuntu-latest
     env:
-      DEPLOYED_HERMES_REVISION: f80f453ae0679347e38abc917c7f94f717bf96c5
+      DEPLOYED_HERMES_REVISION: 939e45c91d751fadd94dcd1b873ac3cb44846213
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           repository: NousResearch/hermes-agent
-          ref: f80f453ae0679347e38abc917c7f94f717bf96c5
+          ref: 939e45c91d751fadd94dcd1b873ac3cb44846213
           path: .hermes-agent-deployed
       - name: Install the locked toolchain (mise.toml)
         uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e  # v5.2.3
+        with:
+          # Pinned so every run, release jobs included, installs the same mise;
+          # the action still verifies it against mise's signed checksums.
+          version: "2026.10.18"
       # The full suite, not the context-engine module alone. It is green at
       # this pin, so widening adds coverage rather than risk.
       - name: Run the full contract suite against the deployed Hermes revision

.gitignore

@@ -9,3 +9,5 @@ dist/
 .hermes-agent-pinned/
 .hermes-agent-upstream/
 .DS_Store
+.hermes-venv-pinned/
+.hermes-venv-upstream/

hermes_plugin_kit/testing.py

@@ -78,8 +78,8 @@
 #:
 #: The CI workflow keeps its own literal -- ``actions/checkout`` consumes the
 #: ``ref:`` before a Python toolchain exists, so the workflow cannot read this
-#: value -- and a release-contract test asserts the two agree.
-DEPLOYED_HERMES_REVISION = "f80f453ae0679347e38abc917c7f94f717bf96c5"
+#: value.
+DEPLOYED_HERMES_REVISION = "939e45c91d751fadd94dcd1b873ac3cb44846213"
 
 #: Parameter names the kit's registrars pass positionally, in order.
 #:

justfile

@@ -6,6 +6,8 @@ openspec_min := "1.13.1"
 hermes_agent_repo := env_var_or_default("HERMES_AGENT_REPO", "https://github.com/NousResearch/hermes-agent.git")
 hermes_pinned_dir := env_var_or_default("HERMES_PINNED_DIR", ".hermes-agent-pinned")
 hermes_upstream_dir := env_var_or_default("HERMES_UPSTREAM_DIR", ".hermes-agent-upstream")
+hermes_pinned_venv := env_var_or_default("HERMES_PINNED_VENV", ".hermes-venv-pinned")
+hermes_upstream_venv := env_var_or_default("HERMES_UPSTREAM_VENV", ".hermes-venv-upstream")
 
 # Show available recipes.
 default:
@@ -59,6 +61,26 @@ prepare-hermes-ref ref dir:
     git -C "{{ dir }}" checkout -q FETCH_HEAD
     echo "hermes-agent at $(git -C "{{ dir }}" rev-parse HEAD)"
 
+# Build the host's own environment from the host's own lockfile.
+#
+# The kit's venv lacks the host's dependencies (ruamel, dotenv, ...), so the
+# contract modules died at import there. The host's `pyproject.toml` alone
+# does not resolve either, so its `uv.lock` is the only faithful source.
+[private]
+provision-hermes-host dir venv:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    host="$(cd "{{ dir }}" && pwd)"
+    mkdir -p "{{ venv }}"
+    venv="$(cd "{{ venv }}" && pwd)"
+    test -f "$host/uv.lock" || { echo "no uv.lock in $host; cannot provision the host faithfully" >&2; exit 1; }
+    spec="$({{ uv }} run python -c 'import sys, tomllib; print(tomllib.load(open(sys.argv[1], "rb"))["project"]["requires-python"])' "$host/pyproject.toml")"
+    py="$({{ uv }} python find --system "$spec")" \
+      || { echo "no interpreter on PATH satisfies the host's requires-python $spec; add one to mise.toml" >&2; exit 1; }
+    echo "hermes host python: $("$py" -V 2>&1) at $py (host requires-python $spec)"
+    (cd "$host" && UV_PROJECT_ENVIRONMENT="$venv" {{ uv }} sync --frozen --quiet --python "$py")
+    {{ uv }} pip install --quiet --python "$venv/bin/python" --reinstall-package hermes-plugin-kit .
+
 # Run the full contract suite against the deployed pin.
 #
 # HERMES_EXPECTED_REVISION makes the lane refuse a checkout at any other
@@ -71,9 +93,10 @@ test-contract-pinned:
     if [[ -z "${HERMES_AGENT_PATH:-}" ]]; then
       just prepare-hermes-ref "$pin" "{{ hermes_pinned_dir }}"
     fi
+    just provision-hermes-host "$dir" "{{ hermes_pinned_venv }}"
     HERMES_CONTRACT_REQUIRED=1 HERMES_EXPECTED_REVISION="$pin" \
       HERMES_AGENT_PATH="$(cd "$dir" && pwd)" \
-      just _run-contract-suite "pinned $pin"
+      just _run-contract-suite "pinned $pin" "{{ hermes_pinned_venv }}"
 
 # Run the full contract suite against upstream main -- the drift lane.
 test-contract-upstream:
@@ -84,29 +107,29 @@ test-contract-upstream:
       just prepare-hermes-ref main "{{ hermes_upstream_dir }}"
     fi
     rev="$(git -C "$dir" rev-parse HEAD)"
+    just provision-hermes-host "$dir" "{{ hermes_upstream_venv }}"
     HERMES_CONTRACT_REQUIRED=1 HERMES_AGENT_PATH="$(cd "$dir" && pwd)" \
-      just _run-contract-suite "upstream $rev"
+      just _run-contract-suite "upstream $rev" "{{ hermes_upstream_venv }}"
 
 # Backwards-compatible alias for the upstream lane.
 test-contract: test-contract-upstream
 
 # Run both contract modules and prove the run actually executed checks.
 #
-# The exit status alone cannot say whether anything ran: on Python 3.11 --
-# the version CI pins -- unittest exits 0 both when every test skips and when
-# zero tests are collected. So the executed count is the evidence.
+# The exit status alone cannot say whether anything ran: unittest exits 0 when
+# every test skips. So the executed count is the evidence.
 [private]
-_run-contract-suite label:
+_run-contract-suite label venv:
     #!/usr/bin/env bash
     set -euo pipefail
     log="$(mktemp)"
     trap 'rm -f "$log"' EXIT
     set +e
-    {{ uv }} run python -m unittest tests.test_hermes_contract tests.test_context_engine_contract -v \
+    "{{ venv }}/bin/python" -m unittest tests.test_hermes_contract tests.test_context_engine_contract -v \
       > "$log" 2>&1
     echo "EXIT=$?" >> "$log"
     set -e
-    tail -25 "$log"
+    cat "$log"
     grep -qE '^EXIT=0' "$log" || { echo "contract suite failed ({{ label }})" >&2; exit 1; }
     ran="$(grep -oE '^Ran [0-9]+' "$log" | grep -oE '[0-9]+' | head -n1)"
     test -n "$ran" && [ "$ran" -gt 0 ] \
@@ -122,7 +145,8 @@ test-context-engine-contract:
     #!/usr/bin/env bash
     set -euo pipefail
     : "${HERMES_AGENT_PATH:?set it to a hermes-agent checkout; this lane does not guess}"
-    HERMES_CONTRACT_REQUIRED=1 {{ uv }} run python -m unittest tests.test_context_engine_contract -v
+    just provision-hermes-host "$HERMES_AGENT_PATH" "{{ hermes_pinned_venv }}"
+    HERMES_CONTRACT_REQUIRED=1 "{{ hermes_pinned_venv }}/bin/python" -m unittest tests.test_context_engine_contract -v
 
 # Refuse to run when the installed OpenSpec predates the archive-correctness fixes.
 [private]

mise.toml

@@ -1,12 +1,10 @@
-# The one home for this repository's toolchain. CI installs these tools with
-# jdx/mise-action from mise.lock, so laptops and CI run the same Python, uv and
-# Just. Run `mise trust && mise install --locked` once per checkout; the
-# justfile recipes stay the entry points.
-min_version = "2026.10.15"
+min_version = "2026.10.18"
 
 [tools]
 # Must equal the Python in the deployed Hermes image (upstream pm/lock.json).
-python = "3.14.9"
+# 3.13 exists only for the pre-upgrade host's venv (its requires-python is <3.14);
+# drop it when DEPLOYED_HERMES_REVISION moves to v0.22.5 or later.
+python = ["3.14.9", "3.13.14"]
 uv = "0.13.5"
 just = "1.59.0"
 

scripts/classify_drift.py

@@ -90,10 +90,38 @@ def fingerprint(body: str) -> str:
     return "unknown"
 
 
+_IMPORT_FAILURE = re.compile(r"^ImportError: Failed to import test module: (\S+)", re.M)
+_RAN = re.compile(r"^Ran (\d+) tests?\b", re.M)
+_EXIT = re.compile(r"^EXIT=(\d+)\s*$", re.M)
+_SKIPPED = re.compile(r"^(?:OK|FAILED) \(.*\bskipped=(\d+)", re.M)
+
+
+def _did_not_execute(log: str, parsed_failures: int) -> list[str]:
+    reasons = [
+        f"{module} — failed to import, so none of its checks ran"
+        for module in dict.fromkeys(_IMPORT_FAILURE.findall(log))
+    ]
+    ran = _RAN.findall(log)
+    if not ran:
+        reasons.append("no `Ran N tests` line — the suite never completed")
+    elif int(ran[-1]) == 0:
+        reasons.append("Ran 0 tests — nothing was checked")
+    skipped = sum(int(n) for n in _SKIPPED.findall(log))
+    if skipped:
+        reasons.append(f"{skipped} contract tests skipped — they checked nothing")
+    exits = [int(code) for code in _EXIT.findall(log)]
+    if not exits:
+        reasons.append("no `EXIT=` line — the run's outcome is unknown")
+    elif any(exits) and parsed_failures == 0:
+        reasons.append(f"suite exited {max(exits)} with no failure the log explains")
+    return reasons
+
+
 def classify(log: str, baseline: dict[str, str]) -> tuple[list[str], list[str]]:
     known: list[str] = []
     new: list[str] = []
-    for match in _BLOCK.finditer(log):
+    blocks = list(_BLOCK.finditer(log))
+    for match in blocks:
         test_id = match.group("test_id")
         print_id = test_id
         fp = fingerprint(match.group("body"))
@@ -105,6 +133,7 @@ def classify(log: str, baseline: dict[str, str]) -> tuple[list[str], list[str]]:
                 f"baselined as {expected!r} but failed as {fp!r}"
             )
             new.append(f"{print_id} — {reason}")
+    new += _did_not_execute(log, len(blocks))
     return known, new
 
 

tests/fixtures/contract-logs/import-failure-full.log

@@ -0,0 +1,54 @@
+test_hermes_contract (unittest.loader._FailedTest.test_hermes_contract) ... ERROR
+test_context_engine_contract (unittest.loader._FailedTest.test_context_engine_contract) ... ERROR
+
+======================================================================
+ERROR: test_hermes_contract (unittest.loader._FailedTest.test_hermes_contract)
+----------------------------------------------------------------------
+ImportError: Failed to import test module: test_hermes_contract
+Traceback (most recent call last):
+  File "/home/runner/.local/share/mise/installs/python/3.14.9/lib/python3.14/unittest/loader.py", line 141, in loadTestsFromName
+    module = __import__(module_name)
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_hermes_contract.py", line 119, in <module>
+    _REAL = _import_real_hermes()
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_hermes_contract.py", line 116, in _import_real_hermes
+    return _try()
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_hermes_contract.py", line 56, in _try
+    from hermes_cli.plugins import (  # type: ignore
+    ...<5 lines>...
+    )
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_cli/plugins.py", line 31, in <module>
+    from utils import env_var_enabled
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/utils.py", line 18, in <module>
+    import hermes_yaml as yaml
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_yaml.py", line 10, in <module>
+    from ruamel.yaml import YAML
+ModuleNotFoundError: No module named 'ruamel'
+
+
+======================================================================
+ERROR: test_context_engine_contract (unittest.loader._FailedTest.test_context_engine_contract)
+----------------------------------------------------------------------
+ImportError: Failed to import test module: test_context_engine_contract
+Traceback (most recent call last):
+  File "/home/runner/.local/share/mise/installs/python/3.14.9/lib/python3.14/unittest/loader.py", line 141, in loadTestsFromName
+    module = __import__(module_name)
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 37, in <module>
+    _HOST = _import_deployed_host()
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 28, in _import_deployed_host
+    from hermes_cli.plugins import (  # type: ignore
+    ...<3 lines>...
+    )
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_cli/plugins.py", line 31, in <module>
+    from utils import env_var_enabled
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/utils.py", line 18, in <module>
+    import hermes_yaml as yaml
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_yaml.py", line 10, in <module>
+    from ruamel.yaml import YAML
+ModuleNotFoundError: No module named 'ruamel'
+
+
+----------------------------------------------------------------------
+Ran 2 tests in 0.000s
+
+FAILED (errors=2)
+EXIT=1

tests/fixtures/contract-logs/import-failure-tail.log

@@ -0,0 +1,29 @@
+----------------------------------------------------------------------
+ImportError: Failed to import test module: test_context_engine_contract
+Traceback (most recent call last):
+  File "/home/runner/.local/share/mise/installs/python/3.14.9/lib/python3.14/unittest/loader.py", line 141, in loadTestsFromName
+    module = __import__(module_name)
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 37, in <module>
+    _HOST = _import_deployed_host()
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 28, in _import_deployed_host
+    from hermes_cli.plugins import (  # type: ignore
+    ...<3 lines>...
+    )
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_cli/plugins.py", line 31, in <module>
+    from utils import env_var_enabled
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/utils.py", line 18, in <module>
+    import hermes_yaml as yaml
+  File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_yaml.py", line 10, in <module>
+    from ruamel.yaml import YAML
+ModuleNotFoundError: No module named 'ruamel'
+
+
+----------------------------------------------------------------------
+Ran 2 tests in 0.000s
+
+FAILED (errors=2)
+EXIT=1
+contract suite failed (upstream 0000000000000000000000000000000000000000)
+error: recipe `_run-contract-suite` failed with exit code 1
+error: recipe `test-contract-upstream` failed with exit code 1
+EXIT=1