diff --git a/.github/workflows/hermes-drift.yml b/.github/workflows/hermes-drift.yml index a6dfb5c..ee6a758 100644 --- a/.github/workflows/hermes-drift.yml +++ b/.github/workflows/hermes-drift.yml @@ -41,13 +41,12 @@ jobs: with: repository: NousResearch/hermes-agent path: .hermes-agent-upstream - - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - python-version: "3.11" - enable-cache: true - - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 - with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Record the upstream revision under test id: upstream run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6cfa285..3f16b84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,19 +53,16 @@ jobs: fetch-depth: 0 persist-credentials: false ref: ${{ github.sha }} - - name: Install Just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Attach the exact trigger to local main env: EXPECTED_SHA: ${{ github.sha }} run: just release-attach-trigger - - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - python-version: "3.11" - enable-cache: true - name: Install the frozen release toolchain run: uv sync --frozen - name: Validate the tagged baseline and strict release intent @@ -177,15 +174,12 @@ jobs: fetch-depth: 0 persist-credentials: false ref: ${{ github.sha }} - - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - python-version: "3.11" - enable-cache: true - - name: Install Just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Download the immutable release source uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -233,15 +227,12 @@ jobs: fetch-depth: 0 persist-credentials: false ref: ${{ github.sha }} - - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - python-version: "3.11" - enable-cache: true - - name: Install Just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 - with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Download the immutable release source uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -317,15 +308,12 @@ jobs: fetch-depth: 0 persist-credentials: false ref: ${{ github.sha }} - - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - python-version: "3.11" - enable-cache: true - - name: Install Just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Install ORAS uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 44eccbd..98c499e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,13 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - python-version: "3.11" - enable-cache: true - - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 - with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Run unittest suite run: just test @@ -28,13 +27,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - python-version: "3.11" - enable-cache: true - - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" - name: Build and validate package metadata run: | just build @@ -50,25 +48,24 @@ jobs: # Amber admission gates on the exact deployed Hermes fork contract. The pin # below is test-enforced against hermes_plugin_kit.testing's constant; the # workflow cannot read that constant, because actions/checkout consumes the - # ref before setup-uv installs Python. + # ref before mise installs Python. hermes-deployed-contract: runs-on: ubuntu-latest env: - DEPLOYED_HERMES_REVISION: f80f453ae0679347e38abc917c7f94f717bf96c5 + DEPLOYED_HERMES_REVISION: 939e45c91d751fadd94dcd1b873ac3cb44846213 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NousResearch/hermes-agent - ref: f80f453ae0679347e38abc917c7f94f717bf96c5 + ref: 939e45c91d751fadd94dcd1b873ac3cb44846213 path: .hermes-agent-deployed - - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - with: - python-version: "3.11" - enable-cache: true - - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Install the locked toolchain (mise.toml) + uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3 with: - just-version: "1.58.0" + # Pinned so every run, release jobs included, installs the same mise; + # the action still verifies it against mise's signed checksums. + version: "2026.10.18" # The full suite, not the context-engine module alone. It is green at # this pin, so widening adds coverage rather than risk. - name: Run the full contract suite against the deployed Hermes revision diff --git a/.gitignore b/.gitignore index 4007cc1..34f7c71 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ dist/ .hermes-agent-pinned/ .hermes-agent-upstream/ .DS_Store +.hermes-venv-pinned/ +.hermes-venv-upstream/ diff --git a/AGENTS.md b/AGENTS.md index 57c9536..c6e2d46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,8 +73,11 @@ plugin. and recovery dispatch on `get_tool_schemas` / `handle_tool_call`; do not duplicate native engine tools through `@tool`. - Redact secret-looking values in logs and avoid logging full untrusted payloads. -- Use `uv` and the `justfile` for local development: - `just install`, `just test`, +- Get Python, `uv` and `just` from `mise.toml` and `mise.lock` + (`mise trust && mise install --locked`), the same toolchain CI installs; + after changing a tool pin, regenerate the lock with + `mise lock --platform linux-x64,linux-arm64,macos-arm64`. Then use the + `justfile` for local development: `just install`, `just test`, `just test-one tests.test_kit.SchemaConventionTests`, and `just build`. ## Release Notes diff --git a/README.md b/README.md index 7ce7b6a..be0c223 100644 --- a/README.md +++ b/README.md @@ -859,10 +859,16 @@ so the linked directory is self-contained. ## Development -Uses [uv](https://docs.astral.sh/uv/) and -[Just](https://github.com/casey/just). On macOS, install both with -`brew install uv just`; otherwise install them using their platform-specific -instructions. +Uses [mise](https://mise.jdx.dev/) to install the pinned Python, +[uv](https://docs.astral.sh/uv/) and [Just](https://github.com/casey/just) +from `mise.toml` and `mise.lock`, the same toolchain CI installs. Once per +checkout: + +```bash +mise trust && mise install --locked +``` + +Then: ```bash just install # uv sync — create/sync the dev environment diff --git a/hermes_plugin_kit/testing.py b/hermes_plugin_kit/testing.py index da198f3..816ded2 100644 --- a/hermes_plugin_kit/testing.py +++ b/hermes_plugin_kit/testing.py @@ -78,8 +78,8 @@ #: #: The CI workflow keeps its own literal -- ``actions/checkout`` consumes the #: ``ref:`` before a Python toolchain exists, so the workflow cannot read this -#: value -- and a release-contract test asserts the two agree. -DEPLOYED_HERMES_REVISION = "f80f453ae0679347e38abc917c7f94f717bf96c5" +#: value. +DEPLOYED_HERMES_REVISION = "939e45c91d751fadd94dcd1b873ac3cb44846213" #: Parameter names the kit's registrars pass positionally, in order. #: diff --git a/justfile b/justfile index 93cab96..64025c9 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,8 @@ openspec_min := "1.13.1" hermes_agent_repo := env_var_or_default("HERMES_AGENT_REPO", "https://github.com/NousResearch/hermes-agent.git") hermes_pinned_dir := env_var_or_default("HERMES_PINNED_DIR", ".hermes-agent-pinned") hermes_upstream_dir := env_var_or_default("HERMES_UPSTREAM_DIR", ".hermes-agent-upstream") +hermes_pinned_venv := env_var_or_default("HERMES_PINNED_VENV", ".hermes-venv-pinned") +hermes_upstream_venv := env_var_or_default("HERMES_UPSTREAM_VENV", ".hermes-venv-upstream") # Show available recipes. default: @@ -59,6 +61,26 @@ prepare-hermes-ref ref dir: git -C "{{ dir }}" checkout -q FETCH_HEAD echo "hermes-agent at $(git -C "{{ dir }}" rev-parse HEAD)" +# Build the host's own environment from the host's own lockfile. +# +# The kit's venv lacks the host's dependencies (ruamel, dotenv, ...), so the +# contract modules died at import there. The host's `pyproject.toml` alone +# does not resolve either, so its `uv.lock` is the only faithful source. +[private] +provision-hermes-host dir venv: + #!/usr/bin/env bash + set -euo pipefail + host="$(cd "{{ dir }}" && pwd)" + mkdir -p "{{ venv }}" + venv="$(cd "{{ venv }}" && pwd)" + test -f "$host/uv.lock" || { echo "no uv.lock in $host; cannot provision the host faithfully" >&2; exit 1; } + spec="$({{ uv }} run python -c 'import sys, tomllib; print(tomllib.load(open(sys.argv[1], "rb"))["project"]["requires-python"])' "$host/pyproject.toml")" + py="$({{ uv }} python find --system "$spec")" \ + || { echo "no interpreter on PATH satisfies the host's requires-python $spec; add one to mise.toml" >&2; exit 1; } + echo "hermes host python: $("$py" -V 2>&1) at $py (host requires-python $spec)" + (cd "$host" && UV_PROJECT_ENVIRONMENT="$venv" {{ uv }} sync --frozen --quiet --python "$py") + {{ uv }} pip install --quiet --python "$venv/bin/python" --reinstall-package hermes-plugin-kit . + # Run the full contract suite against the deployed pin. # # HERMES_EXPECTED_REVISION makes the lane refuse a checkout at any other @@ -71,9 +93,10 @@ test-contract-pinned: if [[ -z "${HERMES_AGENT_PATH:-}" ]]; then just prepare-hermes-ref "$pin" "{{ hermes_pinned_dir }}" fi + just provision-hermes-host "$dir" "{{ hermes_pinned_venv }}" HERMES_CONTRACT_REQUIRED=1 HERMES_EXPECTED_REVISION="$pin" \ HERMES_AGENT_PATH="$(cd "$dir" && pwd)" \ - just _run-contract-suite "pinned $pin" + just _run-contract-suite "pinned $pin" "{{ hermes_pinned_venv }}" # Run the full contract suite against upstream main -- the drift lane. test-contract-upstream: @@ -84,29 +107,29 @@ test-contract-upstream: just prepare-hermes-ref main "{{ hermes_upstream_dir }}" fi rev="$(git -C "$dir" rev-parse HEAD)" + just provision-hermes-host "$dir" "{{ hermes_upstream_venv }}" HERMES_CONTRACT_REQUIRED=1 HERMES_AGENT_PATH="$(cd "$dir" && pwd)" \ - just _run-contract-suite "upstream $rev" + just _run-contract-suite "upstream $rev" "{{ hermes_upstream_venv }}" # Backwards-compatible alias for the upstream lane. test-contract: test-contract-upstream # Run both contract modules and prove the run actually executed checks. # -# The exit status alone cannot say whether anything ran: on Python 3.11 -- -# the version CI pins -- unittest exits 0 both when every test skips and when -# zero tests are collected. So the executed count is the evidence. +# The exit status alone cannot say whether anything ran: unittest exits 0 when +# every test skips. So the executed count is the evidence. [private] -_run-contract-suite label: +_run-contract-suite label venv: #!/usr/bin/env bash set -euo pipefail log="$(mktemp)" trap 'rm -f "$log"' EXIT set +e - {{ uv }} run python -m unittest tests.test_hermes_contract tests.test_context_engine_contract -v \ + "{{ venv }}/bin/python" -m unittest tests.test_hermes_contract tests.test_context_engine_contract -v \ > "$log" 2>&1 echo "EXIT=$?" >> "$log" set -e - tail -25 "$log" + cat "$log" grep -qE '^EXIT=0' "$log" || { echo "contract suite failed ({{ label }})" >&2; exit 1; } ran="$(grep -oE '^Ran [0-9]+' "$log" | grep -oE '[0-9]+' | head -n1)" test -n "$ran" && [ "$ran" -gt 0 ] \ @@ -122,7 +145,8 @@ test-context-engine-contract: #!/usr/bin/env bash set -euo pipefail : "${HERMES_AGENT_PATH:?set it to a hermes-agent checkout; this lane does not guess}" - HERMES_CONTRACT_REQUIRED=1 {{ uv }} run python -m unittest tests.test_context_engine_contract -v + just provision-hermes-host "$HERMES_AGENT_PATH" "{{ hermes_pinned_venv }}" + HERMES_CONTRACT_REQUIRED=1 "{{ hermes_pinned_venv }}/bin/python" -m unittest tests.test_context_engine_contract -v # Refuse to run when the installed OpenSpec predates the archive-correctness fixes. [private] diff --git a/mise.lock b/mise.lock new file mode 100644 index 0000000..696e354 --- /dev/null +++ b/mise.lock @@ -0,0 +1,86 @@ +# @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html + +lockfile_version = 3 + +[[tools.just]] +version = "1.59.0" +backend = "aqua:casey/just" +specifiers = ["1.59.0"] + +[tools.just."platforms.linux-arm64"] +checksum = "sha256:23871b1702bfd550fb5f901778fd4290ecf2ee034a45a889fbe96d89b188086d" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-aarch64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/casey/just/releases/assets/642950086" + +[tools.just."platforms.linux-x64"] +checksum = "sha256:409044617b6f06517c1035e55ca38c8c856ff8b650e41bc2aa1ef82d5903a23e" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/casey/just/releases/assets/642949555" + +[tools.just."platforms.macos-arm64"] +checksum = "sha256:6574cea774ed4cae12e24b6aa11857de33e80dab4667cf628fd5840df96a2a9d" +url = "https://github.com/casey/just/releases/download/1.59.0/just-1.59.0-aarch64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/casey/just/releases/assets/643129724" + +[[tools.python]] +version = "3.14.9" +backend = "core:python" +specifiers = ["3.14.9"] + +[tools.python."platforms.linux-arm64"] +checksum = "sha256:6f77731d85a14ed848dd4041478cd3ed511523caa3d896b84b436a7edf13cf62" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261021/cpython-3.14.9+20261021-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.linux-x64"] +checksum = "sha256:561e91c1cadf30e84aec94d1917384ae3267e5f3e6e135e51fd1578bb19f4064" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261021/cpython-3.14.9+20261021-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.macos-arm64"] +checksum = "sha256:9f8c3480b8f0db553c270db548e8a21ae8e39df412e5661d66e8050c5c6ed2a8" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20261021/cpython-3.14.9+20261021-aarch64-apple-darwin-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[[tools.python]] +version = "3.13.14" +backend = "core:python" +specifiers = ["3.13.14"] + +[tools.python."platforms.linux-arm64"] +checksum = "sha256:f1cda716f5595a97e79f760f81da7150e4f6b4ae9dc49968b4ce5d6d64bba058" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260805/cpython-3.13.14+20260805-aarch64-unknown-linux-gnu-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.linux-x64"] +checksum = "sha256:39e82d05926bdcd206732026bcd878d9f00e288cd227d91c1fabf379b6ea4fa5" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260805/cpython-3.13.14+20260805-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[tools.python."platforms.macos-arm64"] +checksum = "sha256:f0e634654e1d6b55cf81419eb629caf78f50234d01ede318f78e5091bada6085" +url = "https://github.com/astral-sh/python-build-standalone/releases/download/20260805/cpython-3.13.14+20260805-aarch64-apple-darwin-install_only_stripped.tar.gz" +provenance = "github-attestations" + +[[tools.uv]] +version = "0.13.5" +backend = "aqua:astral-sh/uv" +specifiers = ["0.13.5"] + +[tools.uv."platforms.linux-arm64"] +checksum = "sha256:4bb69cf95ed13f23713bb7bc09f241896ea0c75b7c557e615a20604b82f9c403" +url = "https://github.com/astral-sh/uv/releases/download/0.13.5/uv-aarch64-unknown-linux-gnu.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/664335837" +provenance = "github-attestations" + +[tools.uv."platforms.linux-x64"] +checksum = "sha256:f68d291c75783122dad58ab77def0e13f3233f358731db5d6d7e8715603b928f" +url = "https://github.com/astral-sh/uv/releases/download/0.13.5/uv-x86_64-unknown-linux-gnu.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/664336023" +provenance = "github-attestations" + +[tools.uv."platforms.macos-arm64"] +checksum = "sha256:743f85035a1996ecb350659e01cd9faea9709829be09b1b4b03c73eaf3c9cd73" +url = "https://github.com/astral-sh/uv/releases/download/0.13.5/uv-aarch64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/664335803" +provenance = "github-attestations" diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..e89f898 --- /dev/null +++ b/mise.toml @@ -0,0 +1,14 @@ +min_version = "2026.10.18" + +[tools] +# Must equal the Python in the deployed Hermes image (upstream pm/lock.json). +# 3.13 exists only for the pre-upgrade host's venv (its requires-python is <3.14); +# drop it when DEPLOYED_HERMES_REVISION moves to v0.22.5 or later. +python = ["3.14.9", "3.13.14"] +uv = "0.13.5" +just = "1.59.0" + +[env] +# uv must run on mise's Python, never on one it manages or downloads itself. +UV_PYTHON_PREFERENCE = "only-system" +UV_PYTHON_DOWNLOADS = "never" diff --git a/scripts/classify_drift.py b/scripts/classify_drift.py index 73315f1..2e36a38 100644 --- a/scripts/classify_drift.py +++ b/scripts/classify_drift.py @@ -90,10 +90,38 @@ def fingerprint(body: str) -> str: return "unknown" +_IMPORT_FAILURE = re.compile(r"^ImportError: Failed to import test module: (\S+)", re.M) +_RAN = re.compile(r"^Ran (\d+) tests?\b", re.M) +_EXIT = re.compile(r"^EXIT=(\d+)\s*$", re.M) +_SKIPPED = re.compile(r"^(?:OK|FAILED) \(.*\bskipped=(\d+)", re.M) + + +def _did_not_execute(log: str, parsed_failures: int) -> list[str]: + reasons = [ + f"{module} — failed to import, so none of its checks ran" + for module in dict.fromkeys(_IMPORT_FAILURE.findall(log)) + ] + ran = _RAN.findall(log) + if not ran: + reasons.append("no `Ran N tests` line — the suite never completed") + elif int(ran[-1]) == 0: + reasons.append("Ran 0 tests — nothing was checked") + skipped = sum(int(n) for n in _SKIPPED.findall(log)) + if skipped: + reasons.append(f"{skipped} contract tests skipped — they checked nothing") + exits = [int(code) for code in _EXIT.findall(log)] + if not exits: + reasons.append("no `EXIT=` line — the run's outcome is unknown") + elif any(exits) and parsed_failures == 0: + reasons.append(f"suite exited {max(exits)} with no failure the log explains") + return reasons + + def classify(log: str, baseline: dict[str, str]) -> tuple[list[str], list[str]]: known: list[str] = [] new: list[str] = [] - for match in _BLOCK.finditer(log): + blocks = list(_BLOCK.finditer(log)) + for match in blocks: test_id = match.group("test_id") print_id = test_id fp = fingerprint(match.group("body")) @@ -105,6 +133,7 @@ def classify(log: str, baseline: dict[str, str]) -> tuple[list[str], list[str]]: f"baselined as {expected!r} but failed as {fp!r}" ) new.append(f"{print_id} — {reason}") + new += _did_not_execute(log, len(blocks)) return known, new diff --git a/tests/fixtures/contract-logs/import-failure-full.log b/tests/fixtures/contract-logs/import-failure-full.log new file mode 100644 index 0000000..5562277 --- /dev/null +++ b/tests/fixtures/contract-logs/import-failure-full.log @@ -0,0 +1,54 @@ +test_hermes_contract (unittest.loader._FailedTest.test_hermes_contract) ... ERROR +test_context_engine_contract (unittest.loader._FailedTest.test_context_engine_contract) ... ERROR + +====================================================================== +ERROR: test_hermes_contract (unittest.loader._FailedTest.test_hermes_contract) +---------------------------------------------------------------------- +ImportError: Failed to import test module: test_hermes_contract +Traceback (most recent call last): + File "/home/runner/.local/share/mise/installs/python/3.14.9/lib/python3.14/unittest/loader.py", line 141, in loadTestsFromName + module = __import__(module_name) + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_hermes_contract.py", line 119, in + _REAL = _import_real_hermes() + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_hermes_contract.py", line 116, in _import_real_hermes + return _try() + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_hermes_contract.py", line 56, in _try + from hermes_cli.plugins import ( # type: ignore + ...<5 lines>... + ) + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_cli/plugins.py", line 31, in + from utils import env_var_enabled + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/utils.py", line 18, in + import hermes_yaml as yaml + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_yaml.py", line 10, in + from ruamel.yaml import YAML +ModuleNotFoundError: No module named 'ruamel' + + +====================================================================== +ERROR: test_context_engine_contract (unittest.loader._FailedTest.test_context_engine_contract) +---------------------------------------------------------------------- +ImportError: Failed to import test module: test_context_engine_contract +Traceback (most recent call last): + File "/home/runner/.local/share/mise/installs/python/3.14.9/lib/python3.14/unittest/loader.py", line 141, in loadTestsFromName + module = __import__(module_name) + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 37, in + _HOST = _import_deployed_host() + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 28, in _import_deployed_host + from hermes_cli.plugins import ( # type: ignore + ...<3 lines>... + ) + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_cli/plugins.py", line 31, in + from utils import env_var_enabled + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/utils.py", line 18, in + import hermes_yaml as yaml + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_yaml.py", line 10, in + from ruamel.yaml import YAML +ModuleNotFoundError: No module named 'ruamel' + + +---------------------------------------------------------------------- +Ran 2 tests in 0.000s + +FAILED (errors=2) +EXIT=1 diff --git a/tests/fixtures/contract-logs/import-failure-tail.log b/tests/fixtures/contract-logs/import-failure-tail.log new file mode 100644 index 0000000..04853f8 --- /dev/null +++ b/tests/fixtures/contract-logs/import-failure-tail.log @@ -0,0 +1,29 @@ +---------------------------------------------------------------------- +ImportError: Failed to import test module: test_context_engine_contract +Traceback (most recent call last): + File "/home/runner/.local/share/mise/installs/python/3.14.9/lib/python3.14/unittest/loader.py", line 141, in loadTestsFromName + module = __import__(module_name) + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 37, in + _HOST = _import_deployed_host() + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/tests/test_context_engine_contract.py", line 28, in _import_deployed_host + from hermes_cli.plugins import ( # type: ignore + ...<3 lines>... + ) + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_cli/plugins.py", line 31, in + from utils import env_var_enabled + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/utils.py", line 18, in + import hermes_yaml as yaml + File "/home/runner/work/hermes-plugin-kit/hermes-plugin-kit/.hermes-agent-upstream/hermes_yaml.py", line 10, in + from ruamel.yaml import YAML +ModuleNotFoundError: No module named 'ruamel' + + +---------------------------------------------------------------------- +Ran 2 tests in 0.000s + +FAILED (errors=2) +EXIT=1 +contract suite failed (upstream 0000000000000000000000000000000000000000) +error: recipe `_run-contract-suite` failed with exit code 1 +error: recipe `test-contract-upstream` failed with exit code 1 +EXIT=1 diff --git a/tests/test_drift_classification.py b/tests/test_drift_classification.py index d3187ad..7a2e7ea 100644 --- a/tests/test_drift_classification.py +++ b/tests/test_drift_classification.py @@ -9,6 +9,8 @@ from __future__ import annotations import importlib.util +import subprocess +import sys import unittest from pathlib import Path @@ -21,6 +23,12 @@ assert _spec.loader is not None _spec.loader.exec_module(classify_drift) +LOGS = ROOT / "tests" / "fixtures" / "contract-logs" + +#: What a completed run with one baselined failure ends with. Without it a log +#: is not evidence that the suite ran at all. +_RAN_TRAILER = "\n" + "-" * 70 + "\nRan 17 tests in 0.601s\n\nFAILED (failures=1)\nEXIT=1\n" + class FingerprintStabilityTests(unittest.TestCase): """Same failure, different machine, same fingerprint.""" @@ -112,6 +120,7 @@ def test_a_known_failure_classifies_as_known(self) -> None: + "-" * 70 + "\nTraceback (most recent call last):\n" + f"AssertionError: ... {fingerprint.replace('-', ' ')} ...\n" + + _RAN_TRAILER ) known, new = classify_drift.classify(log, classify_drift.load_baseline()) @@ -126,6 +135,7 @@ def test_the_same_test_failing_differently_is_new(self) -> None: + "-" * 70 + "\nTraceback (most recent call last):\n" + "AssertionError: an entirely unrelated cause\n" + + _RAN_TRAILER ) known, new = classify_drift.classify(log, classify_drift.load_baseline()) @@ -134,5 +144,92 @@ def test_the_same_test_failing_differently_is_new(self) -> None: self.assertEqual(1, len(new), "a baselined test failing differently is a new break") +class RunThatDidNotExecuteTests(unittest.TestCase): + """A suite that never ran its checks is drift, not a clean report. + + The captured logs under ``tests/fixtures/contract-logs`` come from a real + run where both contract modules died importing the host + (``No module named 'ruamel'``). The tail-only one is exactly what the drift + workflow handed the classifier while the lane printed ``tail -25``: the + ``ERROR:`` headers were cut, no failure block parsed, and the report read + "No contract failures against upstream." + """ + + def test_tail_of_an_import_failure_is_new_drift_naming_the_module(self) -> None: + log = (LOGS / "import-failure-tail.log").read_text() + + known, new = classify_drift.classify(log, classify_drift.load_baseline()) + + self.assertEqual([], known) + self.assertTrue(new, "an import failure must never classify as clean") + self.assertTrue( + any("test_context_engine_contract" in entry for entry in new), new + ) + + def test_full_import_failure_log_names_both_modules(self) -> None: + log = (LOGS / "import-failure-full.log").read_text() + + _, new = classify_drift.classify(log, classify_drift.load_baseline()) + + joined = "\n".join(new) + self.assertIn("test_hermes_contract", joined) + self.assertIn("test_context_engine_contract", joined) + + def test_the_cli_exits_non_zero_on_the_tail_only_log(self) -> None: + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "classify_drift.py"), + str(LOGS / "import-failure-tail.log"), "0" * 40], + capture_output=True, text=True, check=False, + ) + + self.assertEqual(1, result.returncode, result.stdout) + self.assertNotIn("No contract failures", result.stdout) + + def test_zero_tests_ran_is_new_drift_even_when_unittest_exits_zero(self) -> None: + log = "\n" + "-" * 70 + "\nRan 0 tests in 0.000s\n\nOK\nEXIT=0\n" + + _, new = classify_drift.classify(log, {}) + + self.assertTrue(new) + + def test_a_log_with_no_ran_line_is_new_drift(self) -> None: + """The suite died before its summary, whatever the exit said.""" + log = "test_x (tests.test_hermes_contract.T.test_x) ... ok\nEXIT=0\n" + + _, new = classify_drift.classify(log, {}) + + self.assertTrue(new) + + def test_a_log_with_no_exit_line_is_new_drift(self) -> None: + log = "\n" + "-" * 70 + "\nRan 17 tests in 0.601s\n\nOK\n" + + _, new = classify_drift.classify(log, {}) + + self.assertTrue(new) + + def test_a_non_zero_exit_with_no_parsed_failure_is_new_drift(self) -> None: + log = "\n" + "-" * 70 + "\nRan 17 tests in 0.601s\n\nOK\nEXIT=0\nEXIT=1\n" + + _, new = classify_drift.classify(log, {}) + + self.assertTrue(new) + + def test_a_run_that_skipped_contract_tests_is_new_drift(self) -> None: + """A skipped contract test checked nothing, whatever the exit said.""" + log = "\n" + "-" * 70 + "\nRan 17 tests in 0.601s\n\nOK (skipped=3)\nEXIT=0\n" + + _, new = classify_drift.classify(log, {}) + + self.assertTrue(any("skipped" in entry for entry in new), new) + + def test_a_clean_run_reports_no_drift(self) -> None: + log = ( + "test_x (tests.test_hermes_contract.HermesContractTests.test_x) ... ok\n" + "\n" + "-" * 70 + "\nRan 17 tests in 0.601s\n\nOK\nEXIT=0\nEXIT=0\n" + ) + + self.assertEqual(([], []), classify_drift.classify(log, {})) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_hermes_contract.py b/tests/test_hermes_contract.py index 638c25a..bd29a2e 100644 --- a/tests/test_hermes_contract.py +++ b/tests/test_hermes_contract.py @@ -68,7 +68,11 @@ def _try(): run_tool_execution_middleware, ) from tools.registry import registry # type: ignore - from hermes_state import SCHEMA_VERSION, SessionDB # type: ignore + from hermes_state import SessionDB # type: ignore + try: # v0.22.5 dropped the hermes_state re-export (upstream #126164) + from hermes_state_common import SCHEMA_VERSION # type: ignore + except ImportError: + from hermes_state import SCHEMA_VERSION # type: ignore from agent.context_engine import ContextEngine # type: ignore # A stale checkout that predates plugin-owned skills is not the diff --git a/tests/test_testing_harness.py b/tests/test_testing_harness.py index 3ad45f8..42b6c06 100644 --- a/tests/test_testing_harness.py +++ b/tests/test_testing_harness.py @@ -180,6 +180,20 @@ def test_no_host_reports_unchecked_and_names_the_required_revision(self) -> None def test_deployed_revision_is_a_full_commit_sha(self) -> None: self.assertRegex(hpk_testing.DEPLOYED_HERMES_REVISION, r"^[0-9a-f]{40}$") + def test_the_deployed_contract_job_checks_out_the_same_revision(self) -> None: + """The workflow cannot read the constant, so its two literals must match it.""" + workflow = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "test.yml" + ).read_text() + job = workflow.split("hermes-deployed-contract:", 1)[1] + literals = re.findall(r"(?:DEPLOYED_HERMES_REVISION:|ref:)\s*([0-9a-f]{40})", job) + + self.assertEqual( + [hpk_testing.DEPLOYED_HERMES_REVISION] * 2, literals, + "test.yml's hermes-deployed-contract env and checkout ref must equal " + "hermes_plugin_kit.testing.DEPLOYED_HERMES_REVISION", + ) + class HostShapeModeTests(unittest.TestCase): """Both branches of the kit's own capability probe must be reachable."""