Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/hermes-drift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ jobs:
path: .hermes-agent-upstream
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Record the upstream revision under test
id: upstream
run: |
Expand Down
16 changes: 16 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ jobs:
ref: ${{ github.sha }}
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Attach the exact trigger to local main
env:
EXPECTED_SHA: ${{ github.sha }}
Expand Down Expand Up @@ -172,6 +176,10 @@ jobs:
ref: ${{ github.sha }}
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Download the immutable release source
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
Expand Down Expand Up @@ -221,6 +229,10 @@ jobs:
ref: ${{ github.sha }}
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Download the immutable release source
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
Expand Down Expand Up @@ -298,6 +310,10 @@ jobs:
ref: ${{ github.sha }}
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Install ORAS
uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1
with:
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Run unittest suite
run: just test

Expand All @@ -25,6 +29,10 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
- name: Build and validate package metadata
run: |
just build
Expand Down Expand Up @@ -54,6 +62,10 @@ jobs:
path: .hermes-agent-deployed
- name: Install the locked toolchain (mise.toml)
uses: jdx/mise-action@82a612c5ab8bcda7cc5db6b7e81a8e7584c3dc6e # v5.2.3
with:
# Pinned so every run, release jobs included, installs the same mise;
# the action still verifies it against mise's signed checksums.
version: "2026.10.18"
# The full suite, not the context-engine module alone. It is green at
# this pin, so widening adds coverage rather than risk.
- name: Run the full contract suite against the deployed Hermes revision
Expand Down
2 changes: 1 addition & 1 deletion hermes_plugin_kit/testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
#:
#: The CI workflow keeps its own literal -- ``actions/checkout`` consumes the
#: ``ref:`` before a Python toolchain exists, so the workflow cannot read this
#: value -- and a harness test asserts the two agree.
#: value.
DEPLOYED_HERMES_REVISION = "939e45c91d751fadd94dcd1b873ac3cb44846213"

#: Parameter names the kit's registrars pass positionally, in order.
Expand Down
18 changes: 4 additions & 14 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,11 @@ prepare-hermes-ref ref dir:
git -C "{{ dir }}" checkout -q FETCH_HEAD
echo "hermes-agent at $(git -C "{{ dir }}" rev-parse HEAD)"

# Build the host's own environment from the host's own lockfile (KTD4).
# Build the host's own environment from the host's own lockfile.
#
# The kit's venv lacks the host's dependencies (ruamel, dotenv, ...), so the
# contract modules died at import there. The host's `pyproject.toml` alone
# does not resolve either, so its `uv.lock` is the only faithful source.
#
# The interpreter is the first one on PATH that satisfies the host's own
# `requires-python`. mise.toml installs one per host generation: 3.14.9 for
# v0.22.5 and later, 3.13 for the pre-upgrade deployed pin (<3.14). The venv
# lives in this repository, so the host checkout is read, never rewritten.
[private]
provision-hermes-host dir venv:
#!/usr/bin/env bash
Expand All @@ -79,13 +74,12 @@ provision-hermes-host dir venv:
mkdir -p "{{ venv }}"
venv="$(cd "{{ venv }}" && pwd)"
test -f "$host/uv.lock" || { echo "no uv.lock in $host; cannot provision the host faithfully" >&2; exit 1; }
spec="$(python3 -c 'import sys, tomllib; print(tomllib.load(open(sys.argv[1], "rb"))["project"]["requires-python"])' "$host/pyproject.toml")"
spec="$({{ uv }} run python -c 'import sys, tomllib; print(tomllib.load(open(sys.argv[1], "rb"))["project"]["requires-python"])' "$host/pyproject.toml")"
py="$({{ uv }} python find --system "$spec")" \
|| { echo "no interpreter on PATH satisfies the host's requires-python $spec; add one to mise.toml" >&2; exit 1; }
echo "hermes host python: $("$py" -V 2>&1) at $py (host requires-python $spec)"
(cd "$host" && UV_PROJECT_ENVIRONMENT="$venv" {{ uv }} sync --frozen --quiet --python "$py")
# --no-deps: the environment stays exactly the host's lock.
{{ uv }} pip install --quiet --python "$venv/bin/python" --no-deps --reinstall-package hermes-plugin-kit .
{{ uv }} pip install --quiet --python "$venv/bin/python" --reinstall-package hermes-plugin-kit .

# Run the full contract suite against the deployed pin.
#
Expand Down Expand Up @@ -123,9 +117,7 @@ test-contract: test-contract-upstream
# Run both contract modules and prove the run actually executed checks.
#
# The exit status alone cannot say whether anything ran: unittest exits 0 when
# every test skips, and on Python before 3.12 also when zero tests are
# collected. So the executed count is the evidence. The suite runs on the
# host's own environment (see provision-hermes-host).
# every test skips. So the executed count is the evidence.
[private]
_run-contract-suite label venv:
#!/usr/bin/env bash
Expand All @@ -137,8 +129,6 @@ _run-contract-suite label venv:
> "$log" 2>&1
echo "EXIT=$?" >> "$log"
set -e
# The whole log, not a tail: the drift classifier reads this output, and a
# tail cut the ERROR headers of an import failure and read as clean.
cat "$log"
grep -qE '^EXIT=0' "$log" || { echo "contract suite failed ({{ label }})" >&2; exit 1; }
ran="$(grep -oE '^Ran [0-9]+' "$log" | grep -oE '[0-9]+' | head -n1)"
Expand Down
6 changes: 1 addition & 5 deletions mise.toml
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
# The one home for this repository's toolchain. CI installs these tools with
# jdx/mise-action from mise.lock, so laptops and CI run the same Python, uv and
# Just. Run `mise trust && mise install --locked` once per checkout; the
# justfile recipes stay the entry points.
min_version = "2026.10.15"
min_version = "2026.10.18"

[tools]
# Must equal the Python in the deployed Hermes image (upstream pm/lock.json).
Expand Down
9 changes: 4 additions & 5 deletions scripts/classify_drift.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,10 @@ def fingerprint(body: str) -> str:
_IMPORT_FAILURE = re.compile(r"^ImportError: Failed to import test module: (\S+)", re.M)
_RAN = re.compile(r"^Ran (\d+) tests?\b", re.M)
_EXIT = re.compile(r"^EXIT=(\d+)\s*$", re.M)
_SKIPPED = re.compile(r"^(?:OK|FAILED) \(.*\bskipped=(\d+)", re.M)


def _did_not_execute(log: str, parsed_failures: int) -> list[str]:
"""Reasons the log is not evidence that the suite ran its checks.

These need no failure block: the blocks are exactly what goes missing when
a run dies early or only part of its log arrives.
"""
reasons = [
f"{module} — failed to import, so none of its checks ran"
for module in dict.fromkeys(_IMPORT_FAILURE.findall(log))
Expand All @@ -110,6 +106,9 @@ def _did_not_execute(log: str, parsed_failures: int) -> list[str]:
reasons.append("no `Ran N tests` line — the suite never completed")
elif int(ran[-1]) == 0:
reasons.append("Ran 0 tests — nothing was checked")
skipped = sum(int(n) for n in _SKIPPED.findall(log))
if skipped:
reasons.append(f"{skipped} contract tests skipped — they checked nothing")
exits = [int(code) for code in _EXIT.findall(log)]
if not exits:
reasons.append("no `EXIT=` line — the run's outcome is unknown")
Expand Down
8 changes: 8 additions & 0 deletions tests/test_drift_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,14 @@ def test_a_non_zero_exit_with_no_parsed_failure_is_new_drift(self) -> None:

self.assertTrue(new)

def test_a_run_that_skipped_contract_tests_is_new_drift(self) -> None:
"""A skipped contract test checked nothing, whatever the exit said."""
log = "\n" + "-" * 70 + "\nRan 17 tests in 0.601s\n\nOK (skipped=3)\nEXIT=0\n"

_, new = classify_drift.classify(log, {})

self.assertTrue(any("skipped" in entry for entry in new), new)

def test_a_clean_run_reports_no_drift(self) -> None:
log = (
"test_x (tests.test_hermes_contract.HermesContractTests.test_x) ... ok\n"
Expand Down
6 changes: 5 additions & 1 deletion tests/test_hermes_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,11 @@ def _try():
run_tool_execution_middleware,
)
from tools.registry import registry # type: ignore
from hermes_state import SCHEMA_VERSION, SessionDB # type: ignore
from hermes_state import SessionDB # type: ignore
try: # v0.22.5 dropped the hermes_state re-export (upstream #126164)
from hermes_state_common import SCHEMA_VERSION # type: ignore
except ImportError:
from hermes_state import SCHEMA_VERSION # type: ignore
from agent.context_engine import ContextEngine # type: ignore

# A stale checkout that predates plugin-owned skills is not the
Expand Down
Loading