From 5021ed399a871744d4ccab92b88f297f5191f8f5 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 28 Oct 2026 16:23:15 +0400 Subject: [PATCH 1/3] =?UTF-8?q?fix(sdk):=20MCP=20approval=20re-entry=20?= =?UTF-8?q?=E2=80=94=20approval=5Fid=20on=20the=20gate,=20digest=20binding?= =?UTF-8?q?,=20canonical=20argument=20bag,=20missing=20/gate=20pre-flight?= =?UTF-8?q?=20(NR-058=20/=20NR-015,=20SDK=20half)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The MCP path is the second enforcement path, and it could not complete an approval round trip at all. `MCPAdapter.call_tool` had no `approval_id` parameter, so a retry after the operator approved landed in `**passthrough` and was forwarded to the MCP SERVER — the gate never saw the grant. The adapter sent neither `business_impact` nor `action_digest`, so the approval row was never digest-bound and the backend could not detect a tampered payload at consume time. The tool arguments travelled only as the legacy `input` field, while the backend resolves the canonical bag as `tool_arguments` → `tool_params` → Null (ADR-060 §4) — so every params-based rule saw "no bag". And the adapter called `runtime.execute` with NO pre-flight: the backend's `/api/v1/execute` runs an existence check on the Redis binding `execution:{id}` and answers 404 EXECUTION_NOT_FOUND when no prior `/gate` minted it, so a direct `adapter.call_tool(...)` never reached the MCP server. ### Fixed - **NR-058 (SDK half)** — `MCPAdapter.call_tool` gains three NAMED keyword-only parameters (`approval_id`, `business_impact`, `action_digest`) placed BEFORE `**passthrough`, so they can never fall into the passthrough and be forwarded to the MCP server. The adapter forwards them into `runtime.execute(...)`. - **NR-058 (SDK half, wire)** — `NullRunRuntime.execute` gains `approval_id` and `tool_arguments` and forwards them into the `execute_kwargs` it already builds, using the same conditional pattern as `business_impact` / `action_digest`: the key is added only when the value is not None, because an absent field means "no grant" and that is not the same wire statement as a null. `Transport.execute` already accepted both and put them on the wire; the runtime was the only missing link. The two new parameters are appended AFTER `tools` so the existing positional order of the parameters above is unchanged. - **NR-015 (SDK half)** — the same argument dict now reaches the wire as BOTH `input` (the tool payload, unchanged) and `tool_arguments` (the canonical bag). No new wire field is invented: `tool_arguments` is the field the backend already resolves. - **Pre-flight** — before `runtime.execute(...)` the adapter now ensures a `/gate` call exists for this logical action: when `nullrun.context.get_server_minted_execution_id()` returns None (no prior `/gate` in this context) it calls `runtime.check_workflow_budget()` first; otherwise it SKIPS it. The pre-flight RESERVES money at the gate, so running it twice would charge the org twice for one logical call — the condition is the same contextvar `runtime.execute` reads to decide whether to reuse the execution id, so the two agree by construction. A refusal from the pre-flight propagates and the MCP client is NOT called. Ordering mirrors `@protect` (`decorators._protect_body`): contextvar stamp → `/gate` pre-flight → `/execute` → body. ### Verification | Check | Result | |---|---| | `pytest tests/test_mcp_approval_wire.py` (pre-fix source) | 10 failed, 5 passed — RED | | `pytest tests/test_mcp_approval_wire.py` (post-fix) | 15 passed — GREEN | | `pytest tests/` | 1928 passed, 12 skipped (pristine HEAD: 1925 collected; +15 new) | | `ruff check src/` (the CI gate) | All checks passed | | `ruff check` on the touched test files | All checks passed | | `ruff format --check` | broken repo-wide on the installed ruff 0.16.8 — 114 pristine-HEAD files would be reformatted; pre-existing, untouched by this change | | `mypy src/` | 5 errors, all in `src/nullrun/instrumentation/auto.py` — identical on pristine HEAD; pre-existing, untouched by this change | New test file: `tests/test_mcp_approval_wire.py` (15 tests). The two existing MCP test modules gained a `check_workflow_budget` method on their runtime doubles — the adapter now performs a pre-flight that a runtime double has to model. --- src/nullrun/runtime.py | 41 +++ src/nullrun/toolbox/mcp.py | 87 +++++- tests/test_mcp_adapter.py | 16 + tests/test_mcp_adapter_gate_closed.py | 16 + tests/test_mcp_approval_wire.py | 402 ++++++++++++++++++++++++++ 5 files changed, 558 insertions(+), 4 deletions(-) create mode 100644 tests/test_mcp_approval_wire.py diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index 8d4febb..c383ea0 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -3607,6 +3607,25 @@ def execute( # otherwise the contextvar flows through (populated by # decorators.py from ``fn.__name__`` before this method runs). tools: tuple[str, ...] | None = None, + # The approved grant, replayed on the wire. + # + # ``Transport.execute`` already accepts and puts ``approval_id`` + # on the wire; this method was the only missing link between + # the caller and the transport. Without it a caller that had an + # operator-approved grant had nowhere to put it, and the + # backend's ``/execute`` fell back to "no grant" — the + # approval could never be consumed (NR-058, SDK half). + # + # Deliberately appended AFTER ``tools`` so the existing + # positional order of the parameters above is unchanged. + approval_id: str | None = None, + # The canonical tool-argument bag (ADR-060 §4). The backend + # resolves the argument bag as ``tool_arguments`` → + # ``tool_params`` → Null, so a caller that only sends the + # legacy ``input`` field leaves every params-based rule with + # "no bag" and leaves the approval decision with no argument + # snapshot to bind to (NR-015, SDK half). + tool_arguments: dict[str, Any] | None = None, ) -> dict[str, Any]: """ Pre-execution policy evaluation via /execute endpoint. @@ -3640,6 +3659,19 @@ def execute( supplied alongside ``business_impact``, the grant is digest-bound; without it, the backend falls back to approval_id-only grant consume. + approval_id: The operator-approved grant, replayed on the + wire. The backend consumes it against the approval row + (and, when the row is digest-bound, against the + server-recomputed ``action_digest``). Absent means "no + grant" — the call is decided by policy alone. + tool_arguments: The canonical tool-argument bag. The + backend resolves the argument bag as + ``tool_arguments`` → ``tool_params`` → Null + (ADR-060 §4), so this is the field the backend + snapshots the arguments from when it binds an + approval decision to the payload. It is an + ADDITIONAL field: the legacy ``input`` payload is + unchanged. Returns: Dict with: @@ -3831,6 +3863,15 @@ def execute( execute_kwargs["business_impact"] = business_impact if action_digest is not None: execute_kwargs["action_digest"] = action_digest + # The approved grant + the canonical argument bag. Both are + # forwarded verbatim to ``Transport.execute``, which puts them + # on the wire. Like the two above, the key is added ONLY when + # the value is not None — an absent field means "no grant" / + # "no bag", which is not the same wire statement as a null. + if approval_id is not None: + execute_kwargs["approval_id"] = approval_id + if tool_arguments is not None: + execute_kwargs["tool_arguments"] = tool_arguments try: result = self._transport.execute(**execute_kwargs) except NullRunBlockedException as exc: diff --git a/src/nullrun/toolbox/mcp.py b/src/nullrun/toolbox/mcp.py index d2e5fa8..364db51 100644 --- a/src/nullrun/toolbox/mcp.py +++ b/src/nullrun/toolbox/mcp.py @@ -323,6 +323,10 @@ def call_tool( self, tool_name: str, arguments: dict[str, Any] | None = None, + *, + approval_id: str | None = None, + business_impact: dict[str, Any] | None = None, + action_digest: str | None = None, **passthrough: Any, ) -> Any: """Call a tool on the underlying MCP client, stamping @@ -335,6 +339,21 @@ def call_tool( client-specific kwargs without changing the public surface. + The three GATE kwargs — ``approval_id``, ``business_impact`` + and ``action_digest`` — are NAMED keyword-only parameters, so + they can never fall into ``**passthrough`` and be forwarded to + the MCP server. They are consumed here and sent to the GATE: + + * ``approval_id`` — the operator-approved grant. Without it + the backend had no way to learn that the call was approved, + so the approval round trip could not be completed + (NR-058, SDK half). + * ``business_impact`` / ``action_digest`` — the typed impact + and its SHA-256 digest. The backend stamps the approval row + with the digest and re-derives it from the live payload at + consume time, so a tampered payload is refused (NR-015, + SDK half). + Gate enforcement: ``call_tool`` ALWAYS routes the invocation through ``runtime.execute(...)`` (the /api/v1/execute gate endpoint) BEFORE the underlying MCP client is called. The @@ -347,6 +366,17 @@ def call_tool( caller can route the user through the approval flow and retry with ``approval_id=``. + PRE-FLIGHT: ``/execute`` is bound to an execution id that a + prior ``/gate`` minted — the backend answers 404 + EXECUTION_NOT_FOUND when the ``execution:{id}`` binding is + missing. The ``@protect`` path gets that ``/gate`` from its own + pre-flight; a DIRECT ``adapter.call_tool(...)`` has no such + wrapper, so the adapter runs the pre-flight ITSELF when no + server-minted execution id is in scope. When one IS in scope + the pre-flight is SKIPPED: that path reserves money at the + gate, and a second ``/gate`` would charge the org twice for + one logical tool call. + B2 (2026-09-30): there is no ungated path. ``runtime=None`` used to mean "skip the round-trip entirely"; it now means "resolve the global runtime". A configuration error at this @@ -410,13 +440,62 @@ def call_tool( # called the MCP server with no gate at all. runtime = self._resolve_runtime() execute_input = arguments if arguments is not None else {} + + # ── Pre-flight: /execute needs a prior /gate ─────────────── + # + # The backend's ``/api/v1/execute`` runs an existence check on + # the Redis binding ``execution:{id}`` and answers 404 + # EXECUTION_NOT_FOUND when no prior ``/gate`` minted it + # (``backend/src/proxy/http/gate/execute.rs``). The + # ``@protect`` path runs ``check_workflow_budget()`` BEFORE + # ``runtime.execute()``; a DIRECT ``adapter.call_tool(...)`` + # has no such wrapper, so without this pre-flight the /execute + # round trip was refused and the MCP server was never called. + # + # The pre-flight is CONDITIONAL, and the condition is the + # server-minted execution id: when one is already in scope a + # ``/gate`` has already run for this logical action, and + # running it again would reserve the budget a second time + # (that path charges money at the gate). The contextvar is + # exactly what ``runtime.execute`` reads to decide whether to + # reuse the id, so the two agree by construction. + # + # A refusal from the pre-flight propagates: the MCP client is + # NOT called. Same ordering and same error handling as the + # ``@protect`` wrapper (``decorators._protect_body``: gate + # pre-flight → /execute → body). + from nullrun.context import get_server_minted_execution_id + + if get_server_minted_execution_id() is None: + runtime.check_workflow_budget() + # Every MCP tool call routed through the runtime contacts # /api/v1/execute unconditionally — there is no # ``mode=`` opt-out for audit bypass. - execute_result = runtime.execute( - tool_name=tool_name, - input_data=execute_input, - ) + # + # ``tool_arguments`` is the canonical argument bag + # (ADR-060 §4): the backend resolves the bag as + # ``tool_arguments`` → ``tool_params`` → Null, so a payload + # that only travels as the legacy ``input`` field leaves every + # params-based rule with "no bag" and leaves the approval + # decision with no argument snapshot to bind to. The SAME dict + # is sent as both — ``input`` is the tool payload and is + # unchanged; ``tool_arguments`` is the canonical view of it. + execute_kwargs: dict[str, Any] = { + "tool_name": tool_name, + "input_data": execute_input, + "tool_arguments": execute_input, + } + # The three gate kwargs are forwarded ONLY when supplied — + # an absent field means "no grant" / "no envelope", which is + # not the same wire statement as a null. + if approval_id is not None: + execute_kwargs["approval_id"] = approval_id + if business_impact is not None: + execute_kwargs["business_impact"] = business_impact + if action_digest is not None: + execute_kwargs["action_digest"] = action_digest + execute_result = runtime.execute(**execute_kwargs) decision = execute_result.get("decision") if decision == "block": # ``NullRunBlockedException`` is raised by diff --git a/tests/test_mcp_adapter.py b/tests/test_mcp_adapter.py index 49e622d..dc4c12a 100644 --- a/tests/test_mcp_adapter.py +++ b/tests/test_mcp_adapter.py @@ -578,6 +578,7 @@ class _StubRuntime: def __init__(self, decision_payload: dict[str, Any] | None = None) -> None: self.calls: list[dict[str, Any]] = [] + self.preflight_calls = 0 self._payload = decision_payload or { "decision": "allow", "decision_source": "gateway", @@ -585,6 +586,21 @@ def __init__(self, decision_payload: dict[str, Any] | None = None) -> None: "policy_version": 1, } + def check_workflow_budget(self) -> None: + """Stand-in for the ``/gate`` pre-flight. + + The adapter runs this before ``/execute`` when no + server-minted execution id is in scope: the backend's + ``/execute`` answers 404 EXECUTION_NOT_FOUND unless a prior + ``/gate`` minted the ``execution:{id}`` binding. The + pre-flight's own behaviour (when it runs, when it is skipped, + what it does on a block) is pinned in + ``tests/test_mcp_approval_wire.py``; here it only has to + exist so the adapter's call site is modelled. + """ + + self.preflight_calls += 1 + def execute(self, **kwargs: Any) -> dict[str, Any]: self.calls.append(kwargs) return dict(self._payload) diff --git a/tests/test_mcp_adapter_gate_closed.py b/tests/test_mcp_adapter_gate_closed.py index f55e8b7..5f9f486 100644 --- a/tests/test_mcp_adapter_gate_closed.py +++ b/tests/test_mcp_adapter_gate_closed.py @@ -92,12 +92,28 @@ class _RecordingRuntime: def __init__(self, payload=None): self.calls: list[dict[str, Any]] = [] + self.preflight_calls = 0 self._payload = payload or { "decision": "allow", "decision_source": "gateway", "explanation": "allow", } + def check_workflow_budget(self): + """Stand-in for the ``/gate`` pre-flight. + + The adapter runs this before ``/execute`` when no + server-minted execution id is in scope — the backend's + ``/execute`` answers 404 EXECUTION_NOT_FOUND without a prior + ``/gate``. It deliberately does NOT touch the ``order`` list + used by ``test_gate_runs_before_the_mcp_client``: that test + pins the ORDER OF THE ENFORCEMENT HOP (``/execute`` before the + MCP call), and the pre-flight is a separate hop with its own + pins in ``tests/test_mcp_approval_wire.py``. + """ + + self.preflight_calls += 1 + def execute(self, **kwargs): self.calls.append(kwargs) return dict(self._payload) diff --git a/tests/test_mcp_approval_wire.py b/tests/test_mcp_approval_wire.py new file mode 100644 index 0000000..29a6a06 --- /dev/null +++ b/tests/test_mcp_approval_wire.py @@ -0,0 +1,402 @@ +"""tests/test_mcp_approval_wire.py — NR-058 / NR-015, the SDK half. + +The MCP path is the SECOND enforcement path (the first is the +``@protect``-decorated local function). Before this module the +adapter could not complete an approval round trip at all: + +* ``MCPAdapter.call_tool`` had no ``approval_id`` parameter, so a + retry after the operator approved landed in ``**passthrough`` and + was forwarded to the MCP SERVER instead of the gate. The gate never + saw the grant. +* The adapter sent neither ``business_impact`` nor ``action_digest``, + so the approval row was never digest-bound and the backend could not + detect a tampered payload when the grant was consumed. +* The tool arguments reached the wire only as the legacy ``input`` + field. The backend's canonical argument bag (ADR-060 §4) is + ``tool_arguments`` → ``tool_params`` → Null, so with + ``tool_arguments`` absent every params-based rule saw "no bag" and + the approval decision had no argument snapshot to bind to. +* The adapter called ``runtime.execute`` with NO pre-flight. The + backend's ``/api/v1/execute`` runs an existence check on the Redis + binding ``execution:{id}`` and answers 404 EXECUTION_NOT_FOUND when + no prior ``/gate`` minted it — so a DIRECT + ``adapter.call_tool(...)`` never reached the MCP server at all. + +The doubles here are deliberately shallow — a mock MCP client and a +fake runtime that records the two wire hops (``check_workflow_budget`` +stands for ``/gate``, ``execute`` for ``/execute``). Nothing between +the adapter and the fake runtime is faked, so a change that drops the +gate kwargs or the pre-flight turns these red rather than being +stubbed around. + +Every test below was run against the PRE-FIX code and failed there; +the failure output is the evidence that the property was not already +satisfied. +""" + +from __future__ import annotations + +import inspect +from typing import Any + +import pytest + +from nullrun.breaker.exceptions import NullRunBlockedException +from nullrun.context import ( + clear_server_minted_execution_id, + set_mcp_tool_context, + set_server_minted_execution_id, +) +from nullrun.toolbox.mcp import MCPAdapter + +# --------------------------------------------------------------------------- +# Doubles +# --------------------------------------------------------------------------- + + +class _Ann: + def __init__(self, read=None, destructive=None, open_world=None): + self.readOnlyHint = read + self.destructiveHint = destructive + self.openWorldHint = open_world + + +class _Tool: + def __init__(self, name, annotations=None): + self.name = name + self.annotations = annotations + + +class _MockMcpClient: + """Mock MCP client that records the tool name, the arguments AND + the extra kwargs. + + The kwargs are recorded separately from the arguments on purpose: + the whole point of T1 is that a gate-only kwarg (``approval_id``) + must not travel with the tool call into the MCP server. + """ + + def __init__(self, tools): + self._tools = {t.name: t for t in tools} + self.calls: list[dict[str, Any]] = [] + + def list_tools(self): + return list(self._tools.values()) + + def call_tool(self, name, arguments=None, **kwargs): + record = {"name": name, "arguments": arguments or {}, "kwargs": kwargs} + self.calls.append(record) + if name not in self._tools: + raise KeyError(f"unknown tool {name!r}") + return f"ok:{name}" + + +class _RecordingRuntime: + """Fake runtime recording both wire hops. + + ``check_workflow_budget`` stands for the ``/gate`` pre-flight; + ``execute`` stands for ``/execute``. Both append to ``order`` so a + test can assert the SEQUENCE, not just the counts. + """ + + def __init__(self, payload=None, order=None): + self.execute_calls: list[dict[str, Any]] = [] + self.gate_calls = 0 + self.order: list[str] = order if order is not None else [] + self._payload = payload or { + "decision": "allow", + "decision_source": "gateway", + "explanation": "allow", + } + + def check_workflow_budget(self): + self.gate_calls += 1 + self.order.append("gate") + + def execute(self, **kwargs): + self.execute_calls.append(kwargs) + self.order.append("execute") + return dict(self._payload) + + +def _inventory(): + return [ + _Tool( + "create_issue", + _Ann(read=False, destructive=True, open_world=True), + ), + _Tool("get_file_contents", _Ann(read=True, destructive=False)), + ] + + +def _adapter(runtime, tools=None): + client = _MockMcpClient(tools if tools is not None else _inventory()) + adapter = MCPAdapter( + server_name="github", + mcp_client=client, + runtime=runtime, + ) + return adapter, client + + +@pytest.fixture(autouse=True) +def _isolate_context(): + """Both contextvars the adapter reads are process-wide, so a stale + value from a previous test would make a test pass for the wrong + reason. + + ``server_minted_execution_id`` is the marker the adapter uses to + decide whether a ``/gate`` pre-flight is needed — a leaked id from + an earlier test would silently skip the pre-flight. + """ + + clear_server_minted_execution_id() + set_mcp_tool_context(tool_class=None, annotations=None) + yield + clear_server_minted_execution_id() + set_mcp_tool_context(tool_class=None, annotations=None) + + +# --------------------------------------------------------------------------- +# T1 — approval_id reaches the GATE and never the MCP client +# --------------------------------------------------------------------------- + + +class TestApprovalIdReachesTheGate: + def test_approval_id_is_forwarded_to_the_execute_body(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}, approval_id="apr-123") + + assert len(runtime.execute_calls) == 1 + body = runtime.execute_calls[0] + assert body["approval_id"] == "apr-123", "the grant must reach the gate" + + def test_approval_id_never_reaches_the_mcp_client(self): + runtime = _RecordingRuntime() + adapter, client = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}, approval_id="apr-123") + + assert len(client.calls) == 1 + kwargs = client.calls[0]["kwargs"] + assert "approval_id" not in kwargs, "a gate field leaked to the server" + + def test_impact_kwargs_never_reach_the_mcp_client(self): + runtime = _RecordingRuntime() + adapter, client = _adapter(runtime) + + adapter.call_tool( + "create_issue", + {"repo": "acme/api"}, + business_impact={"kind": "tool_call", "params": {}}, + action_digest="ab" * 32, + ) + + kwargs = client.calls[0]["kwargs"] + assert "business_impact" not in kwargs + assert "action_digest" not in kwargs + + def test_the_three_gate_kwargs_are_named_parameters(self): + """Structural pin for D1: the three kwargs must be NAMED + parameters of ``call_tool``, not part of ``**passthrough``. + + A named parameter is what makes the property above structural + rather than a matter of what the current implementation + happens to forward. + """ + + params = inspect.signature(MCPAdapter.call_tool).parameters + for name in ("approval_id", "business_impact", "action_digest"): + assert name in params, f"{name} must be a named parameter" + kind = params[name].kind + assert kind is inspect.Parameter.KEYWORD_ONLY, f"{name} is not keyword-only" + + assert params["passthrough"].kind is inspect.Parameter.VAR_KEYWORD + + +# --------------------------------------------------------------------------- +# T2 — business_impact / action_digest reach the gate body +# --------------------------------------------------------------------------- + + +class TestImpactReachesTheGate: + def test_business_impact_is_forwarded_to_the_execute_body(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + impact = { + "kind": "tool_call", + "tool": "create_issue", + "params": {"repo": "acme/api"}, + } + + adapter.call_tool( + "create_issue", + {"repo": "acme/api"}, + business_impact=impact, + ) + + assert runtime.execute_calls[0]["business_impact"] == impact + + def test_action_digest_is_forwarded_to_the_execute_body(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool( + "create_issue", + {"repo": "acme/api"}, + action_digest="cd" * 32, + ) + + assert runtime.execute_calls[0]["action_digest"] == "cd" * 32 + + def test_absent_impact_is_not_invented_on_the_wire(self): + """The adapter must not fabricate a digest the caller did not + supply — a made-up one would be a claim the backend cannot + verify.""" + + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + body = runtime.execute_calls[0] + assert "business_impact" not in body + assert "action_digest" not in body + + +# --------------------------------------------------------------------------- +# T3 — the arguments reach the gate as the canonical bag +# --------------------------------------------------------------------------- + + +class TestArgumentsReachTheGate: + def test_arguments_are_forwarded_as_tool_arguments(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + body = runtime.execute_calls[0] + assert body["tool_arguments"] == {"repo": "acme/api"}, "no canonical bag" + + def test_the_legacy_input_payload_is_unchanged(self): + """``input`` is the tool payload and stays exactly as it was — + the canonical bag is an ADDITIONAL field, not a replacement.""" + + runtime = _RecordingRuntime() + adapter, client = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.execute_calls[0]["input_data"] == {"repo": "acme/api"} + assert client.calls[0]["arguments"] == {"repo": "acme/api"} + + def test_no_arguments_still_yields_an_empty_bag(self): + runtime = _RecordingRuntime() + adapter, _ = _adapter(runtime) + + adapter.call_tool("get_file_contents") + + assert runtime.execute_calls[0]["tool_arguments"] == {} + + +# --------------------------------------------------------------------------- +# T4 — the /gate pre-flight +# --------------------------------------------------------------------------- + + +class TestGatePreflight: + def test_gate_runs_before_execute_when_no_execution_id_is_in_scope(self): + """The backend's ``/execute`` 404s (EXECUTION_NOT_FOUND) unless + a prior ``/gate`` minted the ``execution:{id}`` binding. A + direct ``adapter.call_tool(...)`` has no such binding, so the + adapter must run the pre-flight itself.""" + + order: list[str] = [] + runtime = _RecordingRuntime(order=order) + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.gate_calls == 1, "no /gate ran" + assert order == ["gate", "execute"] + + def test_the_preflight_is_skipped_when_a_gate_already_ran(self): + """The pre-flight RESERVES money at the gate. Running it twice + for one logical call would charge the org twice.""" + + set_server_minted_execution_id("0192f0c1-1111-7000-8000-000000000001") + + order: list[str] = [] + runtime = _RecordingRuntime(order=order) + adapter, _ = _adapter(runtime) + + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.gate_calls == 0, "a second /gate would charge twice" + assert len(runtime.execute_calls) == 1 + assert order == ["execute"] + + def test_a_blocked_preflight_never_reaches_the_mcp_client(self): + """The pre-flight is a gate, not a formality: a block there + stops the call before /execute and before the MCP server.""" + + class _BlockingRuntime(_RecordingRuntime): + def check_workflow_budget(self): + super().check_workflow_budget() + raise NullRunBlockedException( + workflow_id="wf-1", + reason="budget exhausted", + tool_name="create_issue", + ) + + runtime = _BlockingRuntime() + adapter, client = _adapter(runtime) + + with pytest.raises(NullRunBlockedException): + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert runtime.execute_calls == [] + assert client.calls == [] + + +# --------------------------------------------------------------------------- +# T5 — regression: a blocked gate decision stops the MCP call +# --------------------------------------------------------------------------- + + +class TestBlockedGateStillShortCircuits: + def test_a_block_decision_never_reaches_the_mcp_client(self): + runtime = _RecordingRuntime( + payload={ + "decision": "block", + "decision_source": "gateway", + "explanation": "destructive tool blocked", + "workflow_id": "wf-1", + } + ) + adapter, client = _adapter(runtime) + + with pytest.raises(NullRunBlockedException): + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert client.calls == [], "a blocked MCP call reached the server" + + def test_a_require_approval_decision_never_reaches_the_mcp_client(self): + runtime = _RecordingRuntime( + payload={ + "decision": "require_approval", + "decision_source": "gateway", + "explanation": "needs approval", + "workflow_id": "wf-1", + "approval_id": "apr-1", + } + ) + adapter, client = _adapter(runtime) + + with pytest.raises(NullRunBlockedException): + adapter.call_tool("create_issue", {"repo": "acme/api"}) + + assert client.calls == [] From 4bc9322cd82b243c72ee1185814c5f6be8ec343a Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 28 Oct 2026 19:02:15 +0400 Subject: [PATCH 2/3] =?UTF-8?q?fix(budget):=20LLM-=D0=BF=D1=83=D1=82=D1=8C?= =?UTF-8?q?=20=D1=80=D0=B5=D0=B7=D0=B5=D1=80=D0=B2=D0=B8=D1=80=D1=83=D0=B5?= =?UTF-8?q?=D1=82=20=D1=81=D1=82=D0=BE=D0=B8=D0=BC=D0=BE=D1=81=D1=82=D1=8C?= =?UTF-8?q?=20=D0=B7=D0=B2=D0=BE=D0=BD=D0=BA=D0=B0=20=D0=94=D0=9E=20=D0=B7?= =?UTF-8?q?=D0=B0=D0=BF=D1=80=D0=BE=D1=81=D0=B0=20=D0=BA=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B0=D0=B9=D0=B4=D0=B5=D1=80=D1=83=20(NR-059)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Проблема: на zero-code пути (транспорт httpx/requests, без @protect и без set_call_context) SDK отправлял llm_call ТОЛЬКО ПОСЛЕ ответа провайдера. Событие — единственное, что двигает счётчик бюджета, поэтому звонок, который потолок должен был остановить, был учтён, а не остановлен. Контроль-центр показывал сумму уже потраченного. Что сделано: - check_workflow_budget() принимает model / estimated_tokens / max_output_tokens / use_gate_cache и возвращает id резерва. - Транспорт httpx (sync + async) и requests делают пре-флайт ДО запроса к провайдеру: читают модель, размер промпта и потолок вывода из тела запроса и резервируют их. Отказ гейта бросается наружу — запрос к провайдеру НЕ уходит. - Резерв едет с событием явно (track(..., reservation_id=...)), а не через контеквар: контеквар — один слот на задачу, два параллельных звонка поменялись бы резервами. - Звонок, который не дал usage (ошибка сети, пустое тело, нет блока usage), освобождает свой резерв нулевым /track — иначе резерв держал бы бюджет до воркера reclaim. - Если в области уже есть свежий резерв (путь @protect резервирует один раз на всю область), пре-флайт НЕ делается: второй резерв осиротел бы первый. - Оценка входа: ~3 символа на токен, CJK — 1 токен на символ, base64 не считается текстом, потолок 32768 токенов. Потолок вывода объявляется только если его объявил вызывающий: без него верхней границы нет, а цена по каталожному максимуму заморозила бы месячный бюджет на одном звонке. - README: правда по каждому пути — что потолок останавливает, а что может остановить только следующий звонок. Проверено: - 1948 тестов проходят, 12 пропущено; ruff чист; mypy — базовая линия (5 существовавших ошибок в auto.py, к этой правке отношения не имеют). - Новое: tests/test_zero_code_preflight.py (20 тестов). - Мутациями доказано, что тесты ловят поломку: A (потеря резерва) — 6 красных; B (отключённое освобождение) — 3 красных; C (однопоточная передача резерва) — конкурентный тест красный, для этого в тесте стоит барьер, без него тест был вакуумным; D (игнор резерва области) — красный. - Пин в tests/test_audit_p0_27_operation_id_hoist.py пришлось переписать: он пиннил старую сигнатуру def check_workflow_budget(self) -> None. Теперь срез идёт по границе метода, а не по сигнатуре. --- README.md | 42 +- src/nullrun/instrumentation/auto.py | 442 +++++++++++++- src/nullrun/instrumentation/auto_requests.py | 56 +- src/nullrun/runtime.py | 101 +++- tests/test_audit_p0_27_operation_id_hoist.py | 12 +- tests/test_zero_code_preflight.py | 592 +++++++++++++++++++ 6 files changed, 1210 insertions(+), 35 deletions(-) create mode 100644 tests/test_zero_code_preflight.py diff --git a/README.md b/README.md index 47fb498..fe91381 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,14 @@ Existing observability tools tell you **after** the fact. NullRun enforces **bef | Without NullRun | With NullRun | |---|---| -| Agent calls `gpt-4o` 10,000 times → surprise $5,000 invoice | Hard budget cap → SDK blocks at 402 before invocation | +| Agent calls `gpt-4o` 10,000 times → surprise $5,000 invoice | Hard budget cap → the SDK reserves the call's expected cost at the gate and refuses at 402 before the provider is called | | Agent runs `bash rm -rf /` | Tool policy → SDK blocks at 403 before execution | | Sensitive action with no human in the loop | Approval flow → SDK pauses and waits for WS `approval_resolved` push | | Cost & calls scattered across 4 libraries | Single source of truth: per-org, per-workflow, per-execution | | Runaway SDK loop calling `/gate` without `/track` | Per-reservation rate cap → 402 budget error (see `docs/errors/NR-R001.md`) | +> The budget cap is enforced **before** the action — but only on the paths where the SDK can reach the provider request before it leaves. [What the cap stops, per path](#what-the-hard-budget-cap-stops--and-what-it-does-not) says exactly which is which. The short version: the zero-code `httpx`/`requests` path and the `@protect` path stop the current call; a bare LangChain callback can only stop the next one, because the provider has already been called by then. + --- ## Features @@ -269,6 +271,44 @@ exceptions are refusals — use `on_denied="message"`. --- +## What the hard budget cap stops — and what it does not + +NullRun enforces before the action. Which *action* depends on the path, and the +difference is worth 5 minutes of reading before you rely on the cap. + +| Path | Where the gate runs | What the cap does | +|---|---|---| +| `@protect` (with or without `set_call_context`) | before the wrapped function body | reserves the declared model / token estimate / output cap; a refusal raises and the body never runs | +| Zero-code `httpx` / `requests` transport | **before the provider request leaves the process** | the SDK reads the model, the prompt size and the caller's output cap out of the request body and reserves them; a refusal raises and the provider is never called | +| Zero-code path where the caller declared **no** output cap | before the provider request leaves | the input side is reserved, the output side is not — there is no upper bound to reserve. The call still counts after the fact, and the cap stops the **next** call | +| LangChain callback with no `@protect` | after the provider answered | the spend is counted correctly, but the cap can only stop the **next** call — a callback cannot un-send a request that already reached the provider | +| Tool call | before the tool runs | reserved and refused before execution | + +**The SDK never sends a price.** It sends *usage* — how many tokens — and the +server turns that into cents from its own catalog. A tampered client cannot +inflate the bill by lying about cost (ADR-068). + +**The estimate is an estimate.** With no tokenizer dependency, the zero-code path +estimates the prompt at ~3 characters per token, counts CJK/Hangul at one token +per character, skips base64 image blobs (the provider bills those as tiles, not +as text), and caps the input estimate at 32 768 tokens. + +**Under-estimating is money-safe; over-estimating is temporary.** If the real +call costs more than the reserve, the backend books the **actual** spend on +`/track` anyway and the org sees a `CONSUME_OVERBUDGET` warning — the money is +counted, the cap still holds. If the reserve was larger than the call, the +difference is refunded when the call settles. + +**A call that never billed does not hold your budget.** If the provider request +fails, the answer carries no usage block, or the body is empty, the SDK releases +the reserve immediately instead of leaving it on the period counter until the +reclaim worker runs. + +**A refusal is a refusal.** The typed exceptions (`NullRunBudgetError` and +friends) are raised from the pre-flight, so the provider request is never sent. + +--- + ## How NullRun compares | | **NullRun** | LangChain callbacks | Helicone | Portkey | OpenLLMetry | diff --git a/src/nullrun/instrumentation/auto.py b/src/nullrun/instrumentation/auto.py index e72a00c..b4c3026 100644 --- a/src/nullrun/instrumentation/auto.py +++ b/src/nullrun/instrumentation/auto.py @@ -40,10 +40,11 @@ import json import logging import os +import re import threading from collections import OrderedDict from collections.abc import Callable -from typing import Any +from typing import Any, NamedTuple import httpx @@ -691,6 +692,352 @@ def _extract_model_from_request_body(request: httpx.Request) -> str | None: return None +# --------------------------------------------------------------------------- +# ADR-101 / NR-059 — the zero-code path declares what the call will cost +# --------------------------------------------------------------------------- +# +# The pre-flight on this path has to answer "will THIS call fit?", and +# the provider request body is the only place the answer exists: the +# zero-code path never goes through `set_call_context(...)`, so the +# model, the prompt size and the caller's output cap are all read off +# the body the provider is about to receive. +# +# None of these numbers are PRICES. ADR-068: the server owns the price, +# the client owns only the USAGE. What follows says how many tokens the +# call is expected to use; the backend turns them into cents from its +# own catalog. + +# Output-cap field names, per vendor dialect. OpenAI's chat API takes +# `max_tokens` (legacy) or `max_completion_tokens` (current); Anthropic +# takes `max_tokens`; the Google GenAI dialect takes `maxOutputTokens`. +# The FIRST match wins, so the current OpenAI spelling is checked +# before the legacy one. +_MAX_OUTPUT_TOKEN_FIELDS = ( + "max_completion_tokens", + "max_tokens", + "maxOutputTokens", +) + +# Characters per token for latin-script text. 3 is deliberately +# CONSERVATIVE (an over-estimate): the usual guidance is ~4 characters +# per token for English prose, and code / JSON run denser. Over-reserving +# is refunded when the call is settled; under-reserving costs a +# CONSUME_OVERBUDGET refusal on the /track. +_CHARS_PER_TOKEN = 3 + +# Ceiling on the input estimate. A multimodal request carries the image +# as a base64 blob — megabytes of characters the provider bills as image +# tiles, not as text tokens. `_looks_like_data_blob` skips those, but a +# ceiling keeps a pathological body from producing a reserve that would +# strand a month's budget on one call. A prompt above the ceiling is +# UNDER-estimated: the consume then overruns the reserve, the backend +# books the real spend anyway (the overage path), and the caller sees +# a CONSUME_OVERBUDGET warning rather than a wrong bill. +_MAX_ESTIMATED_INPUT_TOKENS = 32_768 + +# CJK / Hangul runs. Counted at one token per character — every major +# vendor's vocabulary encodes a CJK character as one to two tokens, so +# one is the safe side. Latin text is counted by characters instead. +_CJK_RUN_RE = re.compile( + "[" + "぀-ヿ" # hiragana + katakana + "㐀-䶿" # CJK extension A + "一-鿿" # CJK unified ideographs + "가-힯" # hangul syllables + "]+" +) + + +def _looks_like_data_blob(text: str) -> bool: + """True for a long, whitespace-free string — a base64 image payload, + a minified blob. NOT natural-language prompt text. + + The provider bills an image as tiles, not as characters; counting a + megabyte of base64 as text tokens would inflate the reserve by + orders of magnitude and could strand a month's budget on one call. + The probe reads only the head and the tail, so the cost does not + scale with the payload. + """ + if len(text) <= 4096: + return False + probe = text[:2048] + text[-2048:] + return not any(ch.isspace() for ch in probe) + + +def _estimate_tokens(text: str) -> int: + """Estimate the token count of ``text`` without a tokenizer. + + No tokenizer dependency is added: the number only sizes a + RESERVE, and the authoritative count arrives with the provider's + usage on /track. + """ + if not text: + return 0 + cjk = sum(len(run) for run in _CJK_RUN_RE.findall(text)) + other = len(text) - cjk + return cjk + (other + _CHARS_PER_TOKEN - 1) // _CHARS_PER_TOKEN + + +def _collect_prompt_text(value: Any, out: list[str]) -> None: + """Walk a decoded request body and collect the text the provider + will read: messages, system prompt, tool schemas, file contents. + + The ``model`` key is skipped — the model id is not part of the + prompt. + """ + if isinstance(value, str): + if not _looks_like_data_blob(value): + out.append(value) + elif isinstance(value, dict): + for key, item in value.items(): + if key == "model": + continue + _collect_prompt_text(item, out) + elif isinstance(value, (list, tuple)): + for item in value: + _collect_prompt_text(item, out) + + +class LlmRequestProfile(NamedTuple): + """What a provider request body says about the call it announces.""" + + model: str + estimated_input_tokens: int + declared_max_output_tokens: int | None + + +def _read_llm_request_profile(request: httpx.Request) -> LlmRequestProfile | None: + """Read the model, the prompt size and the caller's output cap out + of an httpx LLM request body. + """ + try: + raw = request.content + except Exception: # noqa: BLE001 — a streamed body we cannot read + return None + return _profile_from_body_bytes(raw) + + +def _read_prepared_request_profile(request: Any) -> LlmRequestProfile | None: + """Same, for a ``requests.PreparedRequest`` — the `requests` path's + request object. Its body lives on ``.body`` (bytes or str), not on + ``.content``. + """ + body = getattr(request, "body", None) + if isinstance(body, str): + raw: bytes | None = body.encode("utf-8", "replace") + elif isinstance(body, (bytes, bytearray)): + raw = bytes(body) + else: + raw = None + return _profile_from_body_bytes(raw) + + +def _profile_from_body_bytes(raw: bytes | None) -> LlmRequestProfile | None: + """Read the model, the prompt size and the caller's output cap out + of an LLM request body. + + Returns ``None`` when the body is not a JSON object carrying a + ``model`` — i.e. when the request is not a model invocation we can + price (a ``GET /v1/models``, a file upload). Those requests produce + no ``llm_call`` event either, so there is nothing to reserve for + them and nothing to price. + """ + if not raw: + return None + try: + payload = json.loads(raw) + except (json.JSONDecodeError, ValueError, UnicodeDecodeError): + return None + if not isinstance(payload, dict): + return None + model = payload.get("model") + if not isinstance(model, str) or not model: + return None + + declared_max_output_tokens: int | None = None + for field in _MAX_OUTPUT_TOKEN_FIELDS: + value = payload.get(field) + if isinstance(value, int) and not isinstance(value, bool) and value > 0: + declared_max_output_tokens = value + break + + chunks: list[str] = [] + _collect_prompt_text(payload, chunks) + estimated_input_tokens = min( + _estimate_tokens("".join(chunks)), + _MAX_ESTIMATED_INPUT_TOKENS, + ) + return LlmRequestProfile( + model=model, + estimated_input_tokens=estimated_input_tokens, + declared_max_output_tokens=declared_max_output_tokens, + ) + + +def _ambient_reservation_in_scope() -> str | None: + """The reservation the CURRENT scope already holds, or ``None``. + + The ``@protect`` path calls ``/gate`` once for the whole protected + scope, and the LLM call inside that scope settles THAT + reservation — the transport's cost event carries it as the + server-minted execution id. If the transport minted a second + reservation for the same call, the scope's reservation would never + be settled and would hold budget until the reclaim worker ran. + + So: when the scope already holds a FRESH reservation, the transport + does not pre-flight a second time. The scope's own reservation is + the call's reservation. + + "Fresh" is the same 295 s window the runtime uses to drop a stale + server-minted capture (``SERVER_MINTED_RESERVATION_MAX_AGE_SECONDS``) + — the two must agree, or the transport would skip a pre-flight for + a reservation the runtime considers stale. + """ + try: + from nullrun.context import ( + get_server_minted_execution_id, + get_server_minted_reservation_at, + ) + except Exception: # pragma: no cover — defensive + return None + smid = get_server_minted_execution_id() + if not smid: + return None + try: + import time as _time + + from nullrun.runtime import SERVER_MINTED_RESERVATION_MAX_AGE_SECONDS + + age = _time.monotonic() - get_server_minted_reservation_at() + except Exception: # pragma: no cover — defensive + return None + if age >= SERVER_MINTED_RESERVATION_MAX_AGE_SECONDS: + return None + return smid + + +def _preflight_llm_call( + runtime: Any, + profile: LlmRequestProfile | None, +) -> tuple[str | None, str | None]: + """Reserve the money for ONE provider request before it leaves. + + Returns ``(reservation_id, model)``. ``reservation_id`` is ``None`` + when no reservation was made (the scope already holds one, no + workflow bound to the key, the gate failed open, or the body + carried nothing to price); ``model`` is the model's name when the + profile carried one, so the caller can release the reservation with + a model-bearing event. + + A REFUSAL raises out of here, and that is the point: the provider + request must not be sent when the budget says no. The transport + failure posture is unchanged from the rest of the SDK — an + unreachable gate fails OPEN (a dead backend must not freeze the + agent), and the call is then unreserved and its cost event is + dropped, exactly as before this change. + """ + if profile is None: + return (None, None) + ambient = _ambient_reservation_in_scope() + if ambient: + # The scope already holds a reservation (the `@protect` gate + # call). Reuse it — a second reserve would strand the first. + return (None, profile.model) + check = getattr(runtime, "check_workflow_budget", None) + if not callable(check): + # A test double or a runtime that predates the pre-flight. + return (None, profile.model) + reservation_id = check( + model=profile.model, + # 0 means "the body carried no prompt" — the historical + # one-token reserve is the smallest honest number then. + estimated_tokens=profile.estimated_input_tokens or None, + max_output_tokens=profile.declared_max_output_tokens, + # Each provider call needs its OWN reservation. The chain-mode + # cache replays a previous response — including its reservation + # id — so a cached "allow" would hand this call the previous + # call's reservation. + use_gate_cache=False, + ) + if not isinstance(reservation_id, str) or not reservation_id: + return (None, profile.model) + return (reservation_id, profile.model) + + +def _track_with_reservation( + runtime: Any, + event: dict[str, Any], + reservation_id: str | None, +) -> None: + """``runtime.track`` with the reservation attached — but ONLY when + there is one. + + The call shape is unchanged for the no-reservation case, so a + duck-typed runtime (a test double, an older SDK) that implements + ``track(event)`` keeps working. Passing a keyword it does not + accept would raise, and the transport swallows track failures — + which would silently drop the cost event. + """ + if reservation_id: + runtime.track(event, reservation_id=reservation_id) + else: + runtime.track(event) + + +def _release_reservation( + runtime: Any, + reservation_id: str | None, + model: str | None, + host: str, + reason: str, +) -> None: + """Settle a reservation that no usage event will ever consume. + + A reservation is MONEY: the backend books it into the period + counter when it is minted, and takes it back when the matching + /track consumes it. If the call failed, or the provider's answer + carried no usage block, no usage event follows — and the reserve + would sit on the counter until the reclaim worker runs, holding + budget the org never spent. One zero-token /track settles it now. + + A release without a model is skipped: the backend refuses an + ``llm_call`` with no model, and a refused event settles nothing. + The reclaim worker is the backstop for that case. + """ + if not reservation_id or not model: + return + try: + _track_with_reservation( + runtime, + { + "type": "llm_call", + "provider": _provider_label(host), + "host": host, + "model": model, + "tokens": 0, + "input_tokens": 0, + "output_tokens": 0, + "has_usage": False, + "metadata": { + "tracked": True, + "reservation_released": True, + "release_reason": reason, + }, + # Deterministic in the reservation id, so a retry of the + # SAME call still collapses in the dedup LRU, while two + # different calls never share one. + "_fingerprint": f"nullrun-release-{reservation_id}", + }, + reservation_id, + ) + except Exception as e: # pragma: no cover — defensive + logger.warning( + "NullRun transport: releasing reservation %s failed: %s", + reservation_id, + e, + ) + + def _match_extractor(host: str) -> Callable[[bytes, int], ExtractedUsage | None] | None: """Return the extractor for `host`, or None if the host is not a known LLM endpoint. We match exact host first, then any subdomain (e.g. @@ -807,7 +1154,24 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: extractor = _match_extractor(host) if extractor is None: return self._inner.handle_request(request) - response = self._inner.handle_request(request) + # ADR-101: reserve what THIS call is expected to cost BEFORE it + # reaches the provider. A refusal raises out of here, so the + # hard budget cap stops THIS call, not the next one. + reservation_id, reserved_model = _preflight_llm_call( + self._runtime, _read_llm_request_profile(request) + ) + try: + response = self._inner.handle_request(request) + except Exception: + # The provider request failed. No usage event will follow, + # so the reservation would sit on the period counter until + # the reclaim worker runs — holding budget the org never + # spent. Settle it now. + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "request_failed", + ) + raise try: # MAX_RESPONSE_BYTES for tracking purposes. Above the cap # we skip tracking (the user still gets the full body via @@ -823,6 +1187,10 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: # because usage wasn't extractable). Drain the body # so callers don't see a half-consumed response. _emit_streaming_skipped(self._runtime, request, host) + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "body_exceeded_cap", + ) logger.debug( "NullRun transport: response from %s exceeded %d bytes; " "skipping usage tracking", @@ -834,14 +1202,31 @@ def handle_request(self, request: httpx.Request) -> httpx.Response: return response except Exception as e: # pragma: no cover — defensive logger.debug("NullRun transport: failed to read body: %s", e) + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "body_unreadable", + ) return response if not body: + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "empty_body", + ) return response usage = extractor(body, response.status_code) if usage is None: + # The provider answered but its body carried no usage block. + # Nothing to bill, and no event will consume the reserve. + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "usage_not_extractable", + ) # Reconstruct the response so callers can still consume the body. return self._rebuild(response, body, request) - self._emit(request, host, usage, body, response.status_code) + self._emit( + request, host, usage, body, response.status_code, + reservation_id=reservation_id, + ) return self._rebuild(response, body, request) @staticmethod @@ -859,6 +1244,8 @@ def _emit( usage: ExtractedUsage, body: bytes, status: int, + *, + reservation_id: str | None = None, ) -> None: # for ``model`` (response body lacked the field — observed for # some OpenAI Responses-API and Anthropic streaming edge cases) @@ -877,8 +1264,10 @@ def _emit( or _extract_model_from_request_body(request) ) try: - self._runtime.track( - _build_llm_call_event(host, usage, model_for_event) + _track_with_reservation( + self._runtime, + _build_llm_call_event(host, usage, model_for_event), + reservation_id, ) except Exception as e: logger.debug("NullRun transport: track failed: %s", e) @@ -912,7 +1301,19 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: extractor = _match_extractor(host) if extractor is None: return await self._inner.handle_async_request(request) - response = await self._inner.handle_async_request(request) + # ADR-101: same pre-flight as the sync path — reserve the + # expected cost of THIS call before the provider request. + reservation_id, reserved_model = _preflight_llm_call( + self._runtime, _read_llm_request_profile(request) + ) + try: + response = await self._inner.handle_async_request(request) + except Exception: + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "request_failed", + ) + raise try: body = await _aread_body_with_cap(response, MAX_RESPONSE_BYTES) if body is None: @@ -920,6 +1321,10 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: # so the call counts toward coverage (host known # tracked: false because usage wasn't extractable). _emit_streaming_skipped(self._runtime, request, host) + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "body_exceeded_cap", + ) logger.debug( "NullRun transport: async response from %s exceeded %d bytes; " "skipping usage tracking", @@ -931,13 +1336,28 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: return response except Exception as e: # pragma: no cover — defensive logger.debug("NullRun transport: failed to read async body: %s", e) + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "body_unreadable", + ) return response if not body: + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "empty_body", + ) return response usage = extractor(body, response.status_code) if usage is None: + _release_reservation( + self._runtime, reservation_id, reserved_model, host, + "usage_not_extractable", + ) return self._rebuild(response, body, request) - self._emit(request, host, usage, body, response.status_code) + self._emit( + request, host, usage, body, response.status_code, + reservation_id=reservation_id, + ) return self._rebuild(response, body, request) @staticmethod @@ -955,6 +1375,8 @@ def _emit( usage: ExtractedUsage, body: bytes, status: int, + *, + reservation_id: str | None = None, ) -> None: # request-body model fallback (lines 882-885) so async # Anthropic / OpenAI streaming clients without ``usage.model`` @@ -973,8 +1395,10 @@ def _emit( or _extract_model_from_request_body(request) ) try: - self._runtime.track( - _build_llm_call_event(host, usage, model_for_event) + _track_with_reservation( + self._runtime, + _build_llm_call_event(host, usage, model_for_event), + reservation_id, ) except Exception as e: logger.debug("NullRun transport: async track failed: %s", e) diff --git a/src/nullrun/instrumentation/auto_requests.py b/src/nullrun/instrumentation/auto_requests.py index dc4fa88..dd759b0 100644 --- a/src/nullrun/instrumentation/auto_requests.py +++ b/src/nullrun/instrumentation/auto_requests.py @@ -48,7 +48,11 @@ from nullrun.instrumentation.auto import ( _fingerprint_for, _match_extractor, + _preflight_llm_call, _provider_label, + _read_prepared_request_profile, + _release_reservation, + _track_with_reservation, ) logger = logging.getLogger(__name__) @@ -87,6 +91,8 @@ def _emit_to_runtime( usage: dict[str, Any], body: bytes, status: int, + *, + reservation_id: str | None = None, ) -> None: """Single-source-of-truth for emitting an LLM call event from any transport. Kept in this module (rather than re-exported from @@ -97,7 +103,8 @@ def _emit_to_runtime( the SDK's `_match_extractor` identified a known provider. """ try: - runtime.track( + _track_with_reservation( + runtime, { "type": "llm_call", "provider": _provider_label(host), @@ -112,7 +119,8 @@ def _emit_to_runtime( }, "raw_usage": usage, "_fingerprint": _fingerprint_for(host, body, status), - } + }, + reservation_id, ) except Exception as e: logger.debug("NullRun requests transport: track failed: %s", e) @@ -222,14 +230,41 @@ def _wrapped_send(self: Any, request: Any, **kwargs: Any) -> Any: # tagged `metadata.streaming_skipped: true` so the # backend can still see the call in coverage. if _is_streaming_request(request, kwargs): + # ADR-101: the pre-flight still runs for a streamed + # call — that is what lets the budget cap stop THIS + # call before it leaves. The reserve is released right + # after: a streamed body is never read, so its usage + # is never known and nothing will settle it later. + stream_reservation: str | None = None + stream_model: str | None = None + if _match_extractor(host) is not None: + stream_reservation, stream_model = _preflight_llm_call( + runtime, _read_prepared_request_profile(request) + ) _emit_streaming_skipped_to_runtime(runtime, request, host) + _release_reservation( + runtime, stream_reservation, stream_model, host, + "streaming_skipped", + ) return _orig_session_send(self, request, **kwargs) extractor = _match_extractor(host) if extractor is None: return _orig_session_send(self, request, **kwargs) - response = _orig_session_send(self, request, **kwargs) + # ADR-101: reserve the expected cost of THIS call before the + # provider request leaves. A refusal raises out of here. + reservation_id, reserved_model = _preflight_llm_call( + runtime, _read_prepared_request_profile(request) + ) + try: + response = _orig_session_send(self, request, **kwargs) + except Exception: + _release_reservation( + runtime, reservation_id, reserved_model, host, + "request_failed", + ) + raise try: # `response.content` is the fully-materialized bytes. # For non-streaming responses this is the body; for @@ -240,12 +275,24 @@ def _wrapped_send(self: Any, request: Any, **kwargs: Any) -> Any: logger.debug( "NullRun requests transport: failed to read body: %s", e ) + _release_reservation( + runtime, reservation_id, reserved_model, host, + "body_unreadable", + ) return response if not body: + _release_reservation( + runtime, reservation_id, reserved_model, host, + "empty_body", + ) return response usage = extractor(body, response.status_code) if usage is None: + _release_reservation( + runtime, reservation_id, reserved_model, host, + "usage_not_extractable", + ) return response # Mark BEFORE the track call so a track-failure (network @@ -261,7 +308,8 @@ def _wrapped_send(self: Any, request: Any, **kwargs: Any) -> Any: # is deduped by fingerprint at the track sink). pass _emit_to_runtime( - runtime, request, host, usage, body, response.status_code + runtime, request, host, usage, body, response.status_code, + reservation_id=reservation_id, ) return response diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index c383ea0..88c51ea 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -2235,12 +2235,42 @@ def sensitive_fail_open_enabled(self) -> bool: ) return True - def check_workflow_budget(self) -> None: + def check_workflow_budget( + self, + *, + model: str | None = None, + estimated_tokens: int | None = None, + max_output_tokens: int | None = None, + use_gate_cache: bool = True, + ) -> str | None: """ Pre-flight budget check via /api/v1/gate. Called from @protect before the wrapped function runs, so a workflow with no remaining budget never gets to spend tokens. + Returns the server-minted reservation id — the id the backend + settles when the matching `/track` arrives — or ``None`` when no + reservation was minted (no workflow bound, the gate failed open, + or the backend answered without one). + + Keyword arguments (ADR-101): + + model: overrides the call-context model. The zero-code LLM + path reads the model out of the provider request body, + which is the only place it exists on that path. + estimated_tokens: overrides the hardcoded ``1``. ``1`` was + the right answer to the binary question the pre-flight + used to ask ("is ANY budget left?"); it is the wrong + answer to "will THIS call fit?", which is what the + zero-code path needs in order to reserve the call's own + cost instead of one token's worth of it. + max_output_tokens: overrides the call-context output cap. + use_gate_cache: set False by a path that needs its OWN + reservation per call. The chain-mode cache replays a + previous response — including its reservation id — so a + cached "allow" would hand a second LLM call the FIRST + call's reservation. + Decision → exception mapping: "allow" → return "block" / "require_approval" / "deny" → typed by the @@ -2347,7 +2377,7 @@ def check_workflow_budget(self) -> None: metrics.inc_runtime("skip_budget_allowed_in_prod") except Exception: # noqa: BLE001 pass - return + return None # Non-production. Pre-fix this was `logger.debug`, which # means a test suite running the whole budget path with # the bypass on leaves no trace anywhere: the tests go @@ -2379,7 +2409,7 @@ def check_workflow_budget(self) -> None: metrics.inc_runtime("skip_budget_used_non_prod") except Exception: # noqa: BLE001 — metrics never gate pass - return + return None # Bump the ``check_calls`` counter so the dashboard can show # the rate of pre-flight budget checks and the operator can @@ -2459,14 +2489,14 @@ def check_workflow_budget(self) -> None: metrics.inc_runtime("budget_preflight_no_workflow_total") except Exception: # noqa: BLE001 — metrics never gate pass - return + return None # Use the real model name from the call context if the user set # it via `set_call_context(model=...)`. Sending `None` is # fine — backend `calculate_projected_cost` defaults when # model is unset, and tool_block enforcement on /gate is # best-effort when no tools are sent. - call_model = get_call_model() + call_model = model if model is not None else get_call_model() call_tools = get_call_tools() # Chain context for soft-mode enforcement. @@ -2479,7 +2509,12 @@ def check_workflow_budget(self) -> None: "operation_id": op_id, "check_type": "llm", "model": call_model, # may be None if user didn't set it - "estimated_tokens": 1, + # The input side of the reserve. ``1`` is the historical + # value and it is deliberately kept for the @protect path: + # the pre-flight there answers "is any budget left?", not + # "what will this call cost?". The zero-code LLM path + # passes the request's own estimate (ADR-101). + "estimated_tokens": estimated_tokens if estimated_tokens is not None else 1, "stream": False, } @@ -2489,7 +2524,10 @@ def check_workflow_budget(self) -> None: # reserve only when it has BOTH this number and its own per-model # catalog cap (ADR-016 §2.1), and a null would be one signal # short of that pair rather than an honest "I don't know". - call_max_output = get_call_max_output_tokens() + call_max_output = ( + max_output_tokens if max_output_tokens is not None + else get_call_max_output_tokens() + ) if call_max_output is not None: check_req["max_output_tokens"] = call_max_output @@ -2583,6 +2621,7 @@ def check_workflow_budget(self) -> None: cache_key: tuple[str, str | None, str | None, int] | None = None cache_enabled = ( chain_id is not None + and use_gate_cache and not os.environ.get("NULLRUN_GATE_CACHE_DISABLE", "").strip() == "1" ) if cache_enabled: @@ -2591,7 +2630,6 @@ def check_workflow_budget(self) -> None: if cached is not None and (time.monotonic() - cached[0]) < _GATE_CACHE_TTL_SECONDS: # Cache hit within TTL — reuse the response without a response = cached[1] - _capture_server_minted_execution_id(response) else: # Cache miss or expired — go to the server, then store. try: @@ -2631,7 +2669,7 @@ def check_workflow_budget(self) -> None: self._note_backend_attempt(False) logger.warning(f"check_workflow_budget: /gate unavailable, failing open: {exc}") metrics.inc_runtime("gate_fail_open_total") - return + return None except ValueError as exc: # `Transport.check` ends in `response.json()`. A # body that is not JSON — proxy error page, @@ -2676,9 +2714,15 @@ def check_workflow_budget(self) -> None: self._note_backend_attempt(False) logger.warning(f"check_workflow_budget: /gate unavailable, failing open: {exc}") metrics.inc_runtime("gate_fail_open_total") - return + return None - _capture_server_minted_execution_id(response) + # The reservation id the backend minted for THIS call. It is + # captured into the server-minted contextvar (the legacy + # single-slot carrier) AND returned to the caller, which needs + # it explicitly: the zero-code LLM path runs one reservation per + # provider request and cannot rely on a contextvar that two + # concurrent calls in the same task would overwrite. + reservation_id = _capture_server_minted_execution_id(response) decision = self._require_gate_decision(response) decision_source = response.get("decision_source", DecisionSource.GATEWAY) @@ -2699,7 +2743,7 @@ def check_workflow_budget(self) -> None: f"{decision_source!r}, treating as transport error" ) metrics.inc_runtime("gate_fail_open_total") - return + return None if decision == "block": reasons = response.get("explanations") or ( [response["explanation"]] if response.get("explanation") else ["block"] @@ -2875,7 +2919,7 @@ def check_workflow_budget(self) -> None: max_overdraft, remaining, ) - return + return None if decision == "require_approval": # The gate requires human-approval before the call @@ -2962,7 +3006,10 @@ def check_workflow_budget(self) -> None: _captured_eid, None ) logger.info(f"check_workflow_budget: approval {approval_id} approved -- resuming") - return + # The gate minted a reservation for the approved + # execution; hand it back so the caller can settle it + # against the same id. + return reservation_id if outcome == "denied": raise NullRunApprovalDeniedError( workflow_id=workflow_id, @@ -3007,7 +3054,7 @@ def check_workflow_budget(self) -> None: # relying on fall-off-the-end, so that adding a variant to the # known set without adding an arm is a visible no-op here # rather than an implicit allow. - return + return reservation_id # ============================================================================= # v3 wire-protocol helpers @@ -3362,6 +3409,8 @@ def shutdown(self, flush: bool = True) -> None: def track( self, event: dict[str, Any], + *, + reservation_id: str | None = None, ) -> dict[str, Any]: """ Track a cost event. @@ -3380,6 +3429,13 @@ def track( - is_retry: bool (optional) - latency_ms: int (optional) - metadata: dict (optional) + reservation_id: the server-minted reservation this event + settles, when the caller knows it. The legacy + carrier is the server-minted contextvar (one slot per + task); a caller that mints one reservation per call + — the zero-code LLM path — passes it explicitly, so + two concurrent calls cannot swap reservations + (ADR-101). Note: `cost_cents` is NOT a valid event key -- the SDK does not @@ -3482,7 +3538,7 @@ def track( metrics.inc_runtime("dropped_llm_call_no_model") wire_event["__missing_model"] = True - self._route_track(wire_event) + self._route_track(wire_event, reservation_id=reservation_id) # Update metrics (thread-safe) metrics.inc_runtime("track_calls") @@ -4468,7 +4524,11 @@ def _enrich_event(self, event: dict[str, Any]) -> dict[str, Any]: return enriched - def _route_track(self, wire_event: dict[str, Any]) -> None: + def _route_track( + self, + wire_event: dict[str, Any], + reservation_id: str | None = None, + ) -> None: """Route a tracked event to v3 single-event /track or the batch /track/batch endpoint. @@ -4507,7 +4567,12 @@ def _route_track(self, wire_event: dict[str, Any]) -> None: self._transport.track(wire_event) return - smid = get_server_minted_execution_id() + # An explicit reservation id wins over the ambient contextvar. + # The contextvar is ONE slot per task: two LLM calls running + # concurrently in the same task would overwrite each other's + # reservation, and the second /track would settle the first + # call's reservation (ADR-101). + smid = reservation_id or get_server_minted_execution_id() if not smid: # Server-authoritative model (CLAUDE.md §22): an llm_call # event without a paired /check reservation has no diff --git a/tests/test_audit_p0_27_operation_id_hoist.py b/tests/test_audit_p0_27_operation_id_hoist.py index 0f90a82..06e9edd 100644 --- a/tests/test_audit_p0_27_operation_id_hoist.py +++ b/tests/test_audit_p0_27_operation_id_hoist.py @@ -131,11 +131,17 @@ def test_check_workflow_budget_reads_contextvar(self): # require that string to be `op_id` (the contextvar value), # NOT `str(uuid.uuid4())`. runtime = _read(RUNTIME_PY) - # Locate the method body. + # Locate the method body. The signature is deliberately NOT + # pinned: ADR-101 gave check_workflow_budget keyword + # parameters and a return value, and a pin that hard-codes + # the old signature turns every signature change into a + # false red. The slice is anchored on the method boundary + # instead — from the `def` line to the next method or + # top-level class. m = re.search( - r"def check_workflow_budget\(self\) -> None:.*?(?=\n def |\nclass |\Z)", + r"^ def check_workflow_budget\(.*?(?=\n def |\nclass |\Z)", runtime, - re.DOTALL, + re.DOTALL | re.MULTILINE, ) assert m, "could not locate check_workflow_budget method body" body = m.group(0) diff --git a/tests/test_zero_code_preflight.py b/tests/test_zero_code_preflight.py new file mode 100644 index 0000000..4626b4d --- /dev/null +++ b/tests/test_zero_code_preflight.py @@ -0,0 +1,592 @@ +""" +ADR-101 / NR-059 — the zero-code LLM path declares what the call will +cost, BEFORE the provider request leaves. + +The finding this closes: on the zero-code path (httpx / requests +transport, no `@protect`, no `set_call_context`), the SDK only ever +emitted the `llm_call` event AFTER the provider had answered. The +event is the only thing that moves the budget counter, so a call that +the cap should have stopped was billed, not stopped. The control +centre's numbers were the sum of spend that had already happened. + +What these tests pin: + +1. The pre-flight (`check_workflow_budget` → `/gate`) runs BEFORE the + provider request reaches the network. +2. A refusal (the cap says no) aborts the provider request — the + request is never sent. +3. The reservation the gate minted travels with the cost event, so + the backend settles THIS call's reserve and not another call's. +4. A reservation whose call never produced a usage event is RELEASED, + so a failed call does not hold budget until the reclaim worker runs. +5. Two concurrent calls keep their own reservations — the contextvar + carrier is one slot per task and would swap them. +6. The declared numbers are the call's own: the model, the estimated + input size, and the caller's output cap. The gate cache is bypassed + — a cached "allow" carries the PREVIOUS call's reservation. +""" + +from __future__ import annotations + +import json +import threading + +import httpx +import pytest + +from nullrun.breaker.exceptions import NullRunBudgetError +from nullrun.instrumentation.auto import ( + NullRunAsyncTransport, + NullRunSyncTransport, + _read_llm_request_profile, +) + + +# --------------------------------------------------------------------------- +# Doubles +# --------------------------------------------------------------------------- + + +class _FakeRuntime: + """A runtime double that records the ORDER of the wire calls. + + `check_workflow_budget` stands for the `/gate` pre-flight; `track` + stands for `/track`. The order list is what proves the pre-flight + happened before the provider request. + """ + + def __init__( + self, + reservation: str | None = "res-1", + refuse: bool = False, + ) -> None: + self.order: list[str] = [] + self.reservation = reservation + self.refuse = refuse + self.gate_kwargs: dict | None = None + self.tracked: list[tuple[dict, str | None]] = [] + self._counter = 0 + self._lock = threading.Lock() + + def check_workflow_budget(self, **kwargs): + with self._lock: + self._counter += 1 + index = self._counter + self.order.append("gate") + self.gate_kwargs = kwargs + if self.refuse: + raise NullRunBudgetError( + "wf-1", + "period budget exhausted", + status_code=402, + ) + if self.reservation is None: + return None + return f"{self.reservation}-{index}" + + def track(self, event, *, reservation_id=None): + self.order.append("track") + self.tracked.append((event, reservation_id)) + return {"ok": True} + + +class _RecordingTransport(httpx.BaseTransport): + """An inner transport that records the requests it was asked to + send. `fail=True` raises instead of answering. + + `barrier` forces the interleaving a concurrency test needs: every + caller reaches the barrier only AFTER its pre-flight, so two + threads always overlap between "reservation minted" and "cost + event emitted" — exactly the window in which a single-slot + carrier would swap the two reservations. + """ + + def __init__( + self, + body: bytes, + status: int = 200, + fail: bool = False, + barrier: threading.Barrier | None = None, + ) -> None: + self.body = body + self.status = status + self.fail = fail + self.barrier = barrier + self.sent: list[httpx.Request] = [] + + def handle_request(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request) + if self.fail: + raise httpx.ConnectError("provider unreachable") + if self.barrier is not None: + self.barrier.wait(timeout=10) + return httpx.Response(self.status, content=self.body, request=request) + + +class _RecordingAsyncTransport(httpx.AsyncBaseTransport): + def __init__(self, body: bytes, status: int = 200, fail: bool = False) -> None: + self.body = body + self.status = status + self.fail = fail + self.sent: list[httpx.Request] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.sent.append(request) + if self.fail: + raise httpx.ConnectError("provider unreachable") + return httpx.Response(self.status, content=self.body, request=request) + + +def _openai_body() -> bytes: + return json.dumps( + { + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "usage": { + "prompt_tokens": 8, + "completion_tokens": 2, + "total_tokens": 10, + }, + } + ).encode() + + +def _request( + body: dict | None = None, + url: str = "https://api.openai.com/v1/chat/completions", +) -> httpx.Request: + payload = body if body is not None else { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + } + return httpx.Request( + "POST", + url, + json=payload, + headers={"Content-Type": "application/json"}, + ) + + +# --------------------------------------------------------------------------- +# 1. The pre-flight runs BEFORE the provider request +# --------------------------------------------------------------------------- + + +def test_preflight_runs_before_the_provider_request(): + runtime = _FakeRuntime() + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + + assert runtime.order == ["gate", "track"], ( + "the gate pre-flight must come FIRST and the cost event after it" + ) + assert len(inner.sent) == 1 + + +def test_preflight_declares_the_calls_own_numbers(): + runtime = _FakeRuntime() + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request( + _request( + { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "x" * 300}], + "max_completion_tokens": 512, + } + ) + ) + + kwargs = runtime.gate_kwargs + assert kwargs is not None + assert kwargs["model"] == "gpt-4o-mini" + assert kwargs["max_output_tokens"] == 512 + # The historical value was the literal 1 — one token. The whole + # point of ADR-101 is that the call's own size is declared. + assert kwargs["estimated_tokens"] >= 100 + # A cached "allow" carries the PREVIOUS call's reservation id. + assert kwargs["use_gate_cache"] is False + + +def test_no_output_cap_means_no_output_cap_is_declared(): + """A declared cap is a HARD upper bound on the output. Without one + there is no upper bound, and pricing the model cap would strand a + month's budget on one call — so the SDK declares none. + + ``None`` is the "no cap" signal: the runtime drops a null cap from + the wire body, so the backend reserves the input leg only.""" + runtime = _FakeRuntime() + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + + assert runtime.gate_kwargs.get("max_output_tokens") is None + + +def test_a_non_llm_host_is_not_preflighted(): + runtime = _FakeRuntime() + inner = _RecordingTransport(b'{"ok":true}') + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request(url="https://example.com/v1/thing")) + + assert runtime.order == [] + assert runtime.tracked == [] + + +def test_a_scope_that_already_holds_a_reservation_is_not_preflighted_again(): + """``@protect`` reserves once for the whole scope, and the LLM call + inside settles THAT reservation. A second reserve would strand the + first one — it would never be settled and would hold budget until + the reclaim worker ran.""" + import time + + from nullrun.context import ( + clear_server_minted_execution_id, + set_server_minted_execution_id, + set_server_minted_reservation_at, + ) + + runtime = _FakeRuntime(reservation="res-new") + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + set_server_minted_execution_id("scope-reservation") + set_server_minted_reservation_at(time.monotonic()) + try: + transport.handle_request(_request()) + finally: + clear_server_minted_execution_id() + + assert runtime.order == ["track"], ( + "the scope's own reservation is the call's reservation — the " + "transport must not mint a second one" + ) + assert runtime.tracked[0][1] is None, ( + "no transport reservation: the event falls back to the ambient " + "server-minted execution id" + ) + + +# --------------------------------------------------------------------------- +# 2. A refusal aborts the provider request +# --------------------------------------------------------------------------- + + +def test_a_refusal_aborts_the_provider_request(): + runtime = _FakeRuntime(refuse=True) + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + with pytest.raises(NullRunBudgetError): + transport.handle_request(_request()) + + assert inner.sent == [], ( + "the provider request must NOT be sent when the budget says no" + ) + assert runtime.tracked == [] + + +def test_a_refusal_in_the_async_path_aborts_the_provider_request(): + import asyncio + + runtime = _FakeRuntime(refuse=True) + inner = _RecordingAsyncTransport(_openai_body()) + transport = NullRunAsyncTransport(inner, runtime) + + async def _run(): + with pytest.raises(NullRunBudgetError): + await transport.handle_async_request(_request()) + + asyncio.run(_run()) + assert inner.sent == [] + + +# --------------------------------------------------------------------------- +# 3. The reservation travels with the cost event +# --------------------------------------------------------------------------- + + +def test_the_reservation_id_reaches_the_cost_event(): + runtime = _FakeRuntime(reservation="res-42") + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + + assert len(runtime.tracked) == 1 + event, reservation_id = runtime.tracked[0] + assert event["type"] == "llm_call" + assert event["model"] == "gpt-4o-mini" + assert reservation_id == "res-42-1", ( + "the cost event must settle the reservation THIS call minted" + ) + + +def test_two_calls_each_settle_their_own_reservation(): + runtime = _FakeRuntime(reservation="res") + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + transport.handle_request(_request()) + + reservations = [rid for _, rid in runtime.tracked] + assert reservations == ["res-1", "res-2"], ( + "each call must settle its own reserve — a shared one would " + "book the second call's spend against the first call's reserve" + ) + + +def test_concurrent_calls_do_not_swap_reservations(): + """Two calls in two threads. The legacy carrier is ONE contextvar + slot per task, so a contextvar-based hand-off would swap them. + + The barrier forces both calls to reach the provider request before + either emits its cost event, so the overlap really happens. + """ + runtime = _FakeRuntime(reservation="res") + barrier = threading.Barrier(2) + inner = _RecordingTransport(_openai_body(), barrier=barrier) + transport = NullRunSyncTransport(inner, runtime) + + errors: list[BaseException] = [] + + def _call(): + try: + transport.handle_request(_request()) + except BaseException as exc: # pragma: no cover — surfaced below + errors.append(exc) + + threads = [threading.Thread(target=_call) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors + reservations = sorted(rid for _, rid in runtime.tracked) + assert reservations == ["res-1", "res-2"], ( + "concurrent calls must not swap reservations — a contextvar " + "hand-off would" + ) + assert len(set(reservations)) == 2 + + +# --------------------------------------------------------------------------- +# 4. A reservation with no usage event is released +# --------------------------------------------------------------------------- + + +def _release_events(runtime: _FakeRuntime) -> list[dict]: + return [ + event + for event, _ in runtime.tracked + if event.get("metadata", {}).get("reservation_released") + ] + + +def test_a_failed_provider_request_releases_the_reservation(): + runtime = _FakeRuntime(reservation="res-7") + inner = _RecordingTransport(_openai_body(), fail=True) + transport = NullRunSyncTransport(inner, runtime) + + with pytest.raises(httpx.ConnectError): + transport.handle_request(_request()) + + releases = _release_events(runtime) + assert len(releases) == 1, "a failed call must release its reserve" + assert releases[0]["tokens"] == 0 + assert releases[0]["model"] == "gpt-4o-mini" + assert runtime.tracked[0][1] == "res-7-1", ( + "the release must settle the reservation it releases" + ) + + +def test_a_response_without_usage_releases_the_reservation(): + runtime = _FakeRuntime(reservation="res-8") + inner = _RecordingTransport(b'{"id":"x","model":"gpt-4o-mini"}') + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + + releases = _release_events(runtime) + assert len(releases) == 1 + assert releases[0]["metadata"]["release_reason"] == "usage_not_extractable" + + +def test_an_empty_body_releases_the_reservation(): + runtime = _FakeRuntime(reservation="res-9") + inner = _RecordingTransport(b"") + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + + releases = _release_events(runtime) + assert len(releases) == 1 + assert releases[0]["metadata"]["release_reason"] == "empty_body" + + +def test_a_reserved_call_with_usage_does_not_release(): + runtime = _FakeRuntime(reservation="res-10") + inner = _RecordingTransport(_openai_body()) + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request()) + + assert _release_events(runtime) == [], ( + "a call that produced a usage event settles its reserve with " + "that event — a second release would double-settle" + ) + + +def test_a_call_with_no_model_is_not_reserved_at_all(): + """A body with no ``model`` is not a priced call — there is nothing + to reserve for it and no ``llm_call`` event to settle it. The + reclaim worker is the backstop for a reserve that never arrives.""" + runtime = _FakeRuntime(reservation="res-11") + inner = _RecordingTransport(b'{"id":"x"}') + transport = NullRunSyncTransport(inner, runtime) + + transport.handle_request(_request({"messages": []})) + + assert runtime.order == [], "no gate call and no cost event" + assert runtime.tracked == [] + + +# --------------------------------------------------------------------------- +# 5. The profile reader +# --------------------------------------------------------------------------- + + +def test_the_profile_reads_the_model_the_size_and_the_cap(): + request = _request( + { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello world"}], + "max_tokens": 256, + } + ) + profile = _read_llm_request_profile(request) + assert profile is not None + assert profile.model == "gpt-4o-mini" + assert profile.declared_max_output_tokens == 256 + assert profile.estimated_input_tokens > 0 + + +def test_a_body_without_a_model_is_not_a_priced_call(): + request = _request({"messages": []}) + assert _read_llm_request_profile(request) is None + + +def test_a_base64_blob_is_not_counted_as_prompt_text(): + """A multimodal request carries the image as base64 — megabytes of + characters the provider bills as tiles, not as text tokens.""" + blob = "A" * 200_000 + request = _request( + { + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "user", "content": blob}, + ], + } + ) + profile = _read_llm_request_profile(request) + assert profile is not None + assert profile.estimated_input_tokens < 1000, ( + "the base64 blob must not be counted as prompt text" + ) + + +def test_the_input_estimate_is_capped(): + long_text = "word " * 200_000 + request = _request( + { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": long_text}], + } + ) + profile = _read_llm_request_profile(request) + assert profile is not None + assert profile.estimated_input_tokens <= 32_768 + + +# --------------------------------------------------------------------------- +# 6. The `requests` path carries the same pre-flight +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_requests(monkeypatch): + """Install a synthetic ``requests`` module so the patcher can wrap + ``Session.send`` without the real dependency.""" + import sys + from types import ModuleType + + class _FakeResponse: + def __init__(self, body: bytes, status_code: int): + self.content = body + self.status_code = status_code + self.headers = {"Content-Type": "application/json"} + + class _FakeSession: + send_count = 0 + _nullrun_patched = False + + @staticmethod + def send(self_or_cls, request, **kwargs): + _FakeSession.send_count += 1 + return _FakeResponse(_openai_body(), 200) + + module = ModuleType("requests") + module.Session = _FakeSession + monkeypatch.setitem(sys.modules, "requests", module) + yield module + + +def test_the_requests_path_preflights_and_carries_the_reservation( + fake_requests, monkeypatch +): + import importlib + import sys + from types import SimpleNamespace + + if "nullrun.instrumentation.auto_requests" in sys.modules: + importlib.reload(sys.modules["nullrun.instrumentation.auto_requests"]) + from nullrun.instrumentation.auto_requests import patch_requests + + runtime = _FakeRuntime(reservation="req-1") + assert patch_requests(runtime) is True + + body = json.dumps( + { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 128, + } + ).encode() + request = SimpleNamespace( + url="https://api.openai.com/v1/chat/completions", + headers={}, + body=body, + _nullrun_tracked=False, + ) + + fake_requests.Session().send(request) + + assert runtime.order == ["gate", "track"], ( + "the requests path must pre-flight before the provider request" + ) + assert runtime.gate_kwargs["model"] == "gpt-4o-mini" + assert runtime.gate_kwargs["max_output_tokens"] == 128 + assert runtime.gate_kwargs["use_gate_cache"] is False + assert len(runtime.tracked) == 1 + assert runtime.tracked[0][1] == "req-1-1" + + if "nullrun.instrumentation.auto_requests" in sys.modules: + importlib.reload(sys.modules["nullrun.instrumentation.auto_requests"]) From 6ad5468005b6044fa8897ad1a65b9311fe129f25 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Thu, 29 Oct 2026 16:24:56 +0400 Subject: [PATCH 3/3] =?UTF-8?q?chore(release):=200.23.0=20=E2=80=94=20MCP?= =?UTF-8?q?=20approval=20re-entry=20+=20zero-code=20budget=20pre-flight?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Minor release. Two enforcement paths that could not complete their own contract are closed. The MCP path could not finish an approval round trip at all: a retry after the operator approved landed in `**passthrough` and was forwarded to the MCP server, so the gate never saw the grant. The zero-code LLM path booked the call only AFTER the provider had already answered — the `llm_call` event is the only thing that moves the budget counter, so a call the ceiling was supposed to stop was counted instead of stopped, and the control centre showed the sum of spend that had already happened. Both fixes are SDK-side; there is no wire-protocol change. The public surface grows by ten keyword-only parameters across four public methods, and one changed return value. ## Fixed ### 1. MCP approval re-entry (NR-058 / NR-015, SDK half) — 5021ed3 `MCPAdapter.call_tool` had no `approval_id` parameter, so a retry after the operator approved landed in `**passthrough` and was forwarded to the MCP SERVER — the gate never saw the grant. The adapter sent neither `business_impact` nor `action_digest`, so the approval row was never digest-bound and the backend could not detect a tampered payload at consume time. The tool arguments travelled only as the legacy `input` field, while the backend resolves the canonical bag as `tool_arguments` → `tool_params` → Null (ADR-060 §4) — so every params-based rule saw "no bag". Three NAMED keyword-only parameters (`approval_id`, `business_impact`, `action_digest`) are placed BEFORE `**passthrough`, so they can never fall into the passthrough and be forwarded to the MCP server. `NullRunRuntime.execute` gains `approval_id` and `tool_arguments` and forwards them into the `execute_kwargs` it already builds, using the same conditional pattern as `business_impact` / `action_digest`: the key is added only when the value is not None, because an absent field means "no grant" and that is not the same wire statement as a null. The two new parameters are appended AFTER `tools` so the existing positional order of the parameters above is unchanged. The same argument dict now reaches the wire as BOTH `input` (the tool payload, unchanged) and `tool_arguments` (the canonical bag). No new wire field is invented: `tool_arguments` is the field the backend already resolves. Separately, `/api/v1/execute` runs an existence check on the Redis binding `execution:{id}` and answers 404 EXECUTION_NOT_FOUND when no prior `/gate` minted it, so a DIRECT `adapter.call_tool(...)` never reached the MCP server at all. Before `runtime.execute(...)` the adapter now ensures a `/gate` call exists for this logical action: when `get_server_minted_execution_id()` returns None (no prior `/gate` in this context) it calls `runtime.check_workflow_budget()` first; otherwise it SKIPS it. The pre-flight RESERVES money at the gate, so running it twice would charge the org twice for one logical call — the condition is the same contextvar `runtime.execute` reads to decide whether to reuse the execution id, so the two agree by construction. A refusal from the pre-flight propagates and the MCP client is NOT called. Ordering mirrors the `@protect` wrapper (contextvar stamp → `/gate` pre-flight → `/execute` → body). (+558/-4; test file `tests/test_mcp_approval_wire.py`, 15 tests) ### 2. Zero-code budget pre-flight (NR-059) — 4bc9322 On the zero-code path (httpx / requests transport, no `@protect`, no `set_call_context`) the SDK sent the `llm_call` event ONLY AFTER the provider's response. The event is the only thing that moves the budget counter, so a call the cap should have stopped was billed, not stopped. The httpx (sync + async) and requests transports now run the pre-flight BEFORE the request reaches the provider: they read the model, the prompt size and the output cap out of the request body and reserve them. A gate refusal is raised outward — the provider request does NOT leave. The reservation rides with the event explicitly (`track(..., reservation_id=...)`) rather than through a contextvar: the contextvar is one slot per task, so two parallel calls would swap reservations. A call that produced no usage (network error, empty body, no usage block) releases its reservation with a zero-valued `/track` — otherwise the reservation would hold budget until the reclaim worker ran. When the scope already holds a fresh reservation (the `@protect` path reserves once for the whole scope) the pre-flight is SKIPPED: a second reservation would orphan the first. `check_workflow_budget` gains `model` / `estimated_tokens` / `max_output_tokens` / `use_gate_cache` and returns the server-minted reservation id (`str | None`). `NullRunRuntime.track` gains `reservation_id`. Token estimation is ~3 characters per token, 1 token per CJK character, base64 is not counted as text, and the input estimate is capped at 32768 tokens. The output cap is declared ONLY if the caller declared one — without it there is no upper bound, and pricing at the catalog maximum would freeze a monthly budget on a single call. The README now states, per path, what the ceiling stops and what only the NEXT call can stop. (+1210/-35; test file `tests/test_zero_code_preflight.py`, 20 tests) ### 3. Lint hygiene — pre-flight `ruff check src tests` was red on `origin/master`: `tests/test_websocket_extra_is_declared.py` imported `sys` twice (F811 + I001), a duplicate left behind when the 3.10 `tomli` fallback was added in 0.22.2, and the new `tests/test_zero_code_preflight.py` had one blank line too many after its import block. CI only gates `ruff check src/`, so neither was caught there; the release runbook's local gate is `ruff check src tests`. Both edits are import-ordering only — no behaviour changed. ## Verification | Check | Result | |---|---| | `ruff check src tests` | All checks passed | | `ruff check src/` (CI gate) | All checks passed | | `mypy src/nullrun` | Success: no issues found in 39 source files | | `mypy src/` (CI gate) | Success: no issues found in 39 source files | | `pytest -q` | **1948 passed, 12 skipped** in 192.05s (vs baseline 1913 / 12 — **+35 new tests** from `test_mcp_approval_wire` (15) and `test_zero_code_preflight` (20)) | | Scratch diff | clean (`dist_local/`, `*.defect*` absent) | | `nullrun.__version__` | `0.23.0` | | Wire-format compatibility | unchanged from 0.22.2 — no protocol change; `approval_id` and `tool_arguments` are the fields the backend already resolves, and each is sent only when set (absent ≠ null) | --- CHANGELOG.md | 149 ++++++++++++++++++++++ pyproject.toml | 2 +- src/nullrun/__version__.py | 2 +- tests/test_websocket_extra_is_declared.py | 1 - tests/test_zero_code_preflight.py | 1 - uv.lock | 2 +- 6 files changed, 152 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7265d1e..219edc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,152 @@ +## [0.23.0] - 2026-10-29 + +Minor release. Two enforcement paths that could not complete their own +contract are closed. The MCP path could not finish an approval round +trip at all: a retry after the operator approved landed in +`**passthrough` and was forwarded to the MCP server, so the gate never +saw the grant. The zero-code LLM path booked the call only AFTER the +provider had already answered — the `llm_call` event is the only thing +that moves the budget counter, so a call the ceiling was supposed to +stop was counted instead of stopped, and the control centre showed the +sum of spend that had already happened. Both fixes are SDK-side; there +is no wire-protocol change. The public surface grows by ten +keyword-only parameters across four public methods, and one changed +return value. + +### Migration + +Nothing breaks on a healthy configuration, and the one behavioural +change is the point of the release rather than a side effect. Five +things differ from 0.22.2: + +1. **`MCPAdapter.call_tool` can carry an approval grant.** The method + gains three NAMED keyword-only parameters — `approval_id`, + `business_impact`, `action_digest` — placed BEFORE `**passthrough` + so they can never fall into the passthrough and be forwarded to the + MCP server. A caller that wrote + `adapter.call_tool("x", {}, approval_id="...")` was sending the + approval id to the MCP server as a tool argument; now it goes to + the gate. +2. **`NullRunRuntime.execute` gains `approval_id` and + `tool_arguments`.** Both are appended AFTER `tools`, so the existing + positional order of the parameters above is unchanged. Each key is + added to the wire only when the value is not `None` — an absent + field means "no grant", which is not the same wire statement as a + null. `tool_arguments` is the canonical argument bag the backend + already resolves (ADR-060 §4); the SDK now sends it alongside the + legacy `input` field instead of only the latter. +3. **`check_workflow_budget` returns the reservation id.** The method + gains `model`, `estimated_tokens`, `max_output_tokens` and + `use_gate_cache`, and its return type changes from `None` to + `str | None` — the server-minted reservation the backend settles + when the matching `/track` arrives, or `None` when no reservation + was minted. Callers that ignored the return value are unaffected. + The `@protect` pre-flight still sends `estimated_tokens=1`, because + that path answers "is ANY budget left?" and not "what will THIS call + cost?". +4. **`NullRunRuntime.track` gains `reservation_id`.** The reservation + travels with the cost event explicitly rather than through a + contextvar: the contextvar is one slot per task, so two concurrent + calls would swap reservations. +5. **The zero-code LLM path now reserves BEFORE the provider request.** + A call that the ceiling rejects never reaches the provider — the + request is not sent at all. Previously the SDK could only report + the spend after the fact. This is visible: a capped zero-code + caller will now see refusals where it previously saw successful + calls that were billed anyway. + +### Fixed + +- **NR-058 (SDK half) — the MCP path could not complete an approval + round trip.** `MCPAdapter.call_tool` had no `approval_id` + parameter, so a retry after the operator approved landed in + `**passthrough` and was forwarded to the MCP SERVER — the gate never + saw the grant. The adapter also sent neither `business_impact` nor + `action_digest`, so the approval row was never digest-bound and the + backend could not detect a tampered payload at consume time. The + three parameters are now named keyword-only members of the public + signature and are forwarded into `runtime.execute(...)`, which + gained the matching `approval_id` parameter — `Transport.execute` + already accepted it and put it on the wire; the runtime was the only + missing link. Separately, `/api/v1/execute` runs an existence check + on the Redis binding `execution:{id}` and answers 404 + EXECUTION_NOT_FOUND when no prior `/gate` minted it, so a DIRECT + `adapter.call_tool(...)` never reached the MCP server at all. The + adapter now runs the pre-flight itself when + `get_server_minted_execution_id()` is `None` and SKIPS it otherwise + — that path reserves money at the gate, and a second `/gate` would + charge the org twice for one logical call. The condition is the same + contextvar `runtime.execute` reads, so the two agree by construction. + A refusal from the pre-flight propagates and the MCP client is NOT + called. (`5021ed3`, +558/-4; test file + `tests/test_mcp_approval_wire.py`, 15 tests) +- **NR-015 (SDK half) — the canonical argument bag never reached the + wire.** The tool arguments travelled only as the legacy `input` + field, while the backend resolves the canonical bag as + `tool_arguments` → `tool_params` → Null (ADR-060 §4) — so every + params-based rule saw "no bag", and the approval decision had no + argument snapshot to bind to. The same dict now reaches the wire as + BOTH `input` (the tool payload, unchanged) and `tool_arguments` (the + canonical bag). No new wire field is invented: `tool_arguments` is + the field the backend already resolves. (`5021ed3`) +- **NR-059 — the zero-code LLM path booked the call after the provider + had already been paid.** On the zero-code path (httpx / requests + transport, no `@protect`, no `set_call_context`) the SDK sent the + `llm_call` event ONLY AFTER the provider's response. The event is + the only thing that moves the budget counter, so a call the cap + should have stopped was counted, not stopped. Now the httpx (sync + and async) and requests transports run the pre-flight BEFORE the + request leaves: they read the model, the prompt size and the output + cap out of the request body and reserve them, and a gate refusal is + raised outward so the provider request is never sent. The + reservation travels with the event explicitly + (`track(..., reservation_id=...)`), because the contextvar carrier is + one slot per task and two parallel calls would swap reservations. A + call that produced no usage (network error, empty body, no usage + block) releases its reservation with a zero-valued `/track` — + otherwise the reservation would hold budget until the reclaim worker + ran. When the scope already holds a fresh reservation (the + `@protect` path reserves once for the whole scope) the pre-flight is + SKIPPED: a second reservation would orphan the first. Token + estimation is ~3 characters per token, 1 token per CJK character, + base64 is not counted as text, and the estimate is capped at 32768 + tokens. The output cap is declared ONLY if the caller declared one — + without it there is no upper bound, and pricing at the catalog + maximum would freeze a monthly budget on a single call. The README + now states, per path, what the ceiling stops and what only the NEXT + call can stop. (`4bc9322`, +1210/-35; test file + `tests/test_zero_code_preflight.py`, 20 tests) + +### Cleanup + +- **Lint hygiene in two test files** — `ruff check src tests` was red + on `origin/master`: `tests/test_websocket_extra_is_declared.py` + imported `sys` twice (F811 + I001), a duplicate left behind when the + 3.10 `tomli` fallback was added in 0.22.2, and the new + `tests/test_zero_code_preflight.py` had one blank line too many + after its import block. CI only gates `ruff check src/`, so neither + was caught there; the local gate named in the release runbook is + `ruff check src tests`. Both edits are import-ordering only — no + behaviour changed. + +### Verification + +| Check | Result | +|---|---| +| `ruff check src tests` | All checks passed | +| `mypy src/nullrun` | Success: no issues found in 39 source files | +| `pytest -q` | **1948 passed, 12 skipped** in 192.05s (vs baseline 1913 / 12 — **+35 new tests** from `test_mcp_approval_wire` (15) and `test_zero_code_preflight` (20)) | +| Scratch diff | clean (`dist_local/`, `*.defect*` absent) | +| `nullrun.__version__` | `0.23.0` | +| Wire-format compatibility | unchanged from 0.22.2 — no protocol change; `approval_id` and `tool_arguments` are the fields the backend already resolves, and each is sent only when set (absent ≠ null) | + +### Commits included + +``` +5021ed3 fix(sdk): MCP approval re-entry — approval_id on the gate, digest binding, canonical argument bag, missing /gate pre-flight (NR-058 / NR-015, SDK half) +4bc9322 fix(budget): LLM-путь резервирует стоимость звонка ДО запроса к провайдеру (NR-059) +``` + ## [0.22.2] - 2026-10-16 Patch release, no protocol change. Three unrelated bugs closed in one diff --git a/pyproject.toml b/pyproject.toml index 97d4f8c..186da6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "nullrun" # Full release history lives in CHANGELOG.md; only the current version # is pinned here. -version = "0.22.2" +version = "0.23.0" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. The headline is the canonical §1 statement # from positioning.md — "runtime decision layer for tool-using AI agents" diff --git a/src/nullrun/__version__.py b/src/nullrun/__version__.py index a73bf62..a19dd96 100644 --- a/src/nullrun/__version__.py +++ b/src/nullrun/__version__.py @@ -5,5 +5,5 @@ string and the SDK_MIN_VERSION constant. """ -__version__ = "0.22.2" +__version__ = "0.23.0" __platform_version__ = "1.0.0" diff --git a/tests/test_websocket_extra_is_declared.py b/tests/test_websocket_extra_is_declared.py index 61b77d5..b7f195d 100644 --- a/tests/test_websocket_extra_is_declared.py +++ b/tests/test_websocket_extra_is_declared.py @@ -49,7 +49,6 @@ # before any test in the suite can run, so this fallback is the # difference between "3.10 in CI green" and "3.10 errors the # whole module at import". -import sys if sys.version_info >= (3, 11): import tomllib diff --git a/tests/test_zero_code_preflight.py b/tests/test_zero_code_preflight.py index 4626b4d..0796775 100644 --- a/tests/test_zero_code_preflight.py +++ b/tests/test_zero_code_preflight.py @@ -41,7 +41,6 @@ _read_llm_request_profile, ) - # --------------------------------------------------------------------------- # Doubles # --------------------------------------------------------------------------- diff --git a/uv.lock b/uv.lock index abbb578..c1715b6 100644 --- a/uv.lock +++ b/uv.lock @@ -636,7 +636,7 @@ wheels = [ [[package]] name = "nullrun" -version = "0.22.2" +version = "0.23.0" source = { editable = "." } dependencies = [ { name = "httpx" },