From d4f4eac53cd4671179dfcb06ad85987baa8db822 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 31 Oct 2026 21:23:20 -0400 Subject: [PATCH 01/35] Use one observed private lifetime for physical Toad runs --- docs/performance/physical-run.md | 98 +++++ docs/performance/scroll-series.md | 5 + tools/acceptance.py | 347 +++-------------- tools/performance/capture_live.py | 64 +--- tools/performance/scroll_scenario.py | 62 +++ tools/performance/scroll_scenario.sh | 254 ------------- tools/physical_ui.py | 549 +++++++++++++++++++++++++++ tools/physical_ui_remote.py | 360 ++++++++++++++++++ 8 files changed, 1128 insertions(+), 611 deletions(-) create mode 100644 docs/performance/physical-run.md create mode 100644 tools/performance/scroll_scenario.py delete mode 100755 tools/performance/scroll_scenario.sh create mode 100644 tools/physical_ui.py create mode 100644 tools/physical_ui_remote.py diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md new file mode 100644 index 000000000..6df1d4b5a --- /dev/null +++ b/docs/performance/physical-run.md @@ -0,0 +1,98 @@ +# Isolated physical Toad runs + +The shared implementation is `tools/physical_ui.py`: the existing +`XvfbDisplay → OwnedDisplay` family owns Xlib/display allocation and retirement; +`TerminalApp` owns st, the frontend, source observations and all physical input. +`tools/acceptance.py` and the scrolling scenario consume it. There is no public +raw-input method on a display. + +## Entry point + +Run with the **installed source runtime's Python**, with Core's retained +`CheckRoot.create()` and `SuppliedRuntimeHome` APIs available (Core PR #3821). +Do not put the candidate Core checkout on `PYTHONPATH` and call that installed +source qualification. + +```sh +/home/ts/.local/share/agent-comms/current/bin/python \ + tools/performance/scroll_scenario.py \ + --thread YOUR_THREAD --history-thread ANOTHER_THREAD \ + --output /home/ts/.cache/agent-scratch/YOUR_OWNER/fresh-run +``` + +`--runtime /absolute/complete/candidate-prefix` selects only the frontend and +private copy's runtime. The source controller remains installed. The candidate +must contain its own Toad/Core/Textual packages; a dependency `.pth` pointing +those packages outside the selected prefix fails actual module-origin checks. +Supply a new output directory for each distinct run. Do not replay a failed or +UNKNOWN input against an old run. + +The callable entry point is `TerminalApp.copied(source, threads, runtime, +output, thread=..., size=Size(...))`. Its context yields the admitted app and +retains the copied root, frontend-home, operation receipts, screenshots and +recording on exit. The caller still owns retiring those retained files after +acceptance and actual borrower checks. + +## What the owner establishes + +- `CheckRoot` alone selects/imports bounded stopped history; old goals, turns, + tasks, processes and input authority do not travel with it. The installed + source owner creates history; only its disposable copy is published through + `RootRuntime.publish` to a supplied runtime. +- Fresh Toad config/state/data and no copied saved-tab database. Presentation + settings can travel; source identities cannot be repaired by rewriting tabs. +- HOME/XDG inside `st -e env`: st itself resets HOME from passwd before exec. +- Xvfb uses displayfd, not a guessed display number. Xlib checks its actual + local peer UID/process; the run verifies the allocated server incarnation. +- Child exec grants debugging only to the recorded controller PID and its + descendants. It changes no global ptrace policy and needs no sudo. +- Before physical input: the actual frontend incarnation, runtime and loaded + Toad/Core/Textual module locations, HOME/XDG/database, selected/opened source, + every saved entry's source and every borrowed ACP process's creator/session + agent coordination roots. Pending/foreign sources refuse input. +- The read-only observation asks `ToadApp.running_app()` and the actual screen + owner. A window caption is not readiness. `remote_exec` receives an explicit + module filename binding and an adjacent FieldCodec request; a missing or + timed-out result is not a successful observation. +- st text travels with the actually published history viewport geometry. + `ScreenCapture` clips through Textual/Rich's cell geometry, not sidebar + separators that disappear when a sidebar closes or occur inside code. + Textual's installed compiled `Region` rejects subclassing; `CapturedRegion` + adds only the existing `DeclaredWireType` boundary capability and retains the + actual Region object, without copying its coordinate declarations. + +## Evidence limits + +This scenario exercises **bounded stopped history**, not the retired shell +benchmark's active provider response. Historical `scroll-series.md` entries +retain their original source/traffic scopes and old API names. They are not +commands to rerun against the live root. + +A source observation establishes module **locations**, not hashes of already +loaded code. Screen dumps help inspect text; the continuous Xvfb recording owns +visible pixels. Neither an import nor one screenshot proves the whole journey. +A run with no ACP bindings does not demonstrate the bound/pending/foreign ACP +cases. + +Source-observer work is part of the measurement overhead. For example, one +prototype observation took about 97 ms inside the callback before considering +its compilation/dispatch cost. Do not report that overhead as production frame +cost or quietly subtract it from a user-facing latency claim. Correlate complete +controller/call spans and the recording. + +The separate reviewed `capture_history.py` path retains its own explicit +approval, identity, size/time budgets and completion token. This entry point +neither starts that capture nor authorizes heap, GC, repaint, remeasure or +arbitrary trace experiments. The legacy `capture_live.py` diagnostic options do +not become authorized merely because they import the shared display resource. + +## Current qualification + +Core #3821 is an API checkpoint; full acceptance is separate. Prototype7 +completed with exit0: actual copied b34 frontend/source/home/runtime admission, +held PageUp/PageDown, reversal, End and wheel input, screenshots, a continuous +recording and typed copied-root stop. Its source controller still used the +owned Core overlay, and it had no bound ACP. Earlier failed run receipts remain +retained. Candidate admission, installed-source qualification, four isolation +counterexamples, tab/resize/arrival/retirement coverage and final profiling must +be reported individually; none is implied by this bounded scrolling result. diff --git a/docs/performance/scroll-series.md b/docs/performance/scroll-series.md index 97a09e89c..6f00bb0fa 100644 --- a/docs/performance/scroll-series.md +++ b/docs/performance/scroll-series.md @@ -1,5 +1,10 @@ # Scroll series +Historical measurements below retain their original commands, API names and +source/traffic scopes. The unsafe shell controller and raw display-input API +are retired. Use the [shared isolated physical-run owner](physical-run.md) for +new work; its stopped-history scenario is not the old active-response benchmark. + One line per change, from `tools/performance/scroll_scenario.sh` (real `toad` in `st` on display :121, a fork of `perf-base-20261009`, which is a fork of the saved `agent-comms-ux` session; an active response, held PageUp from the input box, wheel bursts with reversal, End, tab return). Frame work is UI-thread time per frame handed to the driver; input-to-paint runs from the driver posting an event to the first frame after its last handler returns. Input-to-paint pairs each input with the first frame after it is handled (fixed 2026-10-09; earlier summaries paired out of order). Raw results are under `~/.cache/agent-scratch/scroll-series//`. | Date | Change | Total frame median (ms) | Total frame p95 (ms) | Input-to-paint median (ms) | Input-to-paint p95 (ms) | Scroll-path lines +/- | What the screen showed | diff --git a/tools/acceptance.py b/tools/acceptance.py index 48b1cdb28..67210103d 100644 --- a/tools/acceptance.py +++ b/tools/acceptance.py @@ -1,284 +1,41 @@ #!/usr/bin/env python3 -"""Toad's half of an installed runtime's check: what a person sees in a terminal. +"""Toad's installed-runtime check in an owned, source-checked physical run. /bin/python -I tools/acceptance.py --runtime RUNTIME_BIN --thread NAME --output DIR -Core's installer runs this with the runtime under check, in a home whose route selects -a rehearsal copy of the bus, on a thread it made there. Everything runs in plain st on -an Xvfb display this tool owns. Plain ``toad`` must start; its ACP entrypoint must -show the thread ready. This tool types one fresh request into that open tab, checks -its final native reply and returns to the tab. It then says SHOWN. Core sends the thread a DM and, -once the thread has answered on the bus, writes the answer here as one JSON string; it -must appear in the thread, in #any and again on return to the thread, and the reader -position must survive leaving and returning with only that DM's category shown. The -tool then says SEEN. The last line printed is the result. +Core supplies a rehearsal root and thread. A fresh frontend home does NOT restore +source-bound saved Toad tabs. Plain st and the real ACP editor share TerminalApp's +post-attachment source/runtime/persistence admission. SHOWN, SEEN and the final +JSON retain Core's existing protocol; failed/UNKNOWN fresh input is never replayed. """ from __future__ import annotations import argparse import json import os -import re +from pathlib import Path import shlex -import subprocess import sys import time -from pathlib import Path +from agent_comms.active_route import ActiveRoute from agent_comms.comms import wire +from agent_comms.launch_root import LaunchRoot +from agent_comms.launch_runtime import LaunchRuntime from agent_comms.ui_model.message_category import InboundCategory, all_categories -from rich.cells import cell_len - from toad.widgets.side_bar import BackTabButton # Isolated Python omits the script directory; this tool owns its sibling imports. sys.path.insert(0, str(Path(__file__).resolve().parent)) -from performance.capture_live import XvfbDisplay - - -def run(*args: str | Path) -> str: - print("+ " + " ".join(map(str, args)), file=sys.stderr, flush=True) - return subprocess.check_output(list(map(str, args)), text=True, timeout=10).strip() - - -class OwnedDisplay(XvfbDisplay): - """The isolated display this check creates and removes.""" - - def __init__(self, output: Path): - output.mkdir(parents=True, exist_ok=True) - self.log = (output / "xvfb.log").open("w") - self.server = subprocess.Popen( - ["Xvfb", "-displayfd", "1", "-screen", "0", "1600x1000x24", "-nolisten", "tcp"], - stdout=subprocess.PIPE, stderr=self.log) - try: - super().__init__(":" + self.server.stdout.readline().decode().strip()) - except BaseException: - self.close_server() - raise - - def close_server(self) -> None: - self.server.terminate() - self.server.wait(timeout=5) - self.log.close() - - def __exit__(self, *exception): - try: - return super().__exit__(*exception) - finally: - self.close_server() - - -class TerminalApp: - """Own one isolated st window, its screen and physical X input.""" - - columns, rows = 140, 40 - - def __init__(self, command, display, output, environment, cwd): - self.command, self.display, self.output = command, display, output - self.environment = dict(environment, DISPLAY=display.name) - self.cwd = cwd - - def __enter__(self): - import psutil - self.output.mkdir(parents=True, exist_ok=True) - self.dump = self.output / "st-screen.txt" - self.log = (self.output / "st.log").open("w") - self.terminal = subprocess.Popen( - ["st", "-f", "DejaVu Sans Mono:pixelsize=16", "-g", f"{self.columns}x{self.rows}", - "-o", str(self.dump), "-e", "env", f"HOME={self.environment['HOME']}", - *map(str, self.command)], - cwd=self.cwd, env=self.environment, stdout=self.log, stderr=self.log) - self.owner = psutil.Process(self.terminal.pid) - self.created = self.owner.create_time() - try: - self.window = int(subprocess.check_output( - ["xdotool", "search", "--sync", "--onlyvisible", "--pid", str(self.terminal.pid)], - env=self.environment, text=True, timeout=10).splitlines()[0]) - self.read_grid() - # -o enables st's raw printer too; Ctrl+Print leaves only explicit dumps. - self.keys("ctrl+Print") - self.wait("terminal", lambda text: bool(text.strip())) - return self - except BaseException: - self.__exit__(*sys.exc_info()) - raise - - def capture(self): - """Read st's own print-screen output; screenshots record the visible window.""" - offset = self.dump.stat().st_size - self.keys("shift+Print") - deadline = time.monotonic() + 2 - while True: - with self.dump.open("rb") as stream: - stream.seek(offset) - data = stream.read() - if data.count(b"\n") >= self.rows: - return data.decode("utf-8") - if self.terminal.poll() is not None or time.monotonic() >= deadline: - raise RuntimeError("st did not finish its screen dump") - time.sleep(.02) - - def message_rows(self, text): - """Each screen row's middle pane, decoded through the sidebar boundaries. - - st's dump drops trailing blanks; padding restores the full row, so a pane's - last columns are its gutter. - """ - for row, line in enumerate(text.splitlines()): - panes = (line + " " * (self.columns - cell_len(line))).split("│") - if len(panes) >= 2: - yield row, "│".join(panes[1:-1] if len(panes) >= 3 else panes[1:]) +from physical_ui import OwnedDisplay, TerminalApp - def messages(self, text): - """Each visible message's body, its wrapped rows joined without whitespace. - A body runs from its ``───`` header to its ``▶`` status row; the last two - columns are the scrollbar gutter. - """ - message = None - for _, cell in self.message_rows(text): - body = cell[:-2].strip() - content = body.lstrip("▌").strip() - if not body.startswith("▌") or content.startswith(("─", "▶")): - if message: - yield message - message = "" if content.startswith("─") else None - elif message is not None: - message += "".join(content.split()) - if message: - yield message - - def reader_position(self, text): - """First visible message and its screen row, excluding the scrollbar gutter.""" - for row, cell in self.message_rows(text): - body = cell[:-2] - if body.startswith("▌") and body.strip("▌ "): - return row, body - return () - - def reader_place(self, name): - """Observe a settled message viewport after one navigation action.""" - previous = () - - def settled(text): - nonlocal previous - rows = self.reader_position(text) - # This connection banner occupies a transcript row until it closes. - same = "Agent Comms connected" not in text and bool(rows) and rows == previous - previous = rows - return same - - return self.reader_position(self.wait(name, settled)) - - def screenshot(self, name): - path = self.output / f"{name}.png" - run("import", "-display", self.display.name, "-window", str(self.window), path) - return path - - def wait(self, name, predicate, seconds=60): - deadline = time.monotonic() + seconds - previous = "" - changes = self.output / f"{name}-changes.txt" - while True: - text = self.capture() - if text != previous: - with changes.open("a") as log: - log.write(f"\n--- {time.monotonic()} ---\n{text}") - previous = text - if predicate(text): - (self.output / f"{name}.txt").write_text(text) - return text - if self.terminal.poll() is not None or time.monotonic() >= deadline: - (self.output / f"{name}-failed.txt").write_text(text) - self.screenshot(f"{name}-failed") - raise RuntimeError(f"Terminal probe {name} failed; see {self.output}") - time.sleep(.2) - - def keys(self, *keys): - XvfbDisplay.input(self.owner, self.created, self.window, "key", "--clearmodifiers", *keys) - - def type_text(self, text): - XvfbDisplay.input(self.owner, self.created, self.window, - "type", "--clearmodifiers", "--delay", "1", text) - - def show_thread(self, thread): - self.wait("ready", lambda text: thread in text - and "▶ Session details · Ready" in "\n".join(text.splitlines()[-8:])) - # The thread sidebar starts collapsed; expand its visible right gutter. - self.click_cell(self.columns - 1, self.rows // 2) - return self.wait("shown", lambda text: "User messages" in text and all( - "Indexing" not in line.split("│", 1)[0] for line in text.splitlines())) - - def click(self, label): - text = self.wait("find-" + label.replace("/", "-"), lambda text: any( - label in line.split("│", 1)[0] for line in text.splitlines())) - row, line = next((row, line) for row, line in enumerate(text.splitlines()) - if label in line.split("│", 1)[0]) - self.click_cell(cell_len(line[:line.index(label)]) + cell_len(label) // 2, row) - - def toggle_group(self, label): - """Click this sidebar group's disclosure, collapsing or expanding its members.""" - def header(line): - return line.split("│", 1)[0].lstrip().startswith(("▾ " + label, "▸ " + label)) - text = self.wait("find-group-" + label, lambda text: any(map(header, text.splitlines()))) - row, line = next((row, line) for row, line in enumerate(text.splitlines()) if header(line)) - glyph = min(index for index in (line.find("▾"), line.find("▸")) if index >= 0) - self.click_cell(cell_len(line[:glyph]), row) - - def hide_category(self, label): - """Click this checkbox in the right sidebar, never matching message text.""" - def control(text): - return next(((row, line) for row, line in enumerate(text.splitlines()) - if "│" in line and label in line.rsplit("│", 1)[-1]), None) - text = self.wait("find-filter-" + label.replace("/", "-"), lambda text: control(text) is not None) - row, line = control(text) - prefix, panel = line.rsplit("│", 1) - self.click_cell(cell_len(prefix + "│" + panel[:panel.index("▐")]) + 1, row) - - def read_grid(self): - """st declares one cell plus its padding as the X minimum window size.""" - geometry = subprocess.check_output( - ["xdotool", "getwindowgeometry", "--shell", str(self.window)], - env=self.environment, text=True) - size = dict(line.split("=", 1) for line in geometry.splitlines() if "=" in line) - hints = subprocess.check_output( - ["xprop", "-display", self.display.name, "-id", str(self.window), "WM_NORMAL_HINTS"], - env=self.environment, text=True) - declared = {name: (int(width), int(height)) for name, width, height in re.findall( - r"program specified (minimum size|base size): (\d+) by (\d+)", hints)} - base_x, base_y = declared["base size"] - minimum_x, minimum_y = declared["minimum size"] - self.cell_width, self.cell_height = minimum_x - base_x, minimum_y - base_y - self.border_x, self.border_y = base_x / 2, base_y / 2 - self.columns = (int(size["WIDTH"]) - base_x) // self.cell_width - self.rows = (int(size["HEIGHT"]) - base_y) // self.cell_height - - def click_cell(self, column, row): - self.read_grid() - x = int(self.border_x + (column + .5) * self.cell_width) - y = int(self.border_y + (row + .5) * self.cell_height) - XvfbDisplay.input(self.owner, self.created, self.window, "mousemove", "--window", - str(self.window), str(x), str(y), "click", "1") - - def __exit__(self, *exception): - try: - if self.terminal.poll() is None: - self.keys("ctrl+q") - try: - self.terminal.wait(timeout=5) - except subprocess.TimeoutExpired: - self.terminal.terminate() - self.terminal.wait(timeout=5) - finally: - self.log.close() - - -def any_channel(text): - return "#any · all comms" in text and "IRC / Markdown" in text +def any_channel(screen): + return "#any · all comms" in screen.text and "IRC / Markdown" in screen.text def check_typed_reply(app: TerminalApp, thread: str, output: Path) -> None: - """A physical submit must finish in the native journal and survive tab return.""" + """A physical submit finishes in the native journal and survives tab return.""" comms = wire() selected = comms.registry.require(thread) session = selected.runtime.history_source(comms.root, selected.incarnation) @@ -287,7 +44,6 @@ def check_typed_reply(app: TerminalApp, thread: str, output: Path) -> None: marker = output / "typed-input.json" app.click_cell(app.columns // 2, app.rows - 4) app.type_text("Reply with " + token + ". Do not use tools.") - # Record before Enter; a failed check must never cause this input to be replayed. marker.write_text(json.dumps(attempt)) started = time.monotonic() app.keys("Return") @@ -298,7 +54,7 @@ def replies(): and entry.message.text.strip() == token) def visible(text): - return any(cell.strip("▌ ") == token for _, cell in app.message_rows(text)) + return any(cell.strip("▌ ") == token for _, cell in text.message_rows()) app.wait("typed-reply", lambda text: bool(replies()) and not comms.registry.require(thread).executing and visible(text), seconds=180) @@ -310,16 +66,15 @@ def visible(text): app.click("#any") app.wait("typed-away", any_channel) app.click(thread) - app.wait("typed-return", lambda text: visible(text) and "User messages" in text) + app.wait("typed-return", lambda screen: visible(screen) and "User messages" in screen.text) app.screenshot("typed-return") def check_answer(app: TerminalApp, thread: str, answer: str) -> None: - """The answer shows in the thread, in #any, and again back in the thread.""" expected = "".join(answer.split()) def visible(text): - return expected in app.messages(text) + return expected in text.messages() app.wait("answer", visible) app.screenshot("answer") @@ -329,20 +84,18 @@ def visible(text): initial = app.reader_place("any-initial") app.click_cell(app.columns // 2, app.rows - 4) app.keys("Page_Up", "Page_Up", "Page_Up") - app.wait("any-page-up", lambda text: bool(app.reader_position(text)) - and app.reader_position(text) != initial) - # #any lists every live agent and can push later channels off screen. + app.wait("any-page-up", lambda screen: bool(screen.reader_position()) + and screen.reader_position() != initial) app.toggle_group("#any") app.click("#comms") - app.wait("comms", lambda text: "#comms" in "\n".join(text.splitlines()[:3]) and "IRC / Markdown" in text) + app.wait("comms", lambda screen: "#comms" in "\n".join(screen.text.splitlines()[:3]) and "IRC / Markdown" in screen.text) app.toggle_group("#any") app.click(thread) - app.wait("answer-return", lambda text: visible(text) and "User messages" in text) + app.wait("answer-return", lambda screen: visible(screen) and "User messages" in screen.text) app.screenshot("answer-return") def check_filtered_return(app: TerminalApp, thread: str) -> None: - """With only the DM into the thread shown, the reader returns to the same rows from #any.""" for category in all_categories() - {InboundCategory}: app.hide_category(category.label) expected = app.reader_place("filtered-before") @@ -350,11 +103,11 @@ def check_filtered_return(app: TerminalApp, thread: str) -> None: app.click("#any") app.wait("filtered-away", any_channel) app.click(BackTabButton.label) - app.wait("filtered-return", lambda text: app.reader_position(text) == expected) + app.wait("filtered-return", lambda screen: screen.reader_position() == expected) app.click("#any") app.wait("filtered-click-away", any_channel) app.click(thread) - app.wait("filtered-click-return", lambda text: app.reader_position(text) == expected) + app.wait("filtered-click-return", lambda screen: screen.reader_position() == expected) app.screenshot("filtered-click-return") @@ -363,39 +116,35 @@ def main() -> None: parser.add_argument("--runtime", type=Path, required=True, help="The runtime's bin directory") parser.add_argument("--thread", required=True) parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - # The protocol owns stdout; everything else this tool prints goes to stderr. + arguments = sys.argv[1:] + LaunchRoot.startup(arguments, ActiveRoute.published_launch) + args = parser.parse_args(arguments) protocol = os.fdopen(os.dup(sys.stdout.fileno()), "w", buffering=1) sys.stdout = sys.stderr - home = Path(os.environ["HOME"]) - environment = {key: value for key, value in os.environ.items() - if not key.startswith(("AGENT_COMMS_ROOT", "AGENT_COMMS_THREAD", "XDG_")) - and key not in ("PYTHONPATH", "PYTHONHOME", "NO_COLOR", "TOAD_VALIDATION")} - environment.update(XDG_CONFIG_HOME=str(home / "config"), XDG_STATE_HOME=str(home / "state"), - XDG_DATA_HOME=str(home / "data"), AGENT_COMMS_RUNTIME_ROOT=str(args.runtime)) + comms = wire() + target = TerminalApp.prepare_target(comms.root, LaunchRuntime.at_prefix(args.runtime.parent)) with OwnedDisplay(args.output) as display: - command = [args.runtime / "toad"] - with TerminalApp(command, display, args.output / "plain", environment, home) as app: - app.wait("store", lambda text: "Toad" in text or "agent" in text.lower()) - app.screenshot("store") - # CheckRoot restores stopped declarations. Ask the lifecycle owner to - # start the selected worker before waiting for a connected editor. - wire().owners.start_many((args.thread,))[0] - command = [*command, "acp", - shlex.quote(str(args.runtime / "agent-comms-acp")), - wire().registry.require(args.thread).worktree, - "--session", args.thread, "--title", "Agent Comms"] - with TerminalApp(command, display, args.output / "comms", environment, home) as app: - app.wait("welcome", lambda text: args.thread in text) - app.click(args.thread) - app.show_thread(args.thread) - app.screenshot("shown") - check_typed_reply(app, args.thread, args.output) - print("SHOWN", file=protocol) - answer = json.loads(sys.stdin.readline()) - check_answer(app, args.thread, answer) - check_filtered_return(app, args.thread) - print("SEEN", file=protocol) + command = (str(args.runtime / "toad"),) + plain = TerminalApp(command, display, args.output / "plain", target, target.home) + with display.recording(args.output / "journey.mp4", plain.environment): + with plain as app: + app.wait("store", lambda screen: "Toad" in screen.text or "agent" in screen.text.lower()) + app.screenshot("store") + comms.owners.start_many((args.thread,))[0] + command = (*command, "acp", shlex.quote(str(args.runtime / "agent-comms-acp")), + str(comms.registry.require(args.thread).worktree), + "--session", args.thread, "--title", "Agent Comms") + with TerminalApp(command, display, args.output / "comms", target, target.home) as app: + app.wait("welcome", lambda screen: args.thread in screen.text) + app.click(args.thread) + app.show_thread(args.thread) + app.screenshot("shown") + check_typed_reply(app, args.thread, args.output) + print("SHOWN", file=protocol) + answer = json.loads(sys.stdin.readline()) + check_answer(app, args.thread, answer) + check_filtered_return(app, args.thread) + print("SEEN", file=protocol) print(json.dumps(dict(plain="passed", shown="passed", answer="passed", channels="passed", filtered_return="passed", evidence=str(args.output))), file=protocol) diff --git a/tools/performance/capture_live.py b/tools/performance/capture_live.py index 5aa642309..e0b50273b 100644 --- a/tools/performance/capture_live.py +++ b/tools/performance/capture_live.py @@ -1,78 +1,26 @@ """Measure an explicitly selected live Toad process on required --display. -Input controllers use this same XvfbDisplay.input owner and terminal launch identity. +Physical input belongs to physical_ui.TerminalApp's observed private-run lifetime. --profile-seconds record a py-spy speedscope profile. --frame-meter N time UI work per frame and input-to-paint for N seconds (frame_meter.py). --targets write where sidebar thread rows and session tabs are (frame_meter.targets). """ import argparse -from contextlib import AbstractContextManager, ExitStack -import ctypes +from contextlib import ExitStack import json import os -import socket -import struct from pathlib import Path import subprocess +import sys import time import psutil -class XvfbDisplay(AbstractContextManager): - """Own a connection to a local framebuffer, resolving X11 aliases at Xlib.""" - - def __init__(self, name: str): - if not name: - raise ValueError("An explicit isolated Xvfb display is required") - x11 = ctypes.CDLL('libX11.so.6') - x11.XOpenDisplay.argtypes = [ctypes.c_char_p] - x11.XOpenDisplay.restype = ctypes.c_void_p - x11.XConnectionNumber.argtypes = [ctypes.c_void_p] - x11.XConnectionNumber.restype = ctypes.c_int - x11.XCloseDisplay.argtypes = [ctypes.c_void_p] - x11.XCloseDisplay.restype = ctypes.c_int - connection = x11.XOpenDisplay(name.encode()) - if not connection: - raise ValueError(f'Cannot connect to isolated Xvfb display: {name}') - with ExitStack() as close_display: - close_display.callback(x11.XCloseDisplay, connection) - with socket.fromfd(x11.XConnectionNumber(connection), socket.AF_UNIX, socket.SOCK_STREAM) as peer: - pid, uid, _gid = struct.unpack('3i', peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) - address = Path(os.fsdecode(peer.getpeername()).lstrip('\0')) - if address.name == 'X0' or uid != os.getuid() or Path(psutil.Process(pid).exe()).name != 'Xvfb': - raise ValueError(f'Not an owned isolated Xvfb resource: {name}') - self.name = name - self.close_display = close_display.pop_all() - - @property - def environment(self) -> dict[str, str]: - return {**os.environ, "DISPLAY": self.name} - - @classmethod - def input(cls, frontend: psutil.Process, created: float, window: int, *commands: str) -> None: - """Take the display from the actual st frontend, never the controller environment.""" - if frontend.create_time() != created or not frontend.is_running(): - raise ValueError("Frontend launch identity changed") - with cls(frontend.environ()["DISPLAY"]) as display: - env = display.environment - owner = subprocess.check_output( - ["xprop", "-display", display.name, "-id", str(window), "_NET_WM_PID"], - env=env, text=True, timeout=5, - ) - if int(owner.partition("=")[2].strip()) != frontend.pid: - raise ValueError("Window does not belong to the frontend on its display") - print(json.dumps({"at_ns": time.time_ns(), "monotonic_ns": time.monotonic_ns(), - "input_target": {"display": display.name, "window": window, - "frontend_pid": frontend.pid, "created": created}}), flush=True) - subprocess.run( - ["xdotool", "windowfocus", "--sync", str(window), *commands], - env=env, check=True, - ) - - def __exit__(self, *exception): - return self.close_display.__exit__(*exception) +# The shared physical resource owns Xlib admission; capture never owns raw input. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from physical_ui import XvfbDisplay def main(): diff --git a/tools/performance/scroll_scenario.py b/tools/performance/scroll_scenario.py new file mode 100644 index 000000000..419e551ca --- /dev/null +++ b/tools/performance/scroll_scenario.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Record real stopped-history scrolling through the common private-run owner. + +Run with the installed SOURCE runtime's Python. --runtime chooses only the +frontend/copy runtime; it never opens the original Comms through that candidate. +No provider input, live-root fork or automatic metadata-capture approval occurs. +This retained-history scenario is NOT comparable to the old active-response +shell benchmark without stating their different source/traffic scopes. +""" +import argparse +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from agent_comms.active_route import ActiveRoute +from agent_comms.comms import wire +from agent_comms.launch_root import LaunchRoot +from agent_comms.launch_runtime import LaunchRuntime +from physical_ui import TerminalApp +from physical_ui_remote import ObservedResult, PhysicalOperation +from textual.geometry import Size + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--thread", required=True) + parser.add_argument("--history-thread", action="append", default=[]) + parser.add_argument("--runtime", type=Path, default=LaunchRuntime.installed().prefix) + parser.add_argument("--output", type=Path, required=True, help="Fresh owned agent-scratch directory") + parser.add_argument("--columns", type=int, default=140) + parser.add_argument("--rows", type=int, default=40) + arguments = sys.argv[1:] + LaunchRoot.startup(arguments, ActiveRoute.published_launch) + args = parser.parse_args(arguments) + if LaunchRuntime.running().prefix != LaunchRuntime.installed().prefix: + parser.error("History must be copied by the installed source runtime, not the candidate") + if args.columns <= 0 or args.rows <= 0: + parser.error("A terminal needs positive columns/rows") + threads = tuple(dict.fromkeys((args.thread, *args.history_thread))) + with TerminalApp.copied( + wire(), threads, LaunchRuntime.at_prefix(args.runtime), args.output, + thread=args.thread, size=Size(args.columns, args.rows), + ) as app: + app.wait("history", lambda screen: bool(screen.reader_position())) + PhysicalOperation.publish(args.output / "admitted.json", ObservedResult(app.observe())) + app.screenshot("startup") + app.click_cell(app.columns // 2, app.rows // 3) + app.input("key", "--clearmodifiers", "--repeat", "20", "--delay", "75", "Page_Up") + app.screenshot("page-up") + app.input("key", "--clearmodifiers", "--repeat", "8", "--delay", "75", "Page_Down") + app.screenshot("reversal") + app.keys("End") + app.screenshot("end-immediate") + app.input("click", "--repeat", "12", "--delay", "60", "4") + app.screenshot("wheel-up") + app.keys("End") + app.screenshot("end-return") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/tools/performance/scroll_scenario.sh b/tools/performance/scroll_scenario.sh deleted file mode 100755 index 8fe6394f9..000000000 --- a/tools/performance/scroll_scenario.sh +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/env bash -# The fixed scroll scenario for docs/performance/scroll-series.md. -# Usage: scroll_scenario.sh NAME [BIN_DIR] -# Forks perf-base-20261009 (a fork of Tristan's saved agent-comms-ux session), opens it -# with the real `toad` in st on explicit SCENARIO_DISPLAY, and drives real X input: -# an active response, held PageUp from the input box, wheel bursts with reversal, -# End, and a return to the open tab. capture_live.py --frame-meter times every frame. -set -euo pipefail -name=$1 -bin=${2:-$(dirname "$(readlink -f "$(command -v toad)")")} -tools=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -out=/home/ts/.cache/agent-scratch/scroll-series/$name -fork=perf-$name-$(date +%H%M%S) -project=/home/ts/wt/comms-post-feature-debt-audit-20260928 -meter_seconds=${METER_SECONDS:-45} -display=${SCENARIO_DISPLAY:?Set an explicit owned Xvfb display} -export DISPLAY=$display -PYTHONPATH="$tools" python3 -c ' -import sys -from capture_live import XvfbDisplay -with XvfbDisplay(sys.argv[1]): - pass -' "$display" -mkdir -p "$(dirname "$out")" -mkdir "$out" - -agent-comms fork --name "$fork" --parent perf-base-20261009 > "$out/fork.json" -env -u NO_COLOR -u TOAD_VALIDATION st -g145x50 -t "$fork" -e "$bin/toad" acp "$bin/agent-comms-acp" \ - --session "$fork" --project-dir "$project" --title "$fork" > "$out/st.log" 2>&1 & -st_pid=$! -read -r st_created display < <(python3 -c ' -import psutil,sys -frontend = psutil.Process(int(sys.argv[1])) -print(frontend.create_time(), frontend.environ()["DISPLAY"]) -' "$st_pid") -# Recorder and input both take their display from the actual frontend. -xdotool() { - PYTHONPATH="$tools" python3 -c ' -import psutil,sys -from capture_live import XvfbDisplay -frontend = psutil.Process(int(sys.argv[1])) -XvfbDisplay.input(frontend, float(sys.argv[2]), int(sys.argv[3]), *sys.argv[4:]) -' "$st_pid" "$st_created" "$window" "$@" -} -cleanup() { - local status=$? - xdotool key --window "$window" ctrl+q 2>/dev/null || true - # Wait for the app to exit by itself: killing the terminal sends SIGHUP to - # the app and its agents, and a process killed inside a guarded registry - # write leaves the live bus's registry guard pending (fail-closed for all). - for _ in $(seq 60); do kill -0 "$st_pid" 2>/dev/null || break; sleep 1; done - if kill -0 "$st_pid" 2>/dev/null; then - echo "app did not exit within 60 s; leaving it running rather than killing it mid-write" >&2 - fi - if (( status != 0 )); then - echo "Retaining failed run and fork $fork; no stop/delete or replay" >&2 - return "$status" - fi - # A fork is a full copy of the parent's 141 MB history: delete it, which - # removes its registration, per-thread state and session files. - agent-comms stop --name "$fork" > /dev/null 2>&1 || true - agent-comms delete --name "$fork" > "$out/delete.json" 2>&1 || true -} -trap cleanup EXIT - -for _ in $(seq 60); do - app_pid=$(pgrep -P "$st_pid" | head -1 || true) - window=$(DISPLAY="$display" command xdotool search --pid "$st_pid" 2>/dev/null | head -1 || true) - [[ -n "$app_pid" && -n "$window" ]] && break - sleep 1 -done -# Optional: raise the UI thread's scheduling priority (for example UI_NICE=-10) -# to separate the app's own work from waiting for a CPU behind other processes. -if [[ -n "${UI_NICE:-}" ]]; then - sudo -n renice -n "$UI_NICE" -p "$app_pid" > "$out/renice.log" 2>&1 -fi -sleep "${LOAD_SECONDS:-25}" -read -r width height < <(DISPLAY="$display" command xdotool getwindowgeometry --shell "$window" | sed -n 's/^\(WIDTH\|HEIGHT\)=//p' | paste -sd' ') -cell() { xdotool mousemove --window "$window" $(( ($1) * width / 145 + 3 )) $(( ($2) * height / 50 + height / 100 )) click 1; } -regions() { python3 "$tools/capture_live.py" --display "$display" --pid "$app_pid" --output-dir "$out" --name "$1" --targets --sudo > /dev/null; } -# Who asks the app to exit is written as it happens (the app has quit by -# itself when the second tab opened, with no error recorded). -python3 "$tools/capture_live.py" --display "$display" --pid "$app_pid" --output-dir "$out" --name app --trace-exit --sudo > /dev/null 2>&1 & -# Optional: measure from here, so opening the second tab (and the closing of -# the deleted thread's tab at the end) fall inside the measured window. -if [[ -n "${METER_FROM_OPEN:-}" ]]; then - # One attach at a time: the exit trace above must be installed first. - for _ in $(seq 40); do [[ -s "$out/app-exit.json" ]] && break; sleep 0.5; done - # PROFILE_LAYOUT profiles the window's layouts instead of metering it. - if [[ -n "${PROFILE_LAYOUT:-}" ]]; then - measure=(--profile-layout "$PROFILE_LAYOUT" ${PROFILE_TARGET:+--profile-target "$PROFILE_TARGET"} - ${PROFILE_WITHIN:+--profile-within "$PROFILE_WITHIN"} ${PROFILE_MIN_MS:+--profile-min-ms "$PROFILE_MIN_MS"}) - else - measure=(--frame-meter "${METER_SECONDS:-150}") - fi - python3 "$tools/capture_live.py" --display "$display" --pid "$app_pid" --output-dir "$out" --name run "${measure[@]}" \ - ${TRACE_CALLS:+--trace-calls "$TRACE_CALLS"} ${TRACE_REFLOWS:+--trace-reflows} ${TRACE_RULES:+--trace-rules} \ - ${TRACE_GARBAGE:+--trace-garbage} ${HEAP_CENSUS:+--heap-census "$HEAP_CENSUS"} ${COLLECT_ALL:+--collect-all "$COLLECT_ALL"} ${HEAP_COMPONENTS:+--heap-components "$HEAP_COMPONENTS"} ${FREEZE_HEAP:+--freeze-heap} ${TRACE_ABC:+--trace-abc} ${TRACE_POSTS:+--trace-posts} --sudo > "$out/capture.log" 2>&1 & - capture=$! - # Optional: every app thread's CPU and run-queue time, every 10 ms, over - # the same window (THREAD_CPU_SAMPLER names the sampler script). - if [[ -n "${THREAD_CPU_SAMPLER:-}" ]]; then - python3 "$THREAD_CPU_SAMPLER" "$app_pid" "$out/thread-cpu.json" "${METER_SECONDS:-150}" & - fi - sleep 2 -fi -# Open a second tab from the sidebar so the scenario can return to this one. -cell 1 25 -sleep 15 -regions sidebar -# Prefer the usual idle thread; with many agents running it may be scrolled -# out of the sidebar, so take the first thread row on screen instead. -read -r row_x row_y < <(python3 -c " -import json,sys -rows=json.load(open(sys.argv[1]))['threads'] -r=rows.get('parent-real-ui-20261009') or next(v for k, v in rows.items() if not k.startswith('perf-')) -print(r[0]+4, r[1])" "$out/sidebar-targets.json") -cell "$row_x" "$row_y" -for attempt in $(seq 15); do - sleep 2 - regions "tabs-$attempt" - tabs=$(python3 -c "import json,sys; print(len(json.load(open(sys.argv[1]))['tabs']))" "$out/tabs-$attempt-targets.json") - [[ "$tabs" -ge 2 ]] && { cp "$out/tabs-$attempt-targets.json" "$out/tabs-targets.json"; break; } -done -if [[ "$tabs" -lt 2 ]]; then - import -window "$window" "$out/tab-open-failed.png" 2>/dev/null || true - echo "second tab did not open; see $out/tab-open-failed.png" >&2 - exit 1 -fi -read -r first_x second_x tab_y < <(python3 -c "import json,sys; t=json.load(open(sys.argv[1]))['tabs']; print(t[0][0]+8, t[1][0]+8, t[0][1])" "$out/tabs-targets.json") -cell "$first_x" "$tab_y" -sleep 3 -# Close the sidebar (its toggle sits on its right edge while open) and confirm. -for attempt in 1 2 3 4 5; do - regions "closed-$attempt" - open_rows=$(python3 -c "import json,sys; print(len(json.load(open(sys.argv[1]))['threads']))" "$out/closed-$attempt-targets.json") - [[ "$open_rows" == 0 ]] && break - cell 54 25 - sleep 5 -done -[[ "$open_rows" == 0 ]] || { echo "sidebar did not close" >&2; exit 1; } -import -window "$window" "$out/loaded.png" 2>/dev/null || true -regions screen-loaded - -if [[ -n "${METER_FROM_OPEN:-}" ]]; then - : # already measuring since before the second tab opened -elif [[ -n "${PROFILE_SECONDS:-}" ]]; then - measure=(--profile-seconds "$PROFILE_SECONDS" ${PROFILE_IDLE:+--idle} ${PROFILE_GIL:+--gil}) -elif [[ -n "${PROFILE_LAYOUT:-}" ]]; then - measure=(--profile-layout "$PROFILE_LAYOUT" ${PROFILE_TARGET:+--profile-target "$PROFILE_TARGET"} - ${PROFILE_WITHIN:+--profile-within "$PROFILE_WITHIN"} ${PROFILE_MIN_MS:+--profile-min-ms "$PROFILE_MIN_MS"}) -else - measure=(--frame-meter "$meter_seconds") -fi -if [[ -z "${METER_FROM_OPEN:-}" ]]; then -python3 "$tools/capture_live.py" --display "$display" --pid "$app_pid" --output-dir "$out" --name run \ - "${measure[@]}" ${TRACE_CALLS:+--trace-calls "$TRACE_CALLS"} ${TRACE_REFLOWS:+--trace-reflows} ${TRACE_RULES:+--trace-rules} --sudo > "$out/capture.log" 2>&1 & -capture=$! -fi -sleep 2 -# Optional load: a real channel message whose replies arrive during the scenario. -if [[ -n "${CHANNEL:-}" && -n "${CHANNEL_MESSAGE:-}" ]]; then - "$HOME/.local/bin/agent-comms" user-send --to "$CHANNEL" --body "$CHANNEL_MESSAGE" --worktree "$project" > "$out/channel-send.json" -fi - -# Active response: a real configured-provider turn that streams while we scroll. -cell 40 47 -xdotool type --delay 5 "Explain in about 300 words why a terminal chat history should scroll prepared text instead of rebuilding widgets. No tools." -xdotool key Return -sleep 4 -# Held PageUp from the input box: 30 key repeats per second for 3 seconds. -# Screen text is sampled during the hold, not only after it. -( sleep 0.8; regions screen-pageup-hold-1; sleep 0.6; regions screen-pageup-hold-2 ) & -holding=$! -xdotool key --repeat "${PAGEUP_REPEAT:-90}" --delay 33 Prior -wait "$holding" -import -window "$window" "$out/pageup.png" 2>/dev/null || true -regions screen-pageup -# Without input the reader's view holds still: loading earlier history above -# it must not move what is shown (checked by screen_checks.py). -sleep 2 -regions screen-pageup-idle-1 -sleep 2 -regions screen-pageup-idle-2 -# Wheel bursts with reversal over the history. -xdotool mousemove --window "$window" 400 300 -for _ in 1 2 3; do - xdotool click --repeat 25 --delay 10 4 - xdotool click --repeat 25 --delay 10 5 -done -sleep 2 -regions screen-wheel-idle-1 -sleep 2 -regions screen-wheel-idle-2 -# End returns to the tail (End belongs to the history once it has focus). -xdotool click 1 -xdotool key End -sleep 2 -import -window "$window" "$out/end.png" 2>/dev/null || true -regions screen-end -# Tab return: switch to the other open tab and back. -cell "$second_x" "$tab_y" -sleep 2 -cell "$first_x" "$tab_y" -sleep 1 -import -window "$window" "$out/tab-return.png" 2>/dev/null || true -regions screen-tab-return - -# A retired thread closes its open tab: delete this run's fork (cleanup -# would delete it anyway) and the observation service must close the tab. -# Archiving keeps the incarnation current, so it does not retire the view. -"$HOME/.local/bin/agent-comms" stop --name "$fork" > /dev/null 2>&1 || true -# Core refuses while any process still references the session (the agent -# exiting, a read in progress); retry as a person would, recording holders. -for attempt in 1 2 3 4 5; do - "$HOME/.local/bin/agent-comms" delete --name "$fork" > "$out/delete-during-run.json" 2>&1 || true - grep -q '"deleted"' "$out/delete-during-run.json" && break - for pid in $(grep -o 'processes \[[0-9, ]*\]' "$out/delete-during-run.json" | grep -o '[0-9]\+'); do - ps -o pid,ppid,args -p "$pid" >> "$out/delete-referrers.txt" 2>&1 || true - done - sleep 1 -done -for attempt in $(seq 10); do - sleep 1 - regions "retire-$attempt" - tabs=$(python3 -c "import json,sys; print(len(json.load(open(sys.argv[1]))['tabs']))" "$out/retire-$attempt-targets.json") - [[ "$tabs" -lt 2 ]] && break -done -echo "{\"tabs_after_delete\": $tabs}" > "$out/retire-check.json" - -wait "$capture" -[[ -n "${PROFILE_SECONDS:-}${PROFILE_LAYOUT:-}" ]] || python3 "$tools/frame_meter.py" "$out/run-frames.json" | tee "$out/summary.json" -# What the reader saw, from the screen text: no placeholders or blank runs in -# the message window, End at the tail, tab return filled and still at the tail. -python3 "$tools/screen_checks.py" "$out" | tee "$out/screen-checks.json" - -# Optional: the per-build paint figure for this live history viewport (after -# the checks: it repaints the history window in full, cold, warm and scrolled by a page). -if [[ -n "${REPAINT:-}" ]]; then - python3 "$tools/capture_live.py" --display "$display" --pid "$app_pid" --output-dir "$out" --name repaint \ - --repaint "$REPAINT" --sudo > /dev/null - python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(d.get('error') or {k: d[k] for k in ('cold', 'warm', 'scroll')})" \ - "$out/repaint-repaint.json" | tee "$out/repaint.txt" -fi - -# Optional: the per-build layout figure for this live widget tree (after the -# checks, since it re-measures the screen at widths it is not shown at). -if [[ -n "${REMEASURE:-}" ]]; then - python3 "$tools/capture_live.py" --display "$display" --pid "$app_pid" --output-dir "$out" --name remeasure \ - --remeasure "$REMEASURE" --sudo > /dev/null - python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print({k: d[k] for k in ('median_ms', 'p10_ms', 'p90_ms')})" \ - "$out/remeasure-remeasure.json" | tee "$out/remeasure.txt" -fi - diff --git a/tools/physical_ui.py b/tools/physical_ui.py new file mode 100644 index 000000000..bbfb1ba49 --- /dev/null +++ b/tools/physical_ui.py @@ -0,0 +1,549 @@ +"""One physical Toad run: private source/home, owned Xvfb/st, observed input. + +Acceptance and performance use these same existing display/terminal types. +The installed source runtime creates history; only that copy is rebound to a +candidate. A closed run retains its files, including failed/UNKNOWN evidence. +""" +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import AbstractContextManager, ExitStack, contextmanager +import ctypes +from dataclasses import dataclass +import itertools +import os +from pathlib import Path +import re +import select +import socket +import struct +import subprocess +import sys +import time + +from agent_comms.active_route import ActiveRoute, active_route_path, route_new_bus +from agent_comms.bus import Bus +from agent_comms.check_root import CheckRoot +from agent_comms.child_process import DetachedChildStdio, ProcessIdentity, SpawnedProcess +from agent_comms.comms import Comms +from agent_comms.installer.practice import SuppliedRuntimeHome +from agent_comms.launch_root import LaunchRoot +from agent_comms.launch_runtime import LaunchRuntime, ThreadLaunch +from agent_comms.recorded_bus import RecordedBusIdentity +from agent_comms.root_runtime import RootRuntime +from agent_comms.store_files import ExclusiveAtomicFile +from rich.cells import cell_len +from rich.segment import Segment +from textual.geometry import Region, Size +from textual.strip import Strip + +from physical_ui_remote import ( + CapturedRegion, ExecOperation, ObserveOperation, PendingRun, PhysicalOperation, + RunObservation, RunTarget, StopRootOperation, +) + + +class XvfbDisplay(AbstractContextManager): + """Own a connection to a local framebuffer, resolving aliases at Xlib. + + This resource has no raw-input entrypoint. Only TerminalApp may drive its + window after checking the actual application/source on that incarnation. + """ + + def __init__(self, name: str): + if not name: + raise ValueError("An explicit isolated Xvfb display is required") + x11 = ctypes.CDLL("libX11.so.6") + x11.XOpenDisplay.argtypes = [ctypes.c_char_p] + x11.XOpenDisplay.restype = ctypes.c_void_p + x11.XConnectionNumber.argtypes = [ctypes.c_void_p] + x11.XConnectionNumber.restype = ctypes.c_int + x11.XCloseDisplay.argtypes = [ctypes.c_void_p] + x11.XCloseDisplay.restype = ctypes.c_int + x11.XDefaultScreen.argtypes = [ctypes.c_void_p] + x11.XDefaultScreen.restype = ctypes.c_int + x11.XDisplayWidth.argtypes = [ctypes.c_void_p, ctypes.c_int] + x11.XDisplayWidth.restype = ctypes.c_int + x11.XDisplayHeight.argtypes = [ctypes.c_void_p, ctypes.c_int] + x11.XDisplayHeight.restype = ctypes.c_int + connection = x11.XOpenDisplay(name.encode()) + if not connection: + raise ValueError(f"Cannot connect to isolated Xvfb display: {name}") + with ExitStack() as close_display: + close_display.callback(x11.XCloseDisplay, connection) + with socket.fromfd(x11.XConnectionNumber(connection), socket.AF_UNIX, socket.SOCK_STREAM) as peer: + pid, uid, _gid = struct.unpack("3i", peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) + address = Path(os.fsdecode(peer.getpeername()).lstrip("\0")) + process = ProcessIdentity.capture(pid) + if address.name == "X0" or uid != os.getuid() or Path(f"/proc/{pid}/exe").resolve().name != "Xvfb": + raise ValueError(f"Not an owned isolated Xvfb resource: {name}") + self.name, self.process = name, process + self.x11, self.connection = x11, connection + self.close_display = close_display.pop_all() + + @property + def environment(self) -> dict[str, str]: + return {**os.environ, "DISPLAY": self.name} + + @property + def size(self) -> Size: + screen = self.x11.XDefaultScreen(self.connection) + return Size(self.x11.XDisplayWidth(self.connection, screen), + self.x11.XDisplayHeight(self.connection, screen)) + + @contextmanager + def recording(self, path: Path, environment: Mapping[str, str]): + """Record this display from startup through frontend retirement.""" + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + size = self.size + with path.with_suffix(".log").open("wb") as log: + with SpawnedProcess.launch( + ("ffmpeg", "-hide_banner", "-loglevel", "warning", "-n", "-f", "x11grab", + "-framerate", "25", "-video_size", f"{size.width}x{size.height}", + "-i", self.name, "-c:v", "libx264", "-preset", "ultrafast", + "-pix_fmt", "yuv420p", str(path)), + env=environment, + stdio=DetachedChildStdio(stdin=subprocess.PIPE, stdout=log, stderr=log), + ) as child: + try: + yield path + finally: + child.process.communicate(b"q", timeout=10) + if child.process.returncode: + raise RuntimeError(f"Physical recording failed; see {log.name}") + + def __exit__(self, *exception): + return self.close_display.__exit__(*exception) + + +class OwnedDisplay(XvfbDisplay): + """The isolated display this run creates and retires.""" + + startup_seconds = 10.0 + stop_seconds = 10.0 + + def __init__(self, output: Path): + output.mkdir(parents=True, exist_ok=True) + self.log = (output / "xvfb.log").open("wb") + self.server = SpawnedProcess.launch( + ("Xvfb", "-displayfd", "1", "-screen", "0", "1600x1000x24", "-nolisten", "tcp"), + stdio=DetachedChildStdio(stdout=subprocess.PIPE, stderr=self.log), + ) + try: + stream = self.server.process.stdout + ready, _, _ = select.select((stream,), (), (), self.startup_seconds) + if not ready: + raise TimeoutError("Xvfb did not publish its allocated display") + name = ":" + stream.readline().decode().strip() + super().__init__(name) + if self.process != self.server.identity: + self.close_display.close() + raise ValueError("Xlib connected to a server this run did not create") + except BaseException: + self.close_server() + raise + + def close_server(self) -> None: + try: + self.server.stop_sync(deadline=time.monotonic() + self.stop_seconds) + finally: + self.server.close_streams() + self.log.close() + + def __exit__(self, *exception): + try: + return super().__exit__(*exception) + finally: + self.close_server() + + +@dataclass(frozen=True) +class ScreenCapture: + """Real st text paired with the published viewport geometry observed for it.""" + + text: str + history_regions: tuple[CapturedRegion, ...] + + def message_rows(self): + lines = self.text.splitlines() + for captured in self.history_regions: + region = captured.region + for row in range(max(0, region.y), min(len(lines), region.bottom)): + yield row, Strip([Segment(lines[row])]).crop(region.x, region.right).text + + def messages(self): + message = None + for _, cell in self.message_rows(): + body = cell.rstrip() + content = body.lstrip("▌").strip() + if not body.startswith("▌") or content.startswith(("─", "▶")): + if message: + yield message + message = "" if content.startswith("─") else None + elif message is not None: + message += "".join(content.split()) + if message: + yield message + + def reader_position(self): + for row, cell in self.message_rows(): + if cell.startswith("▌") and cell.strip("▌ "): + return row, cell + return () + + +class TerminalApp: + """Own one isolated st window, actual-source checks and all physical input.""" + + startup_seconds = 60.0 + stop_seconds = 10.0 + + def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, + target: RunTarget, cwd: Path, *, size: Size = Size(140, 40)): + self.command, self.display, self.output = tuple(map(str, command)), display, output + self.target, self.cwd, self.requested_size = target, cwd, size + environment = LaunchRoot.environment(ThreadLaunch.strip(dict(os.environ))) + for key in tuple(environment): + if key.startswith("XDG_") or key in ("PYTHONPATH", "PYTHONHOME", "NO_COLOR", "TOAD_VALIDATION"): + del environment[key] + environment.update( + HOME=str(target.home), DISPLAY=display.name, + XDG_CONFIG_HOME=str(target.config), XDG_STATE_HOME=str(target.state), + XDG_DATA_HOME=str(target.data), + AGENT_COMMS_RUNTIME_ROOT=str(target.launch_runtime.interpreter.parent), + ) + self.environment = environment + self.operations = itertools.count() + + @property + def columns(self) -> int: + return self.grid_size.width + + @property + def rows(self) -> int: + return self.grid_size.height + + @staticmethod + def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: + """Prepare a fresh frontend home beside an already owned private root. + + Only presentation settings travel. Saved Toad databases/tabs never do. + Existing installer acceptance and copied-history runs share this setup. + """ + from toad.preferences import SavedSettings + + root = root.resolve(strict=True) + home = root.parent / "frontend-home" + # Validate the intended paths before touching the home. RunTarget also + # rejects the real user's home/workspace and an unrelated bus. + if home.exists(): + raise FileExistsError(f"A physical run needs fresh frontend persistence: {home}") + # The source root has already been created by CheckRoot/the installer. + # Its parent is the run workspace, never the user's real home. + import pwd + user_home = Path(pwd.getpwuid(os.getuid()).pw_dir).resolve() + if root.parent == user_home or user_home.is_relative_to(root.parent): + raise ValueError("Refusing frontend setup beside a live user root") + home.mkdir(mode=0o700) + SuppliedRuntimeHome(home, runtime).prepare() + target = RunTarget(root, home.resolve(), runtime.prefix) + for directory in (target.config, target.state, target.data): + directory.mkdir(mode=0o700) + settings = SavedSettings.path() + if settings.is_file(): + destination = target.config / settings.parent.name / settings.name + destination.parent.mkdir(mode=0o700) + ExclusiveAtomicFile(destination).write_bytes(settings.read_bytes()) + route = ActiveRoute(root, RecordedBusIdentity.capture(Bus.path_in(root))) + route_path = home / active_route_path().relative_to(Path.home()) + route_new_bus(route, route_path) + return target + + @classmethod + @contextmanager + def copied(cls, source: Comms, threads: tuple[str, ...], runtime: LaunchRuntime, + output: Path, *, thread: str, size: Size = Size(140, 40)) -> Iterator[TerminalApp]: + """Create under the source owner, bind only its copy, retain run evidence.""" + output = output.expanduser().absolute() + output.mkdir(mode=0o700, parents=True) # Fresh evidence, never overwrite a failed run. + copied = CheckRoot(source, threads, output / "bus").create() + RootRuntime.publish(copied.root, runtime) + target = cls.prepare_target(copied.root, runtime) + try: + with OwnedDisplay(output / "display") as display: + # Keep the real default launcher: its root/runtime succession is + # precisely what the post-attachment observation must establish. + command = (str(Path.home() / ".local/bin/toad-comms"), thread) + app = cls(command, display, output / "frontend", target, target.home, size=size) + with display.recording(output / "journey.mp4", app.environment), app: + yield app + finally: + # Shutdown uses the copy's own runtime, not a foreign source Comms. + # Files remain until the actual acceptance and borrower checks retire them. + cls.stop_copy(target, output / "retirement") + + @staticmethod + def operation_file(operation: PhysicalOperation, directory: Path) -> Path: + directory.mkdir(mode=0o700, parents=True) + source = Path(__file__).with_name("physical_ui_remote.py") + with source.open("rb") as stream: + raw = stream.read(PhysicalOperation.source_bytes + 1) + if len(raw) > PhysicalOperation.source_bytes: + raise ValueError("Physical operation source exceeds its bound") + script = directory / "operation.py" + # remote_exec supplies no module filename. Bind only that external + # datum; the program stays in its reviewed source file and the request + # stays in FieldCodec's adjacent record, as in capture_history. + filename = f"__file__ = {str(script)!r}\n".encode() + ExclusiveAtomicFile(script).write_bytes(filename + raw) + PhysicalOperation.publish(script.with_suffix(".json"), operation) + return script + + @staticmethod + def result(directory: Path): + error = directory / "error.json" + path = error if error.exists() else directory / "result.json" + return PhysicalOperation.read_result(path).require() + + @classmethod + def stop_copy(cls, target: RunTarget, directory: Path) -> None: + script = cls.operation_file(StopRootOperation(target), directory) + with SpawnedProcess.launch( + (str(target.launch_runtime.interpreter), "-I", str(script)), + env=LaunchRoot.environment(ThreadLaunch.strip(dict(os.environ))), + ) as child: + child.collect(timeout=30).check_returncode() + cls.result(directory) + + def _packet(self, operation: PhysicalOperation) -> Path: + return self.operation_file(operation, self.output / "operations" / f"{next(self.operations):06d}") + + def __enter__(self): + self.output.mkdir(mode=0o700, parents=True) + self.dump = self.output / "st-screen.txt" + self.log = (self.output / "st.log").open("wb") + script = self._packet(ExecOperation(self.target, ProcessIdentity.current(), self.command)) + self.terminal = SpawnedProcess.launch( + ("st", "-f", "DejaVu Sans Mono:pixelsize=16", + "-g", f"{self.requested_size.width}x{self.requested_size.height}", "-o", str(self.dump), "-e", "env", + f"HOME={self.target.home}", f"XDG_CONFIG_HOME={self.target.config}", + f"XDG_STATE_HOME={self.target.state}", f"XDG_DATA_HOME={self.target.data}", + str(self.target.launch_runtime.interpreter), "-I", str(script)), + cwd=self.cwd, env=self.environment, + stdio=DetachedChildStdio(stdout=self.log, stderr=self.log), + ) + try: + self.window = int(subprocess.check_output( + ("xdotool", "search", "--sync", "--onlyvisible", "--pid", str(self.terminal.identity.pid)), + env=self.environment, text=True, timeout=10).splitlines()[0]) + deadline = time.monotonic() + self.startup_seconds + self._await_result(script.parent, deadline) + self.frontend = self.result(script.parent).process() + # Wait until the exec wrapper has left and the real interpreter has + # emitted terminal data. Neither is application/source readiness: + # that answer comes from the live Toad owner in observe() below. + while (Path(f"/proc/{self.frontend.pid}/exe").resolve() + != self.target.launch_runtime.interpreter.resolve() + or os.fsencode(script) in Path(f"/proc/{self.frontend.pid}/cmdline").read_bytes() + or not self.dump.stat().st_size): + self._require_starting(deadline) + time.sleep(.05) + self.read_grid() + while True: + try: + self.observe() + break + except PendingRun: + self._require_starting(deadline) + time.sleep(.1) + # -o enables st's raw printer too. Even these printer keys pass the + # same source admission as every later physical key/click. + self.keys("ctrl+Print") + self.wait("terminal", lambda screen: bool(screen.text.strip())) + return self + except BaseException: + self.terminal.stop_sync(deadline=time.monotonic() + self.stop_seconds) + self.terminal.close_streams() + self.log.close() + raise + + def _require_starting(self, deadline: float) -> None: + if self.terminal.process.poll() is not None or time.monotonic() >= deadline: + raise TimeoutError(f"Physical frontend did not become ready; see {self.output}") + + def _await_result(self, directory: Path, deadline: float) -> None: + while not ((directory / "result.json").exists() or (directory / "error.json").exists()): + self._require_starting(deadline) + time.sleep(.02) + + def observe(self) -> RunObservation: + self.terminal.platform.require(self.frontend) + operation = ObserveOperation(self.target, self.frontend, Path(__file__).resolve().parent) + script = self._packet(operation) + # This controller is an ancestor of its st/frontend. No arbitrary PID, + # helper privilege escalation, installed-file edit or source retry. + sys.remote_exec(self.frontend.pid, str(script)) + self._await_result(script.parent, time.monotonic() + PhysicalOperation.timeout) + observed = self.result(script.parent).observation() + if observed.process != self.frontend: + raise ValueError("Observation belongs to another frontend incarnation") + return self.target.require(observed) + + def input(self, *commands: str) -> RunObservation: + observed = self.observe() + self.terminal.platform.require(self.terminal.identity) + self.server_identity() + owner = subprocess.check_output( + ("xprop", "-display", self.display.name, "-id", str(self.window), "_NET_WM_PID"), + env=self.environment, text=True, timeout=5, + ) + if int(owner.partition("=")[2].strip()) != self.terminal.identity.pid: + raise ValueError("Window does not belong to this run's st frontend") + PhysicalOperation.publish( + self.output / "operations" / f"input-{next(self.operations):06d}.json", observed, + ) + subprocess.run( + ("xdotool", "windowfocus", "--sync", str(self.window), *commands), + env=self.environment, check=True, timeout=10, + ) + return observed + + def server_identity(self) -> None: + with XvfbDisplay(self.display.name) as actual: + if actual.process != self.display.process: + raise ValueError("The run's Xvfb incarnation changed") + + def keys(self, *keys: str) -> RunObservation: + return self.input("key", "--clearmodifiers", *keys) + + def type_text(self, text: str) -> None: + self.input("type", "--clearmodifiers", "--delay", "1", text) + + def capture(self): + """Read st's own print-screen output; screenshots record the visible window.""" + offset = self.dump.stat().st_size + observed = self.keys("shift+Print") + deadline = time.monotonic() + 2 + while True: + with self.dump.open("rb") as stream: + stream.seek(offset) + data = stream.read() + if data.count(b"\n") >= self.rows: + return ScreenCapture(data.decode("utf-8"), observed.history_regions) + if self.terminal.process.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError("st did not finish its screen dump") + time.sleep(.02) + + def reader_place(self, name): + previous = () + + def settled(screen): + nonlocal previous + rows = screen.reader_position() + same = "Agent Comms connected" not in screen.text and bool(rows) and rows == previous + previous = rows + return same + + return self.wait(name, settled).reader_position() + + def screenshot(self, name): + path = self.output / f"{name}.png" + subprocess.run(("import", "-display", self.display.name, "-window", str(self.window), str(path)), + check=True, timeout=10) + return path + + def wait(self, name, predicate, seconds=60): + deadline = time.monotonic() + seconds + previous = "" + changes = self.output / f"{name}-changes.txt" + while True: + screen = self.capture() + text = screen.text + if text != previous: + with changes.open("a") as log: + log.write(f"\n--- {time.monotonic()} ---\n{text}") + previous = text + if predicate(screen): + (self.output / f"{name}.txt").write_text(text) + return screen + if self.terminal.process.poll() is not None or time.monotonic() >= deadline: + (self.output / f"{name}-failed.txt").write_text(text) + self.screenshot(f"{name}-failed") + raise RuntimeError(f"Terminal probe {name} failed; see {self.output}") + time.sleep(.2) + + def show_thread(self, thread): + self.wait("ready", lambda screen: thread in screen.text + and "▶ Session details · Ready" in "\n".join(screen.text.splitlines()[-8:])) + self.click_cell(self.columns - 1, self.rows // 2) + return self.wait("shown", lambda screen: "User messages" in screen.text and all( + "Indexing" not in line.split("│", 1)[0] for line in screen.text.splitlines())) + + def click(self, label): + text = self.wait("find-" + label.replace("/", "-"), lambda screen: any( + label in line.split("│", 1)[0] for line in screen.text.splitlines())).text + row, line = next((row, line) for row, line in enumerate(text.splitlines()) + if label in line.split("│", 1)[0]) + self.click_cell(cell_len(line[:line.index(label)]) + cell_len(label) // 2, row) + + def toggle_group(self, label): + def header(line): + return line.split("│", 1)[0].lstrip().startswith(("▾ " + label, "▸ " + label)) + text = self.wait("find-group-" + label, lambda screen: any(map(header, screen.text.splitlines()))).text + row, line = next((row, line) for row, line in enumerate(text.splitlines()) if header(line)) + glyph = min(index for index in (line.find("▾"), line.find("▸")) if index >= 0) + self.click_cell(cell_len(line[:glyph]), row) + + def hide_category(self, label): + def control(text): + return next(((row, line) for row, line in enumerate(text.splitlines()) + if "│" in line and label in line.rsplit("│", 1)[-1]), None) + text = self.wait("find-filter-" + label.replace("/", "-"), lambda screen: control(screen.text) is not None).text + row, line = control(text) + prefix, panel = line.rsplit("│", 1) + self.click_cell(cell_len(prefix + "│" + panel[:panel.index("▐")]) + 1, row) + + def read_grid(self): + """Derive cells/padding from st's own X size hints, including after resize.""" + geometry = subprocess.check_output( + ("xdotool", "getwindowgeometry", "--shell", str(self.window)), + env=self.environment, text=True, timeout=5) + size = dict(line.split("=", 1) for line in geometry.splitlines() if "=" in line) + hints = subprocess.check_output( + ("xprop", "-display", self.display.name, "-id", str(self.window), "WM_NORMAL_HINTS"), + env=self.environment, text=True, timeout=5) + declared = {name: (int(width), int(height)) for name, width, height in re.findall( + r"program specified (minimum size|base size): (\d+) by (\d+)", hints)} + base_x, base_y = declared["base size"] + minimum_x, minimum_y = declared["minimum size"] + self.cell_width, self.cell_height = minimum_x - base_x, minimum_y - base_y + self.border_x, self.border_y = base_x / 2, base_y / 2 + self.window_region = Region(int(size["X"]), int(size["Y"]), int(size["WIDTH"]), int(size["HEIGHT"])) + self.grid_size = Size((self.window_region.width - base_x) // self.cell_width, + (self.window_region.height - base_y) // self.cell_height) + + def click_cell(self, column, row): + self.read_grid() + x = int(self.border_x + (column + .5) * self.cell_width) + y = int(self.border_y + (row + .5) * self.cell_height) + self.input("mousemove", "--window", str(self.window), str(x), str(y), "click", "1") + + def resize(self, columns: int, rows: int) -> None: + self.read_grid() + self.input("windowsize", "--sync", str(self.window), str(int(2 * self.border_x + columns * self.cell_width)), + str(int(2 * self.border_y + rows * self.cell_height))) + self.read_grid() + + def __exit__(self, *exception): + try: + if self.terminal.process.poll() is None: + try: + self.keys("ctrl+q") + self.terminal.process.wait(timeout=5) + finally: + # The process owner retires the whole exact child lifetime, + # including exceptional/half-started UI teardown. + self.terminal.stop_sync(deadline=time.monotonic() + self.stop_seconds) + finally: + self.terminal.close_streams() + self.log.close() diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py new file mode 100644 index 000000000..c807bd9e9 --- /dev/null +++ b/tools/physical_ui_remote.py @@ -0,0 +1,360 @@ +"""Typed operations of one owned physical run, copied beside their requests. + +The exec operation grants only its recorded controller's descendants debugging +access, then execs the unchanged launcher. Observation reads live source owners; +it never opens a Comms, sends input, repairs a route or traverses the heap. +""" + +from abc import abstractmethod +import ctypes +from dataclasses import dataclass +import json +import os +from pathlib import Path +import pwd +import sys +import time +from typing import ClassVar + +from agent_comms.child_process import ProcessIdentity +from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.nominal.declared_family import DeclaredFamily +from agent_comms.nominal.field_codec import DeclaredWireType, FieldCodec +from textual.geometry import Region +from agent_comms.store_files import ExclusiveAtomicFile + + +class PendingRun(RuntimeError): + """The observed startup has not acquired its source yet; no input is allowed.""" + + +@dataclass(frozen=True) +class RunTarget: + """Intended private resources, distinct from an observation of the app.""" + + root: Path + home: Path + runtime: Path + + def __post_init__(self): + for path in (self.root, self.home, self.runtime): + if path != path.resolve(strict=True): + raise ValueError(f"A physical run needs canonical existing paths: {path}") + workspace = self.home.parent + user_home = Path(pwd.getpwuid(os.getuid()).pw_dir).resolve() + if workspace == user_home or user_home.is_relative_to(workspace): + raise ValueError("A physical run cannot use the real user's home as its workspace") + if not self.root.is_relative_to(workspace) or self.root == workspace: + raise ValueError("The copied root and private home must belong to one run workspace") + if self.root == self.home or self.root.is_relative_to(self.home): + raise ValueError("The copied root is separate from frontend persistence") + + @property + def launch_runtime(self) -> LaunchRuntime: + return LaunchRuntime.at_prefix(self.runtime) + + @property + def config(self) -> Path: + return self.home / "config" + + @property + def state(self) -> Path: + return self.home / "state" + + @property + def data(self) -> Path: + return self.home / "data" + + def require(self, observed): + if observed.home != self.home or observed.runtime != self.runtime: + raise ValueError("Post-attachment HOME/runtime differs from the run target") + if (observed.config, observed.state, observed.data) != (self.config, self.state, self.data): + raise ValueError("Post-attachment XDG persistence differs from the private run") + if not observed.database.is_relative_to(self.state): + raise ValueError("Toad's actual database is outside private state") + if observed.root != self.root: + raise ValueError("Core selected a foreign source") + if not observed.opened: + raise PendingRun("Core has not opened the intended private source") + for root in (*observed.saved_roots, *observed.acp_roots): + if root != self.root: + raise ValueError(f"Foreign saved/ACP source refuses physical input: {root}") + return observed + + +@dataclass(frozen=True) +class CapturedRegion(DeclaredWireType): + """Wire capability for Textual's final compiled Region, without copying its fields. + + The installed Region rejects subclassing; geometry remains on that existing + type. Only this private boundary's representation lives here. + """ + + region: Region + wire_type = tuple[int, ...] + + def to_wire(self): + return list(self.region) + + @classmethod + def from_wire(cls, data): + return cls(Region(*FieldCodec.decode(cls.wire_type, data))) + + +@dataclass(frozen=True) +class RunObservation: + process: ProcessIdentity + started_ns: int + finished_ns: int + runtime: Path + home: Path + config: Path + state: Path + data: Path + database: Path + root: Path + opened: bool + saved_roots: tuple[Path, ...] + acp_roots: tuple[Path, ...] + history_regions: tuple[CapturedRegion, ...] + modules_checked: int + toad_module: Path + core_module: Path + textual_module: Path + + +class PhysicalResult(DeclaredFamily, affix="Result"): + def require(self): + return self + + def observation(self) -> RunObservation: + raise ValueError("This physical operation did not observe an application") + + def process(self) -> ProcessIdentity: + raise ValueError("This physical operation did not exec a frontend") + + +@dataclass(frozen=True) +class ObservedResult(PhysicalResult): + value: RunObservation + + def observation(self) -> RunObservation: + return self.value + + +@dataclass(frozen=True) +class ExecResult(PhysicalResult): + identity: ProcessIdentity + debugger: ProcessIdentity + + def process(self) -> ProcessIdentity: + return self.identity + + +@dataclass(frozen=True) +class StoppedResult(PhysicalResult): + root: Path + + +@dataclass(frozen=True) +class PendingResult(PhysicalResult): + detail: str + + def require(self): + raise PendingRun(self.detail) + + def observation(self) -> RunObservation: + self.require() + + +@dataclass(frozen=True) +class FailedResult(PhysicalResult): + error: str + detail: str + + def require(self): + raise RuntimeError(f"Physical operation failed: {self.error}: {self.detail}") + + def observation(self) -> RunObservation: + self.require() + + def process(self) -> ProcessIdentity: + self.require() + + +class PhysicalOperation(DeclaredFamily, affix="Operation"): + metadata_bytes: ClassVar[int] = 65536 + source_bytes: ClassVar[int] = 256 << 10 + timeout: ClassVar[float] = 10.0 + + @abstractmethod + def execute(self, directory: Path) -> None: ... + + @classmethod + def publish(cls, path: Path, value: object) -> None: + encoded = json.dumps(FieldCodec.encode(value), ensure_ascii=False, allow_nan=False).encode() + if len(encoded) > cls.metadata_bytes: + raise ValueError("Physical metadata exceeds its byte bound") + ExclusiveAtomicFile(path).write_bytes(encoded) + + @classmethod + def read_result(cls, path: Path) -> PhysicalResult: + with path.open("rb") as source: + raw = source.read(cls.metadata_bytes + 1) + if len(raw) > cls.metadata_bytes: + raise ValueError("Oversize physical result") + return FieldCodec.decode(PhysicalResult, FieldCodec.parse(raw.decode())) + + @classmethod + def execute_file(cls, script: Path) -> None: + directory = script.parent + result = directory / "result.json" + try: + with script.with_suffix(".json").open("rb") as source: + raw = source.read(cls.metadata_bytes + 1) + if len(raw) > cls.metadata_bytes: + raise ValueError("Oversize physical request") + operation = FieldCodec.decode(cls, FieldCodec.parse(raw.decode())) + operation.execute(directory) + except PendingRun as error: + cls.publish(result, PendingResult(str(error)[:2048])) + except BaseException as error: + cls.publish(directory / "error.json", FailedResult(type(error).__name__, str(error)[:2048])) + + +@dataclass(frozen=True) +class ExecOperation(PhysicalOperation): + """One new child, not an arbitrary existing trace target or global policy.""" + + target: RunTarget + debugger: ProcessIdentity + arguments: tuple[str, ...] + PR_SET_PTRACER: ClassVar[int] = 0x59616D61 # Linux's declared prctl ABI. + + def execute(self, directory: Path) -> None: + if not self.arguments or not self.debugger.alive(): + raise ValueError("Exec needs its original living controller and command") + if Path.home().resolve() != self.target.home: + raise ValueError("st changed HOME before frontend exec") + prctl = ctypes.CDLL(None, use_errno=True).prctl + prctl.argtypes = (ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong) + prctl.restype = ctypes.c_int + if prctl(self.PR_SET_PTRACER, self.debugger.pid, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "Cannot admit this controller's debugger descendants") + self.publish(directory / "result.json", ExecResult(ProcessIdentity.current(), self.debugger)) + os.execvpe(self.arguments[0], self.arguments, os.environ) + + +@dataclass(frozen=True) +class ObserveOperation(PhysicalOperation): + target: RunTarget + identity: ProcessIdentity + tools: Path + + def execute(self, directory: Path) -> None: + started = time.monotonic_ns() + if ProcessIdentity.current() != self.identity: + raise ValueError("Post-attachment frontend incarnation changed") + if Path.home().resolve() != self.target.home: + raise ValueError("Post-attachment HOME is not private") + # The existing reviewed validator remains the single module-location policy. + # The temporary tool import path is retired before inspecting application state. + previous = sys.path[:] + try: + sys.path.insert(0, str(self.tools)) + from performance.history_capture_remote import require_modules + finally: + sys.path[:] = previous + import agent_comms + import textual + import toad + from agent_comms.acp_extension import CoordinationChangedUpdate + from metaclass_registry.mro_dispatch import handles + from toad.acp.agent import Agent + from toad.agent_binding import NoAgentConnection + from toad.app import ToadApp + from textual._context import NoActiveAppError + from textual.app import ScreenStackError + from toad.core.projection import MroProjection + from toad.widgets.history_anchor import HistoryWindow + from xdg_base_dirs import xdg_config_home, xdg_data_home, xdg_state_home + + class AcpRoots(MroProjection[tuple[Path, ...]]): + """Decode foreign running-UI connections once, without guessing readiness.""" + + @handles(object) + def unknown(self, value): + raise ValueError(f"Unadmitted ACP metadata type: {type(value).__name__}") + + @handles(type(None)) + def pending(self, value): + raise PendingRun("A bound ACP has not published its coordination root") + + @handles(NoAgentConnection) + def unbound(self, value): + return () + + @handles(Agent) + def bound(self, value): + return self.project_value(value.coordination) + + @handles(CoordinationChangedUpdate) + def coordinated(self, value): + return (Path(value.wire_root).resolve(),) + + try: + app = ToadApp.running_app() + screen = app.screen + except (NoActiveAppError, ScreenStackError) as error: + raise PendingRun("No sole running Toad application with a screen yet") from error + checked = require_modules(self.target.runtime) + access = app.coordination_access + root = access.selection.root + sessions = app.session_navigation + saved = tuple(Path(source).resolve() for entry in sessions.members + for source, _identity in (*entry.original_threads(sessions), *entry.original_channels())) + decoder = AcpRoots() + acp = tuple(source for process in dict.fromkeys(sessions.processes) + for agent in (process.agent, *(session.agent for session in process.sessions)) + for source in decoder.project_value(agent)) + # This is geometry of actually published history windows, not a guess + # from sidebar separators that may also occur inside a user's message. + regions = tuple(CapturedRegion(region) for window in screen.query(HistoryWindow) + for region in (window.published_content_region,) if region is not None) + observed = RunObservation( + self.identity, started, time.monotonic_ns(), Path(sys.prefix).resolve(), + Path.home().resolve(), xdg_config_home().resolve(), xdg_state_home().resolve(), + xdg_data_home().resolve(), app.tab_order._db.path.resolve(), root, + access.observation.observes(root), saved, acp, regions, checked, + Path(toad.__file__).resolve(), Path(agent_comms.__file__).resolve(), + Path(textual.__file__).resolve(), + ) + self.target.require(observed) + self.publish(directory / "result.json", ObservedResult(observed)) + + +@dataclass(frozen=True) +class StopRootOperation(PhysicalOperation): + """Stop only the private copy, in its own declared runtime; retain its files.""" + + target: RunTarget + + def execute(self, directory: Path) -> None: + if LaunchRuntime.running().prefix != self.target.runtime: + raise ValueError("Private-copy shutdown must run in its declared runtime") + from agent_comms.comms import wire + from agent_comms.launch_root import LaunchRoot + from agent_comms.registry_document import RosterVisibility + + with LaunchRoot.from_path(self.target.root).selected(): + copied = wire() + for thread in copied.registry.snapshot().visible_threads(RosterVisibility(archived=True)): + copied.owners.stop(thread.name) + self.publish(directory / "result.json", StoppedResult(self.target.root)) + + +# Normal imports define the family. A controller-bound module filename selects +# only that immutable copy's adjacent request, including under remote_exec. +_source = Path(__file__) +if _source.with_suffix(".json").is_file(): + PhysicalOperation.execute_file(_source) From d67941cb3dfbd7c4862ae53bee9b6e89e20a8f1d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 31 Oct 2026 21:52:13 -0400 Subject: [PATCH 02/35] Freeze run operations and retain exact child retirement evidence --- docs/performance/physical-run.md | 38 +++++++++--- tools/physical_ui.py | 103 +++++++++++++++++++++---------- tools/physical_ui_remote.py | 21 ++++++- 3 files changed, 118 insertions(+), 44 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 6df1d4b5a..74d518bbd 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -40,12 +40,18 @@ acceptance and actual borrower checks. source owner creates history; only its disposable copy is published through `RootRuntime.publish` to a supplied runtime. - Fresh Toad config/state/data and no copied saved-tab database. Presentation - settings can travel; source identities cannot be repaired by rewriting tabs. + settings derive from the declarations on the `INTERFACE` page; session/shell/ + launcher settings and the original anonymous identity do not travel. Collection + is disabled through `SavedSettings`. The optional palette is copied/validated + through `WalFile` and the actual private palette directory is observed. + Source identities cannot be repaired by rewriting tabs. - HOME/XDG inside `st -e env`: st itself resets HOME from passwd before exec. - Xvfb uses displayfd, not a guessed display number. Xlib checks its actual local peer UID/process; the run verifies the allocated server incarnation. - Child exec grants debugging only to the recorded controller PID and its - descendants. It changes no global ptrace policy and needs no sudo. + descendants. It changes no global ptrace policy and needs no sudo. The + controller owns one bounded operation-source snapshot for its entire Python + lifetime; editing the checkout cannot change an already started run's wire. - Before physical input: the actual frontend incarnation, runtime and loaded Toad/Core/Textual module locations, HOME/XDG/database, selected/opened source, every saved entry's source and every borrowed ACP process's creator/session @@ -88,11 +94,23 @@ not become authorized merely because they import the shared display resource. ## Current qualification -Core #3821 is an API checkpoint; full acceptance is separate. Prototype7 -completed with exit0: actual copied b34 frontend/source/home/runtime admission, -held PageUp/PageDown, reversal, End and wheel input, screenshots, a continuous -recording and typed copied-root stop. Its source controller still used the -owned Core overlay, and it had no bound ACP. Earlier failed run receipts remain -retained. Candidate admission, installed-source qualification, four isolation -counterexamples, tab/resize/arrival/retirement coverage and final profiling must -be reported individually; none is implied by this bounded scrolling result. +Core #3821 is installed in 7d0d; the merger reports 23 selected checks passed. +Fresh runs10 and12 used that installed source interpreter with `-I` and no Core +source overlay, and observed all three actual package origins in the supplied +mobile candidate. Run12 completed exit0 through held PageUp/PageDown, reversal, +End and wheel input, with screenshots/recording and typed private-copy stop. +The st, Xvfb and encoder receipts name their exact process births and existing +child-owner outcomes; their PIDs and the frontend were independently absent. +There were **no bound ACPs**. Four isolation counterexamples, the wider tab/ +resize/arrival journey and actual profiling remain separate obligations. + +Run8's foreign candidate-module origin, run9's missing-palette warning and +callback timeout, and run11's first-observation timeout remain retained. A +missing palette was demonstrated, not proved to explain every timeout. Failed +or uncertain operations now go straight to the existing owned-process shutdown; +exit does not attempt another attachment/key. No such attempt is replayed. + +Current helpers are `tools/physical_ui.py` and `tools/physical_ui_remote.py` on +PR #1393. Consumers must start a new controller from one published checkpoint, +not combine a live process with a newly imported wire shape. A completed +stopped-history scenario is not bound-ACP or performance acceptance. diff --git a/tools/physical_ui.py b/tools/physical_ui.py index bbfb1ba49..ad0917fc6 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -20,6 +20,7 @@ import subprocess import sys import time +from typing import get_args, get_type_hints from agent_comms.active_route import ActiveRoute, active_route_path, route_new_bus from agent_comms.bus import Bus @@ -29,6 +30,7 @@ from agent_comms.installer.practice import SuppliedRuntimeHome from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime, ThreadLaunch +from agent_comms.nominal.field_codec import FieldCodec from agent_comms.recorded_bus import RecordedBusIdentity from agent_comms.root_runtime import RootRuntime from agent_comms.store_files import ExclusiveAtomicFile @@ -36,10 +38,11 @@ from rich.segment import Segment from textual.geometry import Region, Size from textual.strip import Strip +from ui_theme.wal import WalFile from physical_ui_remote import ( CapturedRegion, ExecOperation, ObserveOperation, PendingRun, PhysicalOperation, - RunObservation, RunTarget, StopRootOperation, + ProcessRetirement, RunObservation, RunTarget, StopRootOperation, ) @@ -105,12 +108,18 @@ def recording(self, path: Path, environment: Mapping[str, str]): env=environment, stdio=DetachedChildStdio(stdin=subprocess.PIPE, stdout=log, stderr=log), ) as child: + PhysicalOperation.publish(path.with_suffix(".process.json"), child.identity) try: yield path finally: - child.process.communicate(b"q", timeout=10) - if child.process.returncode: - raise RuntimeError(f"Physical recording failed; see {log.name}") + try: + child.process.communicate(b"q", timeout=10) + if child.process.returncode: + raise RuntimeError(f"Physical recording failed; see {log.name}") + finally: + outcome = child.stop_sync(deadline=time.monotonic() + 10) + PhysicalOperation.publish(path.with_suffix(".retired.json"), + ProcessRetirement(child.identity, outcome)) def __exit__(self, *exception): return self.close_display.__exit__(*exception) @@ -123,6 +132,7 @@ class OwnedDisplay(XvfbDisplay): stop_seconds = 10.0 def __init__(self, output: Path): + self.output = output output.mkdir(parents=True, exist_ok=True) self.log = (output / "xvfb.log").open("wb") self.server = SpawnedProcess.launch( @@ -139,13 +149,16 @@ def __init__(self, output: Path): if self.process != self.server.identity: self.close_display.close() raise ValueError("Xlib connected to a server this run did not create") + PhysicalOperation.publish(output / "xvfb-process.json", self.process) except BaseException: self.close_server() raise def close_server(self) -> None: try: - self.server.stop_sync(deadline=time.monotonic() + self.stop_seconds) + outcome = self.server.stop_sync(deadline=time.monotonic() + self.stop_seconds) + PhysicalOperation.publish(self.output / "xvfb-retired.json", + ProcessRetirement(self.server.identity, outcome)) finally: self.server.close_streams() self.log.close() @@ -197,6 +210,12 @@ class TerminalApp: startup_seconds = 60.0 stop_seconds = 10.0 + # One source snapshot belongs to the controller's Python module lifetime, + # not to each callback executing inside a foreign running application. + with Path(__file__).with_name("physical_ui_remote.py").open("rb") as _source: + operation_program = _source.read(PhysicalOperation.source_bytes + 1) + if len(operation_program) > PhysicalOperation.source_bytes: + raise ValueError("Physical operation source exceeds its bound") def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, target: RunTarget, cwd: Path, *, size: Size = Size(140, 40)): @@ -210,6 +229,7 @@ def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, HOME=str(target.home), DISPLAY=display.name, XDG_CONFIG_HOME=str(target.config), XDG_STATE_HOME=str(target.state), XDG_DATA_HOME=str(target.data), + PYWAL_CACHE_DIR=str(target.palette), AGENT_COMMS_RUNTIME_ROOT=str(target.launch_runtime.interpreter.parent), ) self.environment = environment @@ -230,7 +250,7 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: Only presentation settings travel. Saved Toad databases/tabs never do. Existing installer acceptance and copied-history runs share this setup. """ - from toad.preferences import SavedSettings + from toad.preferences import INTERFACE, SETTINGS, SavedSettings, ToadSettings root = root.resolve(strict=True) home = root.parent / "frontend-home" @@ -250,10 +270,24 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: for directory in (target.config, target.state, target.data): directory.mkdir(mode=0o700) settings = SavedSettings.path() + destination = target.config / settings.parent.name / settings.name + destination.parent.mkdir(mode=0o700) + selected = {} if settings.is_file(): - destination = target.config / settings.parent.name / settings.name - destination.parent.mkdir(mode=0o700) - ExclusiveAtomicFile(destination).write_bytes(settings.read_bytes()) + raw = FieldCodec.decode(dict[str, object], FieldCodec.parse(settings.read_text())) + annotations = get_type_hints(ToadSettings, include_extras=True) + presentation = frozenset(key for field, key in FieldCodec.record_fields(ToadSettings) + if INTERFACE in get_args(annotations[field.name])) + selected = {key: value for key, value in raw.items() if key in presentation} + private_settings = SavedSettings(selected) + private_settings.set(SETTINGS.statistics.allow_collect, False) + ExclusiveAtomicFile(destination).write_bytes(private_settings.json.encode()) + palette = WalFile.directory(os.environ) / WalFile.NAME + if palette.is_file(): + target.palette.mkdir(mode=0o700, parents=True) + destination = target.palette / WalFile.NAME + ExclusiveAtomicFile(destination).write_bytes(palette.read_bytes()) + WalFile.read(destination) # Validate the exact copied bytes through their format owner. route = ActiveRoute(root, RecordedBusIdentity.capture(Bus.path_in(root))) route_path = home / active_route_path().relative_to(Path.home()) route_new_bus(route, route_path) @@ -282,14 +316,12 @@ def copied(cls, source: Comms, threads: tuple[str, ...], runtime: LaunchRuntime, # Files remain until the actual acceptance and borrower checks retire them. cls.stop_copy(target, output / "retirement") - @staticmethod - def operation_file(operation: PhysicalOperation, directory: Path) -> Path: + @classmethod + def operation_file(cls, operation: PhysicalOperation, directory: Path) -> Path: directory.mkdir(mode=0o700, parents=True) - source = Path(__file__).with_name("physical_ui_remote.py") - with source.open("rb") as stream: - raw = stream.read(PhysicalOperation.source_bytes + 1) - if len(raw) > PhysicalOperation.source_bytes: - raise ValueError("Physical operation source exceeds its bound") + # The declaration module owns one bounded program snapshot for this + # controller lifetime. Later source edits cannot change a run's wire. + raw = cls.operation_program script = directory / "operation.py" # remote_exec supplies no module filename. Bind only that external # datum; the program stays in its reviewed source file and the request @@ -333,6 +365,7 @@ def __enter__(self): stdio=DetachedChildStdio(stdout=self.log, stderr=self.log), ) try: + PhysicalOperation.publish(self.output / "st-process.json", self.terminal.identity) self.window = int(subprocess.check_output( ("xdotool", "search", "--sync", "--onlyvisible", "--pid", str(self.terminal.identity.pid)), env=self.environment, text=True, timeout=10).splitlines()[0]) @@ -362,9 +395,7 @@ def __enter__(self): self.wait("terminal", lambda screen: bool(screen.text.strip())) return self except BaseException: - self.terminal.stop_sync(deadline=time.monotonic() + self.stop_seconds) - self.terminal.close_streams() - self.log.close() + self.close() raise def _require_starting(self, deadline: float) -> None: @@ -387,10 +418,7 @@ def observe(self) -> RunObservation: observed = self.result(script.parent).observation() if observed.process != self.frontend: raise ValueError("Observation belongs to another frontend incarnation") - return self.target.require(observed) - - def input(self, *commands: str) -> RunObservation: - observed = self.observe() + self.target.require(observed) self.terminal.platform.require(self.terminal.identity) self.server_identity() owner = subprocess.check_output( @@ -399,6 +427,10 @@ def input(self, *commands: str) -> RunObservation: ) if int(owner.partition("=")[2].strip()) != self.terminal.identity.pid: raise ValueError("Window does not belong to this run's st frontend") + return observed + + def input(self, *commands: str) -> RunObservation: + observed = self.observe() PhysicalOperation.publish( self.output / "operations" / f"input-{next(self.operations):06d}.json", observed, ) @@ -447,6 +479,7 @@ def settled(screen): return self.wait(name, settled).reader_position() def screenshot(self, name): + self.observe() path = self.output / f"{name}.png" subprocess.run(("import", "-display", self.display.name, "-window", str(self.window), str(path)), check=True, timeout=10) @@ -534,16 +567,22 @@ def resize(self, columns: int, rows: int) -> None: str(int(2 * self.border_y + rows * self.cell_height))) self.read_grid() - def __exit__(self, *exception): + def close(self) -> None: + """Retire one child lifetime on normal exit and failed entry alike.""" try: - if self.terminal.process.poll() is None: - try: - self.keys("ctrl+q") - self.terminal.process.wait(timeout=5) - finally: - # The process owner retires the whole exact child lifetime, - # including exceptional/half-started UI teardown. - self.terminal.stop_sync(deadline=time.monotonic() + self.stop_seconds) + outcome = self.terminal.stop_sync(deadline=time.monotonic() + self.stop_seconds) + PhysicalOperation.publish(self.output / "st-retired.json", + ProcessRetirement(self.terminal.identity, outcome)) finally: self.terminal.close_streams() self.log.close() + + def __exit__(self, exception_type, exception, traceback): + try: + # An exceptional/uncertain operation is not permission for another + # attachment or input. Retire only the already owned child instead. + if exception_type is None and self.terminal.process.poll() is None: + self.keys("ctrl+q") + self.terminal.process.wait(timeout=5) + finally: + self.close() diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index c807bd9e9..0ed1627e4 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -16,11 +16,12 @@ import time from typing import ClassVar -from agent_comms.child_process import ProcessIdentity +from agent_comms.child_process import ChildOutcome, ProcessIdentity from agent_comms.launch_runtime import LaunchRuntime from agent_comms.nominal.declared_family import DeclaredFamily from agent_comms.nominal.field_codec import DeclaredWireType, FieldCodec from textual.geometry import Region +from ui_theme.wal import WalFile from agent_comms.store_files import ExclusiveAtomicFile @@ -53,6 +54,10 @@ def __post_init__(self): def launch_runtime(self) -> LaunchRuntime: return LaunchRuntime.at_prefix(self.runtime) + @property + def palette(self) -> Path: + return WalFile.directory({"HOME": str(self.home)}) + @property def config(self) -> Path: return self.home / "config" @@ -70,6 +75,8 @@ def require(self, observed): raise ValueError("Post-attachment HOME/runtime differs from the run target") if (observed.config, observed.state, observed.data) != (self.config, self.state, self.data): raise ValueError("Post-attachment XDG persistence differs from the private run") + if observed.palette_directory != self.palette: + raise ValueError("The actual theme palette is outside the private home") if not observed.database.is_relative_to(self.state): raise ValueError("Toad's actual database is outside private state") if observed.root != self.root: @@ -112,6 +119,7 @@ class RunObservation: state: Path data: Path database: Path + palette_directory: Path root: Path opened: bool saved_roots: tuple[Path, ...] @@ -123,6 +131,14 @@ class RunObservation: textual_module: Path +@dataclass(frozen=True) +class ProcessRetirement: + """The existing child owner's result, qualified by the exact resource birth.""" + + identity: ProcessIdentity + outcome: ChildOutcome + + class PhysicalResult(DeclaredFamily, affix="Result"): def require(self): return self @@ -324,7 +340,8 @@ def coordinated(self, value): observed = RunObservation( self.identity, started, time.monotonic_ns(), Path(sys.prefix).resolve(), Path.home().resolve(), xdg_config_home().resolve(), xdg_state_home().resolve(), - xdg_data_home().resolve(), app.tab_order._db.path.resolve(), root, + xdg_data_home().resolve(), app.tab_order._db.path.resolve(), + app.colours.wal_directory.resolve(), root, access.observation.observes(root), saved, acp, regions, checked, Path(toad.__file__).resolve(), Path(agent_comms.__file__).resolve(), Path(textual.__file__).resolve(), From 2c862054ac64d12a1dc3caca407e549eb68820d0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 31 Oct 2026 21:58:03 -0400 Subject: [PATCH 03/35] Give physical runs an explicit framebuffer and reject clipped windows --- docs/performance/physical-run.md | 11 ++++++++--- tools/performance/scroll_scenario.py | 11 +++++++++-- tools/physical_ui.py | 22 ++++++++++++++++------ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 74d518bbd..5613f9b4b 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -25,10 +25,14 @@ private copy's runtime. The source controller remains installed. The candidate must contain its own Toad/Core/Textual packages; a dependency `.pth` pointing those packages outside the selected prefix fails actual module-origin checks. Supply a new output directory for each distinct run. Do not replay a failed or -UNKNOWN input against an old run. +UNKNOWN input against an old run. Terminal dimensions are **cells**; framebuffer +sizes are **pixels**. For the mobile48×48 →140×40 journey, supply +`--columns 48 --rows 48 --framebuffer 1600 1200 --resize 140 40`, or pass +`framebuffer=Size(1600, 1200)` to the callable factory. The owner checks actual +native window bounds and rejects clipping instead of recording half a terminal. The callable entry point is `TerminalApp.copied(source, threads, runtime, -output, thread=..., size=Size(...))`. Its context yields the admitted app and +output, thread=..., size=Size(...), framebuffer=Size(...))`. Its context yields the admitted app and retains the copied root, frontend-home, operation receipts, screenshots and recording on exit. The caller still owns retiring those retained files after acceptance and actual borrower checks. @@ -101,7 +105,8 @@ mobile candidate. Run12 completed exit0 through held PageUp/PageDown, reversal, End and wheel input, with screenshots/recording and typed private-copy stop. The st, Xvfb and encoder receipts name their exact process births and existing child-owner outcomes; their PIDs and the frontend were independently absent. -There were **no bound ACPs**. Four isolation counterexamples, the wider tab/ +Run13 also completed the48×48 →140×40 candidate geometry journey in a1600×1200 +owned framebuffer. There were **no bound ACPs**. Four isolation counterexamples, the wider tab/ resize/arrival journey and actual profiling remain separate obligations. Run8's foreign candidate-module origin, run9's missing-palette warning and diff --git a/tools/performance/scroll_scenario.py b/tools/performance/scroll_scenario.py index 419e551ca..3046f594b 100644 --- a/tools/performance/scroll_scenario.py +++ b/tools/performance/scroll_scenario.py @@ -16,7 +16,7 @@ from agent_comms.comms import wire from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime -from physical_ui import TerminalApp +from physical_ui import OwnedDisplay, TerminalApp from physical_ui_remote import ObservedResult, PhysicalOperation from textual.geometry import Size @@ -29,6 +29,10 @@ def main(): parser.add_argument("--output", type=Path, required=True, help="Fresh owned agent-scratch directory") parser.add_argument("--columns", type=int, default=140) parser.add_argument("--rows", type=int, default=40) + parser.add_argument("--framebuffer", nargs=2, type=int, default=OwnedDisplay.default_size, + metavar=("WIDTH", "HEIGHT"), help="Owned display size in pixels, not terminal cells") + parser.add_argument("--resize", nargs=2, type=int, action="append", default=[], + metavar=("COLUMNS", "ROWS"), help="Resize through the same owner after scrolling") arguments = sys.argv[1:] LaunchRoot.startup(arguments, ActiveRoute.published_launch) args = parser.parse_args(arguments) @@ -39,7 +43,7 @@ def main(): threads = tuple(dict.fromkeys((args.thread, *args.history_thread))) with TerminalApp.copied( wire(), threads, LaunchRuntime.at_prefix(args.runtime), args.output, - thread=args.thread, size=Size(args.columns, args.rows), + thread=args.thread, size=Size(args.columns, args.rows), framebuffer=Size(*args.framebuffer), ) as app: app.wait("history", lambda screen: bool(screen.reader_position())) PhysicalOperation.publish(args.output / "admitted.json", ObservedResult(app.observe())) @@ -55,6 +59,9 @@ def main(): app.screenshot("wheel-up") app.keys("End") app.screenshot("end-return") + for columns, rows in args.resize: + app.resize(columns, rows) + app.screenshot(f"resized-{columns}x{rows}") print(args.output) diff --git a/tools/physical_ui.py b/tools/physical_ui.py index ad0917fc6..d64b4ed1f 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -130,13 +130,14 @@ class OwnedDisplay(XvfbDisplay): startup_seconds = 10.0 stop_seconds = 10.0 + default_size = Size(1600, 1000) - def __init__(self, output: Path): + def __init__(self, output: Path, *, size: Size = default_size): self.output = output output.mkdir(parents=True, exist_ok=True) self.log = (output / "xvfb.log").open("wb") self.server = SpawnedProcess.launch( - ("Xvfb", "-displayfd", "1", "-screen", "0", "1600x1000x24", "-nolisten", "tcp"), + ("Xvfb", "-displayfd", "1", "-screen", "0", f"{size.width}x{size.height}x24", "-nolisten", "tcp"), stdio=DetachedChildStdio(stdout=subprocess.PIPE, stderr=self.log), ) try: @@ -296,7 +297,8 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: @classmethod @contextmanager def copied(cls, source: Comms, threads: tuple[str, ...], runtime: LaunchRuntime, - output: Path, *, thread: str, size: Size = Size(140, 40)) -> Iterator[TerminalApp]: + output: Path, *, thread: str, size: Size = Size(140, 40), + framebuffer: Size = OwnedDisplay.default_size) -> Iterator[TerminalApp]: """Create under the source owner, bind only its copy, retain run evidence.""" output = output.expanduser().absolute() output.mkdir(mode=0o700, parents=True) # Fresh evidence, never overwrite a failed run. @@ -304,7 +306,7 @@ def copied(cls, source: Comms, threads: tuple[str, ...], runtime: LaunchRuntime, RootRuntime.publish(copied.root, runtime) target = cls.prepare_target(copied.root, runtime) try: - with OwnedDisplay(output / "display") as display: + with OwnedDisplay(output / "display", size=framebuffer) as display: # Keep the real default launcher: its root/runtime succession is # precisely what the post-attachment observation must establish. command = (str(Path.home() / ".local/bin/toad-comms"), thread) @@ -427,6 +429,7 @@ def observe(self) -> RunObservation: ) if int(owner.partition("=")[2].strip()) != self.terminal.identity.pid: raise ValueError("Window does not belong to this run's st frontend") + self.read_grid() return observed def input(self, *commands: str) -> RunObservation: @@ -552,9 +555,14 @@ def read_grid(self): self.cell_width, self.cell_height = minimum_x - base_x, minimum_y - base_y self.border_x, self.border_y = base_x / 2, base_y / 2 self.window_region = Region(int(size["X"]), int(size["Y"]), int(size["WIDTH"]), int(size["HEIGHT"])) + self.require_visible(self.window_region) self.grid_size = Size((self.window_region.width - base_x) // self.cell_width, (self.window_region.height - base_y) // self.cell_height) + def require_visible(self, region: Region) -> None: + if not Region(0, 0, *self.display.size).contains_region(region): + raise ValueError(f"Terminal {region} exceeds the owned framebuffer {self.display.size}") + def click_cell(self, column, row): self.read_grid() x = int(self.border_x + (column + .5) * self.cell_width) @@ -563,8 +571,10 @@ def click_cell(self, column, row): def resize(self, columns: int, rows: int) -> None: self.read_grid() - self.input("windowsize", "--sync", str(self.window), str(int(2 * self.border_x + columns * self.cell_width)), - str(int(2 * self.border_y + rows * self.cell_height))) + width = int(2 * self.border_x + columns * self.cell_width) + height = int(2 * self.border_y + rows * self.cell_height) + self.require_visible(Region(self.window_region.x, self.window_region.y, width, height)) + self.input("windowsize", "--sync", str(self.window), str(width), str(height)) self.read_grid() def close(self) -> None: From 5340278ba19e6aa8e0196d855eb27cfe45cdd3bc Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 00:04:24 -0400 Subject: [PATCH 04/35] Record the debugger context repair and private functional journey --- docs/performance/cpython-debugger-context.md | 122 ++++++++++++++++++ .../cpython-debugger-context.patch | 79 ++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 docs/performance/cpython-debugger-context.md create mode 100644 docs/performance/cpython-debugger-context.patch diff --git a/docs/performance/cpython-debugger-context.md b/docs/performance/cpython-debugger-context.md new file mode 100644 index 000000000..00dd40776 --- /dev/null +++ b/docs/performance/cpython-debugger-context.md @@ -0,0 +1,122 @@ +# Remote debugger context: functional candidate, not an installed fix + +The accompanying `cpython-debugger-context.patch` belongs to CPython's +`Python/ceval_gil.c`, not Toad's renderer or its runner. It is retained here so +this runner's platform dependency, exact proposed repair and demonstrated scope +are reviewable. It is **not** an alternate launcher, import preload, retry or +permission bypass. No live Python binary/library was replaced. + +## Failure and owner + +On CPython3.14.8, retained physical run11 and independently read mobile06 fail +before opening the helper: + +``` +Can't open debugger script .../operation.py + File "", line 1, in +KeyError: '__import__' +``` + +`run_remote_debugger_script` audits/opens/reads/closes source under the +interrupted frame. Only afterward does `run_remote_debugger_source` create its +fresh namespace. The default `PyFile_OpenCodeObject` imports `_io.open`; +`PyImport_Import` consults the interrupted frame's builtins. A legitimate empty +builtin mapping therefore breaks the debugger itself. The interrupted constructor +is unidentified; run14's separate timeout remains unexplained. + +Exact source: v3.14.8, commit +`8e6e75d9102e39bed2a2b279203a396741180f12`. The debugger owner file is identical +on the checked compatible3.14.9/3.14.10 tags;3.14.10's file-open/import owners are +also unchanged. NRA's Python AST does not parse these C files; the C call chain +was read manually and checked against installed disassembly. + +## Repair and audit difference + +The original remote-debugger audit refusal remains first. The existing loader +body becomes one private C callable, invoked through a real Python frame with +interpreter-owned builtins. The original application frame stays in the normal +frame/GC chain. The embedder's open-code hook is still called; global import +behavior, source reading/closing and payload execution are not replaced. + +**The complete audit trace is not equivalent.** Successful execution now emits: + +1. `cpython.remote_debugger_script(path)`; +2. `compile` / `exec` for the context lambda, filename ``; +3. the existing `compile` / `exec` for payload source, filename ``. + +The initial proposal incorrectly put context preparation ahead of the original +audit. Review caught this; the published patch restores that audit gate first. +The extra context events remain explicit. A policy refusing them prevents source +opening and reports a context-entry error. A close error still reports the error +then executes already-read source, matching the original loader body. + +## Functional build and native controls + +A separately owned3.14.8 build used existing GCC16.1.1/make, no new toolchain, +PGO/LTO/BOLT/JIT/tail-call chase, and at most two make/compileall jobs. Configure, +make and private altinstall completed0 in18.4s/109.6s/13.0s. Built patch SHA256: +`5a679cd8875271a4c7eace7ce088b5f93f261076b929987e1276b6161ef9857d`; +functional binary SHA256: +`e18b6c81a63f9683beca2d20c9e56225da3207b68530bae2eaa26b6bf351e4fe`. +These identify measured artifacts, not a runtime publication. Existing `BoundedRun` +owned process retirement and deadlines; no output for180s failed the stage. +This nonmatching compiler build is **not release-performance evidence**. + +One native batch completed0: + +- one fresh unpatched empty-builtins child reproduced the source-open failure; +- 18 patched controls: nine cases under full and empty interrupted builtins; +- normal execution; refusing request/compile/exec audit hooks; real embedder-hook + acceptance/refusal; read/close failure; payload failure; +- executing cases retained the original frame and cyclic guard through + `gc.collect`; every case preserved its active exception and builtin mapping; +- all19 typed target identities independently absent after retirement. + +These are exercised counterexamples/controls, not a proof over every CPython state. +The guard has other strong references; this is not a sole-root liveness proof. +The fresh control target deliberately runs a busy select loop, not a production +workload. The separate actual Toad run below exercises the unchanged application. +The disposable control extension/script targets never loaded into a user app. + +## Actual private Toad + +The unchanged shared runner then completed one real default-launcher journey0: +saved history, held PageUp/PageDown, reversal, End, wheel scrolling and normal +retirement. It obtained31 observed callbacks,17 explicit startup-pending results +and one exec result. The screenshot shows saved message bodies and STANDARD +mode; no physical timeout or source-open failure occurred in this run. + +Admission checked1013 module locations; HOME/XDG/database/palette, selected and +opened root, and saved-source identities were private. There were **zero ACP +bindings**: this does not qualify bound-ACP behavior, mobile1391 or errors1389. +The controller used the installed source owner to copy history. The target used +the functional interpreter with a byte-compared copy of5295 regular installed +package files, declaration-derived console scripts and an existing Neovim payload +verified through its archive owner. No native download or fabricated build.json. +A setup assertion initially confused Core's package root with site-packages; +that failure is retained, and only script generation was completed afterward. + +st/Xvfb/encoder retired0, all four frontend/resource identities were independently +absent, and the copied root was stopped. Original sessions, input dispositions, +phone/editor processes, desktop/pointer and scientific data were untouched. +The separately approved metadata/profile capture remains unused. + +## Receipts and remaining work + +Owned retained evidence under `/home/ts/.cache/agent-scratch/pi-ui-performance/`: + +- `cpython-functional-build.log`, `cpython-functional-startup.log`; +- `cpython-native-controls.log`, `cpython-native-control-absence.log`; +- `cpython-functional-toad-{prepare,finish}.log` and native-payload log; +- `cpython-functional-toad-01/{admitted.json,frontend/end-return.png}`, + typed retirement/copy-stop records and `cpython-functional-toad-absence.log`. + +Native case inputs/results and the functional source/prefix are retained under +the allocated Core checkout's `platform-source/` pending review. One-shot build +code was retired after its successful result. Input hashes are observations, +not a new runtime stamp. Integration still needs a qualified interpreter/build +selection: `RuntimeStack.build` currently keys reuse by Core commit and lock +hash only. The runner's repository-identity work must use the existing +`RecordedBuildIdentity`/`RuntimeResolution`/`RuntimePackage` owners; a mutable +prefix/module location is not a repository commit. Installed acceptance and +measured production source-lifetime/performance repair remain open. diff --git a/docs/performance/cpython-debugger-context.patch b/docs/performance/cpython-debugger-context.patch new file mode 100644 index 000000000..30078214e --- /dev/null +++ b/docs/performance/cpython-debugger-context.patch @@ -0,0 +1,79 @@ +--- a/Python/ceval_gil.c ++++ b/Python/ceval_gil.c +@@ -1216,22 +1216,16 @@ + return 0; + } + +-// Note that this function is inline to avoid creating a PLT entry +-// that would be an easy target for a ROP gadget. +-static inline void run_remote_debugger_script(PyObject *path) ++// This private callback is invoked only inside the debugger's Python frame. ++// It remains static: there is no new exported or PLT entry point. ++static PyObject * ++run_remote_debugger_script_body(PyObject *Py_UNUSED(self), PyObject *path) + { +- if (0 != PySys_Audit("cpython.remote_debugger_script", "O", path)) { +- PyErr_FormatUnraisable( +- "Audit hook failed for remote debugger script %U", path); +- return; +- } +- +- // Open the debugger script with the open code hook, and reopen the +- // resulting file object to get a C FILE* object. ++ // Preserve the existing open-code hook, read/close ordering and errors. + PyObject* fileobj = PyFile_OpenCodeObject(path); + if (!fileobj) { + PyErr_FormatUnraisable("Can't open debugger script %U", path); +- return; ++ Py_RETURN_NONE; + } + + PyObject* source = PyObject_CallMethodNoArgs(fileobj, &_Py_ID(read)); +@@ -1253,6 +1247,47 @@ + } + Py_DECREF(source); + } ++ Py_RETURN_NONE; ++} ++ ++// Note that this function is inline to avoid creating a PLT entry ++// that would be an easy target for a ROP gadget. ++static inline void run_remote_debugger_script(PyObject *path) ++{ ++ // Keep the existing debugger refusal before context compilation/execution. ++ if (0 != PySys_Audit("cpython.remote_debugger_script", "O", path)) { ++ PyErr_FormatUnraisable( ++ "Audit hook failed for remote debugger script %U", path); ++ return; ++ } ++ static PyMethodDef body_definition = { ++ "_run_remote_debugger_script_body", ++ run_remote_debugger_script_body, ++ METH_O, ++ NULL ++ }; ++ PyObject *ns = PyDict_New(); ++ if (!ns) { ++ PyErr_FormatUnraisable("Can't prepare debugger namespace for %U", path); ++ return; ++ } ++ // PyRun_String initializes this namespace from the interpreter's builtins. ++ // A real Python call frame then owns audit/open/read/close as well as exec; ++ // it neither borrows the interrupted frame's builtins nor detaches its GC roots. ++ PyObject *entry = PyRun_String("lambda load, path: load(path)", Py_eval_input, ns, ns); ++ if (entry) { ++ PyObject *body = PyCFunction_New(&body_definition, NULL); ++ if (body) { ++ PyObject *result = PyObject_CallFunctionObjArgs(entry, body, path, NULL); ++ Py_XDECREF(result); ++ Py_DECREF(body); ++ } ++ Py_DECREF(entry); ++ } ++ Py_DECREF(ns); ++ if (PyErr_Occurred()) { ++ PyErr_FormatUnraisable("Can't enter debugger execution context for %U", path); ++ } + } + + int _PyRunRemoteDebugger(PyThreadState *tstate) From 47829173c816a7825d9cf9d418dcc0659b54d33b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 01:12:53 -0500 Subject: [PATCH 05/35] Detach pager retirement joins from the cancelled worker context --- docs/performance/physical-run.md | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 5613f9b4b..204036122 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -119,3 +119,40 @@ Current helpers are `tools/physical_ui.py` and `tools/physical_ui_remote.py` on PR #1393. Consumers must start a new controller from one published checkpoint, not combine a live process with a newly imported wire shape. A completed stopped-history scenario is not bound-ACP or performance acceptance. + +## Pager failure retirement + +The retained configured-fork attempt failed when multiprocessing spawn re-entered +an unguarded controller. Its `BrokenProcessPool` also exposed a separate pager +cleanup defect; neither is an image verdict and the failed root is not replayed. + +`WorkingTranscript.execute` reports a read failure inside the pager Worker. +`TranscriptSourcePreparation.source_failed` immediately revokes its source and +starts `retire_source`. Textual's `AwaitComplete` eagerly schedules the join in a +new task, copying the caller's `active_worker`. `Worker.wait` then rejects that +inherited identity as a self-wait before awaiting anything. The traceback is not +evidence of a synchronous UI stall. + +The existing shared preparation base now creates that independent `AwaitComplete` +in an empty native `Context`. Its join needs only the cancelled Worker handles; +it must not borrow the caller's worker or message pump. Revocation, generation, +subscription closure, cancellation, awaited outcomes and error propagation stay +with their existing owners. Textual's genuine self-wait guard is unchanged. +All six pager descendants inherit the correction, including saved, wire and +projected history; there is no per-view exception handler or second cleanup path. + +Before editing, the retained NRA parser/syntax/canonical-class query covered +1,533 modules: 9,005 original class declarations, 8,966 canonical projections, +39 unprojected OPEN rows and 190 selected lifecycle sites. The manifest retains +source hashes. This is source evidence, not domain completeness or live binding +proof; external/unresolved bases, dynamic mutation and non-Python behavior stay +OPEN. Actual imported MROs confirm the shared base on the relevant saved-history +classes. The source query uses NRA's existing owners, not another parser. + +The existing lifecycle controls now observe original coverage publications and +saved error artifacts instead of replacing our own methods. They cover caller- +initiated retirement, pager-initiated retirement, resume, real self-wait rejection, +and a failed read that retires once without an unhandled cleanup exception. +Both changed Python files compile and the source imports from this checkout. +The controls have **not yet run**; the combined host-lease run and affected +physical journey remain open. This checkpoint is not a performance acceptance. From 134c56b4652257439899d91a0fddec42f8a2a484 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 01:41:54 -0500 Subject: [PATCH 06/35] Record the qualified pager retirement app controls --- docs/performance/physical-run.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 204036122..ee0dbeb7a 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -154,5 +154,17 @@ saved error artifacts instead of replacing our own methods. They cover caller- initiated retirement, pager-initiated retirement, resume, real self-wait rejection, and a failed read that retires once without an unhandled cleanup exception. Both changed Python files compile and the source imports from this checkout. -The controls have **not yet run**; the combined host-lease run and affected -physical journey remain open. This checkpoint is not a performance acceptance. +The exact source head `59d8218a4b70896aa0f8226367202cff035dd411` +passed both app controls: **2 passed in 31.58s, exit 0**, under Core's shared +`TestLease`. The controller asserted the actual Core and Toad source origins +before invoking `tests/textual/test_transcript_lifecycle.py`; the retained log +prints both paths, the interpreter and the acquired lease. Its first attempt +failed before pytest because the controller shadowed an imported module with a +`Path`; that attempt is not a test run. The passing run paired Core's earlier +`a5b` lifecycle source, not its subsequent parallel-reader correction. + +Receipts: `pi-commander/heartbeat-publication/pager-lifetime-controls-2.log` +and `.rc` under `/home/ts/.cache/agent-scratch/`. These are app-level lifecycle +controls, not installed or physical acceptance. The affected physical journey, +full isolation counterexamples and production performance measurement remain +open; no failed configured-fork root was replayed. From 9d8eccda23e17f9ed33527507d2126bed343e65e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 02:20:53 -0500 Subject: [PATCH 07/35] Compare recorded runtime inputs before physical UI input --- docs/performance/physical-run.md | 30 ++++++++++++++++++++++++++++++ tools/physical_ui.py | 5 ++++- tools/physical_ui_remote.py | 11 ++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index ee0dbeb7a..3d04cc3b7 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -71,6 +71,36 @@ acceptance and actual borrower checks. adds only the existing `DeclaredWireType` boundary capability and retains the actual Region object, without copying its coordinate declarations. +## Runtime input identity (Core #3825 dependency) + +The current source requires a `RuntimeResolution` with a recorded interpreter +observation, not just a prefix and three module locations. `RunTarget` retains +that intended build; `ObserveOperation` obtains the actual process's resolution +through the same target-owned probe before `RunTarget.require` permits input. +An old build record without interpreter evidence is explicitly unrecorded and +refuses admission. It is not retrofitted by reading its intended executable. + +`WheelRuntime` retains copies of its exact inputs under the new prefix's +`inputs/` directory; borrowed originals remain read-only. The package probe +compares every root-wheel member except the installer's rewritten `RECORD` +against installed bytes, then records the verified archive SHA256. Root-layout +wheels are the supported domain; `.data` relocation is refused, not silently +skipped. Remote archive hashes remain metadata, never permission to fetch them. +The deployed donor package probe keeps its existing core-only wire; a target +subclass adds wheel verification without changing old workers' request shape. + +The interpreter observation keeps the invoked environment separate from the +resolved executable and the native image actually named by the loader for +`Py_GetVersion`. Stable file reads use `FileRevision`; their SHA256 values are +native **disk-artifact** evidence, not complete loaded-code or stdlib-content +equivalence. Repository ancestry remains `RecordedBuildIdentity`'s separate +contract: a verified archive still does not acquire a Git revision. + +This implementation is a published source checkpoint, not candidate/carry or +physical qualification. The first prepared candidate remains immutable and +unqualified under the new interpreter requirement. No frontend/native/model +launch follows from the earlier single artifact-preparation approval. + ## Evidence limits This scenario exercises **bounded stopped history**, not the retired shell diff --git a/tools/physical_ui.py b/tools/physical_ui.py index d64b4ed1f..225937989 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -28,6 +28,7 @@ from agent_comms.child_process import DetachedChildStdio, ProcessIdentity, SpawnedProcess from agent_comms.comms import Comms from agent_comms.installer.practice import SuppliedRuntimeHome +from agent_comms.installer.runtime_resolution import RuntimeResolution from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime, ThreadLaunch from agent_comms.nominal.field_codec import FieldCodec @@ -254,6 +255,8 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: from toad.preferences import INTERFACE, SETTINGS, SavedSettings, ToadSettings root = root.resolve(strict=True) + build = RuntimeResolution.at(runtime.prefix) + build.interpreter.require_recorded() home = root.parent / "frontend-home" # Validate the intended paths before touching the home. RunTarget also # rejects the real user's home/workspace and an unrelated bus. @@ -267,7 +270,7 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: raise ValueError("Refusing frontend setup beside a live user root") home.mkdir(mode=0o700) SuppliedRuntimeHome(home, runtime).prepare() - target = RunTarget(root, home.resolve(), runtime.prefix) + target = RunTarget(root, home.resolve(), runtime.prefix, build) for directory in (target.config, target.state, target.data): directory.mkdir(mode=0o700) settings = SavedSettings.path() diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index 0ed1627e4..af186396e 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -18,6 +18,8 @@ from agent_comms.child_process import ChildOutcome, ProcessIdentity from agent_comms.launch_runtime import LaunchRuntime +from agent_comms.installer.probes.resolution import RuntimeResolutionProbe +from agent_comms.installer.runtime_resolution import RuntimeResolution from agent_comms.nominal.declared_family import DeclaredFamily from agent_comms.nominal.field_codec import DeclaredWireType, FieldCodec from textual.geometry import Region @@ -36,8 +38,10 @@ class RunTarget: root: Path home: Path runtime: Path + build: RuntimeResolution def __post_init__(self): + self.build.interpreter.require_recorded() for path in (self.root, self.home, self.runtime): if path != path.resolve(strict=True): raise ValueError(f"A physical run needs canonical existing paths: {path}") @@ -71,6 +75,7 @@ def data(self) -> Path: return self.home / "data" def require(self, observed): + self.build.require_runtime(observed.build) if observed.home != self.home or observed.runtime != self.runtime: raise ValueError("Post-attachment HOME/runtime differs from the run target") if (observed.config, observed.state, observed.data) != (self.config, self.state, self.data): @@ -114,6 +119,7 @@ class RunObservation: started_ns: int finished_ns: int runtime: Path + build: RuntimeResolution home: Path config: Path state: Path @@ -337,8 +343,11 @@ def coordinated(self, value): # from sidebar separators that may also occur inside a user's message. regions = tuple(CapturedRegion(region) for window in screen.query(HistoryWindow) for region in (window.published_content_region,) if region is not None) + build = RuntimeResolutionProbe( + self.target.build.core_commit, self.target.build.wheel_inputs() + ).run() observed = RunObservation( - self.identity, started, time.monotonic_ns(), Path(sys.prefix).resolve(), + self.identity, started, time.monotonic_ns(), Path(sys.prefix).resolve(), build, Path.home().resolve(), xdg_config_home().resolve(), xdg_state_home().resolve(), xdg_data_home().resolve(), app.tab_order._db.path.resolve(), app.colours.wal_directory.resolve(), root, From f436d766437c6d2bbf9c41712324b33117d4004a Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 03:25:18 -0500 Subject: [PATCH 08/35] Verify physical runtime builds at the frontend resource lifetime --- docs/performance/physical-run.md | 44 +++++++++++- tests/model/test_physical_run.py | 118 +++++++++++++++++++++++++++++++ tools/physical_ui.py | 26 ++++--- tools/physical_ui_remote.py | 81 ++++++++++++++++----- 4 files changed, 242 insertions(+), 27 deletions(-) create mode 100644 tests/model/test_physical_run.py diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 3d04cc3b7..2fe37a344 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -75,11 +75,45 @@ acceptance and actual borrower checks. The current source requires a `RuntimeResolution` with a recorded interpreter observation, not just a prefix and three module locations. `RunTarget` retains -that intended build; `ObserveOperation` obtains the actual process's resolution -through the same target-owned probe before `RunTarget.require` permits input. +that intended build. `AdmitOperation(ObserveOperation)` obtains the actual +process's resolution once at frontend admission through the target-owned probe. +Its `AdmittedResult` belongs to that exact frontend birth; a plain +`ObservedResult` cannot substitute for it. `RunObservation` describes current UI +resources, not a freshly hashed build. An old build record without interpreter evidence is explicitly unrecorded and refuses admission. It is not retrofitted by reading its intended executable. +`TerminalApp` owns the admission result with its frontend resource. Every later +observation still checks the actual process, runtime, module origins, HOME/XDG, +database, palette, selected/opened source and all saved/ACP roots before keys. +The declared `FrontendIdentity` groups process birth and primary module-file +paths; equality is derived from that record rather than restating its fields in +two comparison tuples. It must agree with the admission. +Only full wheel/native hashing leaves that repeated callback; there is no +permission flag or product-widget cache replacing those current checks. + +A normal exit performs full build verification again and requires equality with +the admitted snapshot before quitting. `frontend/build-admitted.json` and +`frontend/build-final.json` retain the two build observations; ordinary operation +results and `admitted.json` remain current UI/source observations. An exceptional +or uncertain operation still retires without another attachment or input. + +The selected candidate must remain immutable during the run. Entry/final disk +checks do not prove that arbitrary intermediate disk mutations were impossible, +and a final refusal cannot undo earlier input. This does not claim continuous +content hashing or complete loaded-code equivalence. Process/root/module-origin +and saved/ACP-root invalidation remain current before every key. + +Current observations also carry the actual typed `StorageRule` from the already +observing service's `Registration`. The rule must equal its own `within(root)` +projection; observation neither constructs a Comms nor writes a policy. This +checks declared allocation/default scope, not private HOME alone. Canonical +filesystem containment, registered allocations, persisted reopen and future-child +qualification remain separate prerequisites owned by the isolation work. In +particular, a `checkouts` symlink can defeat equality-only projection in the +current Core owner; that counterexample is handed to the isolation owner, not +silently counted as qualified. + `WheelRuntime` retains copies of its exact inputs under the new prefix's `inputs/` directory; borrowed originals remain read-only. The package probe compares every root-wheel member except the installer's rewritten `RECORD` @@ -114,6 +148,12 @@ visible pixels. Neither an import nor one screenshot proves the whole journey. A run with no ACP bindings does not demonstrate the bound/pending/foreign ACP cases. +The new admission/resource-lifetime split has written model controls for build +admission versus ordinary observation, changed build/process/resources/module +paths, new saved/ACP roots, actual storage policy and source readiness after admission. These controls +are not yet executed. The source/codec checks and eventual candidate preparation +must not be reported as physical or performance acceptance. + Source-observer work is part of the measurement overhead. For example, one prototype observation took about 97 ms inside the callback before considering its compilation/dispatch cost. Do not report that overhead as production frame diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py new file mode 100644 index 000000000..186ac34cc --- /dev/null +++ b/tests/model/test_physical_run.py @@ -0,0 +1,118 @@ +"""Admission and current-resource contracts; no frontend, provider or private bus.""" +from dataclasses import replace +from pathlib import Path + +import pytest + +from agent_comms.child_process import ProcessIdentity +from agent_comms.installer.probes.interpreter import InterpreterBuildProbe +from agent_comms.installer.runtime_resolution import RuntimeResolution +from agent_comms.storage_rules import StorageRule +from tools.physical_ui_remote import ( + AdmittedResult, FrontendIdentity, ObservedResult, PendingRun, RunObservation, RunTarget, +) + + +@pytest.fixture +def physical_target(tmp_path: Path) -> RunTarget: + """Synthetic resource paths and metadata, not an application admission proof.""" + root, home, runtime = (tmp_path / name for name in ("bus", "frontend-home", "runtime")) + for path in (root, home, runtime): + path.mkdir() + build = RuntimeResolution( + core_commit="a" * 40, packages={}, interpreter=InterpreterBuildProbe().run(), + ) + return RunTarget(root, home, runtime, build) + + +@pytest.fixture +def physical_observation(physical_target: RunTarget) -> RunObservation: + target = physical_target + return RunObservation( + frontend=FrontendIdentity( + ProcessIdentity.current(), target.runtime / "toad/__init__.py", + target.runtime / "agent_comms/__init__.py", target.runtime / "textual/__init__.py", + ), + started_ns=1, finished_ns=2, runtime=target.runtime, home=target.home, + config=target.config, state=target.state, data=target.data, + database=target.state / "toad.db", palette_directory=target.palette, + root=target.root, opened=True, storage_rule=StorageRule().within(target.root), + saved_roots=(target.root,), + acp_roots=(target.root,), history_regions=(), modules_checked=3, + ) + + +@pytest.fixture +def physical_admission( + physical_target: RunTarget, physical_observation: RunObservation, +) -> AdmittedResult: + return AdmittedResult(physical_observation, physical_target.build).admitted(physical_target) + + +def test_current_observation_cannot_substitute_for_build_admission( + physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + with pytest.raises(ValueError, match="did not verify the runtime build"): + ObservedResult(physical_observation).admitted(physical_target) + assert physical_admission.observe(physical_target, physical_observation) == physical_observation + + +def test_admission_refuses_different_build_and_frontend_birth( + physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + different = physical_target.build.model_copy(update={"core_commit": "b" * 40}) + with pytest.raises(ValueError): + AdmittedResult(physical_observation, different).admitted(physical_target) + original = physical_observation.frontend + replacement = replace(original, process=replace(original.process, pid=original.process.pid + 1)) + with pytest.raises(ValueError, match="frontend process or module-file locations changed"): + physical_admission.observe(physical_target, replace(physical_observation, frontend=replacement)) + + +@pytest.mark.parametrize("field", ( + "runtime", "home", "config", "state", "data", "database", "palette_directory", "root", +)) +def test_admission_does_not_cache_permission_for_changed_resources( + field: str, tmp_path: Path, physical_target: RunTarget, + physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + current = replace(physical_observation, **{field: tmp_path / "foreign"}) + with pytest.raises(ValueError): + physical_admission.observe(physical_target, current) + + +@pytest.mark.parametrize("field", ("toad_module", "core_module", "textual_module")) +def test_changed_primary_module_location_invalidates_admission( + field: str, tmp_path: Path, physical_target: RunTarget, + physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + changed = replace(physical_observation.frontend, **{field: tmp_path / "foreign"}) + with pytest.raises(ValueError, match="frontend process or module-file locations changed"): + physical_admission.observe(physical_target, replace(physical_observation, frontend=changed)) + + +@pytest.mark.parametrize("field", ("saved_roots", "acp_roots")) +def test_new_saved_and_acp_roots_are_checked_after_admission( + field: str, tmp_path: Path, physical_target: RunTarget, + physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + current = replace(physical_observation, **{field: (physical_target.root, tmp_path / "foreign")}) + with pytest.raises(ValueError, match="Foreign saved/ACP source"): + physical_admission.observe(physical_target, current) + + +def test_actual_source_policy_is_checked_after_admission( + tmp_path: Path, physical_target: RunTarget, + physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + foreign = physical_observation.storage_rule.within(tmp_path / "foreign") + current = replace(physical_observation, storage_rule=foreign) + with pytest.raises(ValueError, match="actual source storage policy"): + physical_admission.observe(physical_target, current) + + +def test_source_readiness_is_current_not_remembered_admission( + physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, +) -> None: + with pytest.raises(PendingRun): + physical_admission.observe(physical_target, replace(physical_observation, opened=False)) diff --git a/tools/physical_ui.py b/tools/physical_ui.py index 225937989..c4fe7bf3e 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -42,7 +42,7 @@ from ui_theme.wal import WalFile from physical_ui_remote import ( - CapturedRegion, ExecOperation, ObserveOperation, PendingRun, PhysicalOperation, + AdmitOperation, CapturedRegion, ExecOperation, ObserveOperation, PendingRun, PhysicalOperation, ProcessRetirement, RunObservation, RunTarget, StopRootOperation, ) @@ -389,7 +389,8 @@ def __enter__(self): self.read_grid() while True: try: - self.observe() + self.admission = self._observe(AdmitOperation).admitted(self.target) + PhysicalOperation.publish(self.output / "build-admitted.json", self.admission) break except PendingRun: self._require_starting(deadline) @@ -412,18 +413,16 @@ def _await_result(self, directory: Path, deadline: float) -> None: self._require_starting(deadline) time.sleep(.02) - def observe(self) -> RunObservation: + def _observe(self, operation: type[ObserveOperation]): self.terminal.platform.require(self.frontend) - operation = ObserveOperation(self.target, self.frontend, Path(__file__).resolve().parent) - script = self._packet(operation) + script = self._packet(operation(self.target, self.frontend, Path(__file__).resolve().parent)) # This controller is an ancestor of its st/frontend. No arbitrary PID, # helper privilege escalation, installed-file edit or source retry. sys.remote_exec(self.frontend.pid, str(script)) self._await_result(script.parent, time.monotonic() + PhysicalOperation.timeout) - observed = self.result(script.parent).observation() - if observed.process != self.frontend: + result = self.result(script.parent) + if result.observation().frontend.process != self.frontend: raise ValueError("Observation belongs to another frontend incarnation") - self.target.require(observed) self.terminal.platform.require(self.terminal.identity) self.server_identity() owner = subprocess.check_output( @@ -433,7 +432,11 @@ def observe(self) -> RunObservation: if int(owner.partition("=")[2].strip()) != self.terminal.identity.pid: raise ValueError("Window does not belong to this run's st frontend") self.read_grid() - return observed + return result + + def observe(self) -> RunObservation: + current = self._observe(ObserveOperation).observation() + return self.admission.observe(self.target, current) def input(self, *commands: str) -> RunObservation: observed = self.observe() @@ -595,6 +598,11 @@ def __exit__(self, exception_type, exception, traceback): # An exceptional/uncertain operation is not permission for another # attachment or input. Retire only the already owned child instead. if exception_type is None and self.terminal.process.poll() is None: + final = self._observe(AdmitOperation).admitted(self.target) + self.admission.observe(self.target, final.observation()) + if final.build != self.admission.build: + raise ValueError("The verified runtime build changed during the physical run") + PhysicalOperation.publish(self.output / "build-final.json", final) self.keys("ctrl+q") self.terminal.process.wait(timeout=5) finally: diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index af186396e..b8166e0b0 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -20,6 +20,7 @@ from agent_comms.launch_runtime import LaunchRuntime from agent_comms.installer.probes.resolution import RuntimeResolutionProbe from agent_comms.installer.runtime_resolution import RuntimeResolution +from agent_comms.storage_rules import StorageRule from agent_comms.nominal.declared_family import DeclaredFamily from agent_comms.nominal.field_codec import DeclaredWireType, FieldCodec from textual.geometry import Region @@ -74,8 +75,8 @@ def state(self) -> Path: def data(self) -> Path: return self.home / "data" - def require(self, observed): - self.build.require_runtime(observed.build) + def require_resources(self, observed): + """Current UI/source resources; build evidence belongs to AdmittedResult.""" if observed.home != self.home or observed.runtime != self.runtime: raise ValueError("Post-attachment HOME/runtime differs from the run target") if (observed.config, observed.state, observed.data) != (self.config, self.state, self.data): @@ -88,6 +89,8 @@ def require(self, observed): raise ValueError("Core selected a foreign source") if not observed.opened: raise PendingRun("Core has not opened the intended private source") + if observed.storage_rule != observed.storage_rule.within(self.root): + raise ValueError("The actual source storage policy escapes the private root") for root in (*observed.saved_roots, *observed.acp_roots): if root != self.root: raise ValueError(f"Foreign saved/ACP source refuses physical input: {root}") @@ -114,12 +117,21 @@ def from_wire(cls, data): @dataclass(frozen=True) -class RunObservation: +class FrontendIdentity: + """One process birth and its primary module-file locations.""" + process: ProcessIdentity + toad_module: Path + core_module: Path + textual_module: Path + + +@dataclass(frozen=True) +class RunObservation: + frontend: FrontendIdentity started_ns: int finished_ns: int runtime: Path - build: RuntimeResolution home: Path config: Path state: Path @@ -128,13 +140,11 @@ class RunObservation: palette_directory: Path root: Path opened: bool + storage_rule: StorageRule saved_roots: tuple[Path, ...] acp_roots: tuple[Path, ...] history_regions: tuple[CapturedRegion, ...] modules_checked: int - toad_module: Path - core_module: Path - textual_module: Path @dataclass(frozen=True) @@ -149,6 +159,9 @@ class PhysicalResult(DeclaredFamily, affix="Result"): def require(self): return self + def admitted(self, target: RunTarget) -> AdmittedResult: + raise ValueError("This physical operation did not verify the runtime build") + def observation(self) -> RunObservation: raise ValueError("This physical operation did not observe an application") @@ -164,6 +177,24 @@ def observation(self) -> RunObservation: return self.value +@dataclass(frozen=True) +class AdmittedResult(ObservedResult): + """Build evidence for one frontend birth, not a fresh hash on every UI read.""" + + build: RuntimeResolution + + def admitted(self, target: RunTarget) -> AdmittedResult: + target.build.require_runtime(self.build) + target.require_resources(self.value) + return self + + def observe(self, target: RunTarget, current: RunObservation) -> RunObservation: + if current.frontend != self.value.frontend: + raise ValueError("The admitted frontend process or module-file locations changed") + # Dynamic persistence and every saved/ACP root still precede each input. + return target.require_resources(current) + + @dataclass(frozen=True) class ExecResult(PhysicalResult): identity: ProcessIdentity @@ -273,6 +304,9 @@ class ObserveOperation(PhysicalOperation): identity: ProcessIdentity tools: Path + def observed_result(self, observed: RunObservation) -> PhysicalResult: + return ObservedResult(observed) + def execute(self, directory: Path) -> None: started = time.monotonic_ns() if ProcessIdentity.current() != self.identity: @@ -332,6 +366,11 @@ def coordinated(self, value): checked = require_modules(self.target.runtime) access = app.coordination_access root = access.selection.root + observation = access.observation + opened = observation.observes(root) + if not opened: + raise PendingRun("Core has not opened the intended private source") + storage_rule = observation.service.registry.storage_rule sessions = app.session_navigation saved = tuple(Path(source).resolve() for entry in sessions.members for source, _identity in (*entry.original_threads(sessions), *entry.original_channels())) @@ -343,20 +382,30 @@ def coordinated(self, value): # from sidebar separators that may also occur inside a user's message. regions = tuple(CapturedRegion(region) for window in screen.query(HistoryWindow) for region in (window.published_content_region,) if region is not None) - build = RuntimeResolutionProbe( - self.target.build.core_commit, self.target.build.wheel_inputs() - ).run() + frontend = FrontendIdentity( + self.identity, Path(toad.__file__).resolve(), Path(agent_comms.__file__).resolve(), + Path(textual.__file__).resolve(), + ) observed = RunObservation( - self.identity, started, time.monotonic_ns(), Path(sys.prefix).resolve(), build, + frontend, started, time.monotonic_ns(), Path(sys.prefix).resolve(), Path.home().resolve(), xdg_config_home().resolve(), xdg_state_home().resolve(), xdg_data_home().resolve(), app.tab_order._db.path.resolve(), app.colours.wal_directory.resolve(), root, - access.observation.observes(root), saved, acp, regions, checked, - Path(toad.__file__).resolve(), Path(agent_comms.__file__).resolve(), - Path(textual.__file__).resolve(), + opened, storage_rule, saved, acp, regions, checked, ) - self.target.require(observed) - self.publish(directory / "result.json", ObservedResult(observed)) + self.target.require_resources(observed) + self.publish(directory / "result.json", self.observed_result(observed)) + + +@dataclass(frozen=True) +class AdmitOperation(ObserveOperation): + """Full disk-artifact verification at entry and successful retirement only.""" + + def observed_result(self, observed: RunObservation) -> PhysicalResult: + build = RuntimeResolutionProbe( + self.target.build.core_commit, self.target.build.wheel_inputs() + ).run() + return AdmittedResult(observed, build).admitted(self.target) @dataclass(frozen=True) From 72dedf4a0b0f499a05fcef9f7ec0ab20273acd56 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 04:14:30 -0500 Subject: [PATCH 09/35] Ask the storage rule to verify physical-run isolation --- docs/performance/physical-run.md | 26 ++++++++++++++++---------- tests/model/test_physical_run.py | 2 +- tools/physical_ui_remote.py | 3 +-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 2fe37a344..24f3fce0b 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -105,14 +105,18 @@ content hashing or complete loaded-code equivalence. Process/root/module-origin and saved/ACP-root invalidation remain current before every key. Current observations also carry the actual typed `StorageRule` from the already -observing service's `Registration`. The rule must equal its own `within(root)` -projection; observation neither constructs a Comms nor writes a policy. This -checks declared allocation/default scope, not private HOME alone. Canonical -filesystem containment, registered allocations, persisted reopen and future-child -qualification remain separate prerequisites owned by the isolation work. In -particular, a `checkouts` symlink can defeat equality-only projection in the -current Core owner; that counterexample is handed to the isolation owner, not -silently counted as qualified. +observing service's `Registration`. `RunTarget` calls that rule's pure +`require_isolated(root)` observation contract; it no longer repeats the owner's +projection/equality decision. Observation neither constructs a Comms nor writes +a policy. The paired Core must include this API: its `ScopedDirectory` field +declarations own canonical directory checks and future default allocations, +including outward-symlink refusal. There is no fallback to an older API. + +Filesystem behavior, registered allocations, persisted reopen and future-child +qualification remain separate prerequisites owned by the isolation work. The +owner's eleven new filesystem cases are written, not run; a source implementation +is not isolation acceptance. Candidate02's Core7e4/Toad620955 artifact pairing +predates this API and cannot qualify the current consumer; preserve it unchanged. `WheelRuntime` retains copies of its exact inputs under the new prefix's `inputs/` directory; borrowed originals remain read-only. The package probe @@ -120,8 +124,10 @@ compares every root-wheel member except the installer's rewritten `RECORD` against installed bytes, then records the verified archive SHA256. Root-layout wheels are the supported domain; `.data` relocation is refused, not silently skipped. Remote archive hashes remain metadata, never permission to fetch them. -The deployed donor package probe keeps its existing core-only wire; a target -subclass adds wheel verification without changing old workers' request shape. +The deployed donor package probe keeps its existing core-only wire. Updated +built workers derive retained inputs from their own `RuntimeResolution`; the +target subclass supplies explicit inputs before its first resolution exists. +Both use the same verification loop, without changing old workers' request shape. The interpreter observation keeps the invoked environment separate from the resolved executable and the native image actually named by the loader for diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py index 186ac34cc..0aa458580 100644 --- a/tests/model/test_physical_run.py +++ b/tests/model/test_physical_run.py @@ -107,7 +107,7 @@ def test_actual_source_policy_is_checked_after_admission( ) -> None: foreign = physical_observation.storage_rule.within(tmp_path / "foreign") current = replace(physical_observation, storage_rule=foreign) - with pytest.raises(ValueError, match="actual source storage policy"): + with pytest.raises(ValueError): physical_admission.observe(physical_target, current) diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index b8166e0b0..6802f31cb 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -89,8 +89,7 @@ def require_resources(self, observed): raise ValueError("Core selected a foreign source") if not observed.opened: raise PendingRun("Core has not opened the intended private source") - if observed.storage_rule != observed.storage_rule.within(self.root): - raise ValueError("The actual source storage policy escapes the private root") + observed.storage_rule.require_isolated(self.root) for root in (*observed.saved_roots, *observed.acp_roots): if root != self.root: raise ValueError(f"Foreign saved/ACP source refuses physical input: {root}") From e9b9ae7da4cadf941e96e46ec6d273bdcfa0517d Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 14:08:01 -0500 Subject: [PATCH 10/35] Append followed saved history without replacing its painted prefix --- src/toad/transcript_preparation.py | 3 ++ src/toad/widgets/committed_presentation.py | 40 +++++++++---------- src/toad/widgets/conversation.py | 6 ++- .../test_transcript_presentation_fencing.py | 38 ++++++++++++++++++ tools/physical_ui.py | 2 +- 5 files changed, 66 insertions(+), 23 deletions(-) diff --git a/src/toad/transcript_preparation.py b/src/toad/transcript_preparation.py index 3415cede5..9ddd1ca8f 100644 --- a/src/toad/transcript_preparation.py +++ b/src/toad/transcript_preparation.py @@ -21,6 +21,7 @@ from toad.render_backend import ReusableRenderTask from toad.transcript_state import LatestViewportRequest +from toad.widgets.committed_presentation import PreparedAppendCommit, PreparedPageCommit from toad.widgets.history_anchor import LatestPosition from toad.widgets.message_notifications import NothingToRead, PaintedRead from toad.widgets.presentation_window import OLDER_END, PageEnd, PageRequest @@ -287,6 +288,7 @@ class PreparedPageSource: runtime: PreparationRuntime scope: PreparationScope through: TranscriptCursor + commit_preparation: ClassVar[type[PreparedPageCommit]] = PreparedPageCommit def __init__(self) -> None: # This source holds the resources after their widgets detach. These are the @@ -498,6 +500,7 @@ class SavedHistory(ReplaceablePageSource): # The work that reads and prepares one page of this kind of source. work: ClassVar[type[TranscriptPageWork]] = TranscriptPageWork + commit_preparation = PreparedAppendCommit def __init__( self, loader: Callable[..., Awaitable[TranscriptPage]], through: TranscriptCursor, diff --git a/src/toad/widgets/committed_presentation.py b/src/toad/widgets/committed_presentation.py index 82267b020..1cab9331c 100644 --- a/src/toad/widgets/committed_presentation.py +++ b/src/toad/widgets/committed_presentation.py @@ -281,21 +281,34 @@ async def draw(self, cursor, is_current: Callable[[], bool], live_blocks) -> boo if not history.source_ready or not is_current(): return False operation = history.reserve_source_work(partial(CommitTranscript, live_blocks=live_blocks)) - return await operation.execute(history, partial( - history.show_newest, + return await operation.execute(history, partial(self.publish, cursor, is_current)) + + async def publish(self, cursor: TranscriptCursor, is_current: Callable[[], bool]) -> bool: + history = self.history + return await history.show_newest( PreparedReplacementViewportRequest(LatestPosition.hold(history.window), self.page), publication_current=is_current, - )) + ) + + +class PreparedAppendCommit(PreparedPageCommit): + """The saved source extends its original pages, retaining their rendered prefix.""" + + async def publish(self, cursor: TranscriptCursor, is_current: Callable[[], bool]) -> bool: + return await self.history.extend(cursor, publication_current=is_current) class CommitPlan(ABC): """The captured viewport intent owns source acquisition and publication.""" - @abstractmethod async def read_snapshot[Snapshot]( self, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, ) -> Snapshot: - """Acquire the saved destination this plan will publish.""" + """Read after the retained prefix; a different source needs a complete read.""" + try: + return await read(position=history.page_source.through.later_page()) + except TranscriptChanged: + return await read() @abstractmethod def current(self, window: HistoryWindow) -> bool: @@ -339,18 +352,12 @@ class FollowTailCommit(CommitPlan): def __init__(self, window: HistoryWindow) -> None: self.revision = window.scroll_revision - async def read_snapshot[Snapshot]( - self, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, - ) -> Snapshot: - # No after cursor: Core reads newest-first at its captured saved cut. - return await read() - def current(self, window: HistoryWindow) -> bool: return window.follows_tail and window.scroll_revision == self.revision async def extend(self, view: Conversation, history: TranscriptHistory, page: TranscriptPage, sequences: frozenset[int]) -> PreparedPageCommit: - return await PreparedPageCommit.capture(view, history, page, sequences) + return await history.page_source.commit_preparation.capture(view, history, page, sequences) def finish(self, view, cursor) -> None: view.transcript.binding.painted(view.transcript, cursor) @@ -363,15 +370,6 @@ def restore_reader(self, window) -> None: window.anchor() class RetainViewportCommit(CommitPlan): - async def read_snapshot[Snapshot]( - self, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, - ) -> Snapshot: - try: - return await read(position=history.page_source.through.later_page()) - except TranscriptChanged: - # A replaced source cannot extend the old reader's source range. - return await read() - def current(self, window: HistoryWindow) -> bool: # Extending the source leaves the loaded rows and reader position intact. # Retirement checks the current viewport again when applying the change. diff --git a/src/toad/widgets/conversation.py b/src/toad/widgets/conversation.py index 679c50d5e..0fdf3df7a 100644 --- a/src/toad/widgets/conversation.py +++ b/src/toad/widgets/conversation.py @@ -1780,7 +1780,11 @@ async def check_prune(self) -> None: # Height pressure is not source evidence. Dropping the source # pager or an uncovered wire claim lets a later observation # recreate the same record as a new tail arrival. - if self.contents.virtual_size.height > high_mark: + # Saved pages already have a bounded preparation owner. Only + # the native tail consumes the live-block pressure allowance. + live_height = (self.contents.virtual_size.height + - self.transcript.binding.history.outer_size.height) + if live_height > high_mark: self.transcript.require_commit() else: await self.prune_window(low_mark, high_mark) diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index cbfca6add..3779b13ce 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -154,6 +154,44 @@ async def test_a_certified_read_that_expired_acquires_no_application(workspace): assert view.transcript.binding.history.page_source.through.session_file == str(journal) +async def test_followed_saved_append_keeps_painted_page_and_fragments(workspace): + import json + from dataclasses import replace + + thread = workspace.thread("append-source") + journal = write_journal( + thread.project.path / "append.jsonl", + assistant_entry("FIRST_RETAINED_ROW", id="first"), + assistant_entry("SECOND_RETAINED_ROW", id="second", parent="first"), + assistant_entry("THIRD_RETAINED_ROW", id="third", parent="second"), + ) + workspace.comms.registry.declare(replace( + thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) + async with workspace.app(resume="append-source") as ui: + await ui.until_shows("THIRD_RETAINED_ROW") + view = ui.conversation + await ui.app.coordination_access.stop_observing() + await view.transcript.suspend() + history = view.transcript.binding.history + assert view.window.follows_tail + pages = tuple(history.pages) + fragments = tuple(history.fragments_in_range) + assert pages and fragments + with journal.open("a") as output: + output.write(json.dumps(assistant_entry( + "APPENDED_SAVED_ROW", id="appended", parent="third")) + "\n") + view.transcript.require_commit() + await ui.until_shows("APPENDED_SAVED_ROW") + assert view.transcript.binding.history is history + assert all(page in history.pages for page in pages) + current = tuple(history.fragments_in_range) + assert all(any(fragment is kept for kept in current) for fragment in fragments) + painted = ui.painted(view.window) + assert all(marker in painted for marker in ( + "FIRST_RETAINED_ROW", "SECOND_RETAINED_ROW", "THIRD_RETAINED_ROW")) + assert ui.failure() is None + + async def test_closing_the_transcript_mid_application_paints_nothing(workspace): async with workspace.app() as ui: view = ui.conversation diff --git a/tools/physical_ui.py b/tools/physical_ui.py index c4fe7bf3e..f762fd507 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -516,7 +516,7 @@ def wait(self, name, predicate, seconds=60): def show_thread(self, thread): self.wait("ready", lambda screen: thread in screen.text - and "▶ Session details · Ready" in "\n".join(screen.text.splitlines()[-8:])) + and "▶ Ready" in "\n".join(screen.text.splitlines()[-8:])) self.click_cell(self.columns - 1, self.rows // 2) return self.wait("shown", lambda screen: "User messages" in screen.text and all( "Indexing" not in line.split("│", 1)[0] for line in screen.text.splitlines())) From 8b3f7ffbf5759a1388c45cb8f046177984b37d86 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 14:47:09 -0500 Subject: [PATCH 11/35] Require actual append publication and complete closed-prefix acquisition --- src/toad/transcript_source_preparation.py | 21 ++++- src/toad/widgets/committed_presentation.py | 25 +++-- .../test_transcript_presentation_fencing.py | 92 +++++++++++++++++++ 3 files changed, 131 insertions(+), 7 deletions(-) diff --git a/src/toad/transcript_source_preparation.py b/src/toad/transcript_source_preparation.py index 89a2c774e..d3dce7976 100644 --- a/src/toad/transcript_source_preparation.py +++ b/src/toad/transcript_source_preparation.py @@ -310,8 +310,27 @@ def drawn_end_more(self) -> bool: edge = self.pages[-1].prepared return NEWER_END.can_extend(edge) or edge.page.has_newer + async def publish_append(self, through, publication_current: Callable[[], bool]) -> bool: + """A commit may retire live blocks only after publishing its destination. + + extend also serves held readers: its True can mean access advanced + without new paint. A followed commit must instead finish a known gap + through the existing bounded destination owner before reporting success. + """ + if not await self.extend(through, publication_current): + return False + if not publication_current(): + return False + if self.drawn_end_more: + if not await self.jump_latest( + LatestViewportRequest(LatestPosition.hold(self.window)), publication_current, + ): + return False + return (publication_current() and not self.drawn_end_more + and self.pages[-1].prepared.page.after.same_position(through)) + async def extend(self, through, publication_current: Callable[[], bool] = lambda: True) -> bool: - """Append to a complete followed tail; a gap uses the bounded latest read.""" + """Extend source access, preserving held readers that have not drawn its end.""" if not publication_current(): return False self.page_source.through = through diff --git a/src/toad/widgets/committed_presentation.py b/src/toad/widgets/committed_presentation.py index 1cab9331c..05aa6a607 100644 --- a/src/toad/widgets/committed_presentation.py +++ b/src/toad/widgets/committed_presentation.py @@ -263,6 +263,13 @@ class PreparedPageCommit(PreparedCommit): history: TranscriptHistory page: PreparedTranscriptPage + @classmethod + async def read_snapshot[Snapshot]( + cls, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, + ) -> Snapshot: + """Replacement needs the complete destination, even for a closed prefix.""" + return await read() + @classmethod async def capture(cls, view: Conversation, history: TranscriptHistory, page: TranscriptPage, sequences: frozenset[int]) -> Self: from toad.transcript_preparation import TranscriptPageRenderTask @@ -294,8 +301,17 @@ async def publish(self, cursor: TranscriptCursor, is_current: Callable[[], bool] class PreparedAppendCommit(PreparedPageCommit): """The saved source extends its original pages, retaining their rendered prefix.""" + @classmethod + async def read_snapshot[Snapshot]( + cls, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, + ) -> Snapshot: + try: + return await read(position=history.page_source.through.later_page()) + except TranscriptChanged: + return await read() + async def publish(self, cursor: TranscriptCursor, is_current: Callable[[], bool]) -> bool: - return await self.history.extend(cursor, publication_current=is_current) + return await self.history.publish_append(cursor, publication_current=is_current) class CommitPlan(ABC): @@ -304,11 +320,8 @@ class CommitPlan(ABC): async def read_snapshot[Snapshot]( self, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, ) -> Snapshot: - """Read after the retained prefix; a different source needs a complete read.""" - try: - return await read(position=history.page_source.through.later_page()) - except TranscriptChanged: - return await read() + """The source's declared commit member owns acquisition and publication.""" + return await history.page_source.commit_preparation.read_snapshot(read, history) @abstractmethod def current(self, window: HistoryWindow) -> bool: diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 3779b13ce..282f43113 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -192,6 +192,98 @@ async def test_followed_saved_append_keeps_painted_page_and_fragments(workspace) assert ui.failure() is None +async def test_followed_commit_publishes_a_known_undrawn_gap_before_settling(workspace): + import json + from dataclasses import replace + + thread = workspace.thread("gap-source") + journal = write_journal(thread.project.path / "gap.jsonl", + assistant_entry("BEFORE_SOURCE_GAP", id="before")) + workspace.comms.registry.declare(replace( + thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) + async with workspace.app(resume="gap-source") as ui: + await ui.until_shows("BEFORE_SOURCE_GAP") + view = ui.conversation + await ui.app.coordination_access.stop_observing() + await view.transcript.suspend() + history = view.transcript.binding.history + with journal.open("a") as output: + output.write(json.dumps(assistant_entry( + "AFTER_SOURCE_GAP", id="after", parent="before")) + "\n") + page = workspace.comms.transcripts.capture_page_read("gap-source").read() + # The real retained-reader operation advances access, not the drawn end. + history.retain_committed(page.after) + assert view.window.follows_tail and history.drawn_end_more + view.transcript.dirty = view.transcript.commit_required = True + await view.transcript.publish(CommitPublication) + # Check the publication contract before a later lookahead can hide it. + assert not history.drawn_end_more + assert history.pages[-1].prepared.page.after.same_position(page.after) + await ui.until_shows("AFTER_SOURCE_GAP") + assert ui.failure() is None + + +async def test_closed_same_source_commit_acquires_the_complete_replacement(workspace): + import json + from dataclasses import replace + + thread = workspace.thread("closed-prefix") + journal = write_journal(thread.project.path / "closed.jsonl", + assistant_entry("CLOSED_PREFIX_MUST_SURVIVE", id="before")) + workspace.comms.registry.declare(replace( + thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) + async with workspace.app(resume="closed-prefix") as ui: + await ui.until_shows("CLOSED_PREFIX_MUST_SURVIVE") + view = ui.conversation + await ui.app.coordination_access.stop_observing() + await view.transcript.suspend() + history = view.transcript.binding.history + history.close() + with journal.open("a") as output: + output.write(json.dumps(assistant_entry( + "CLOSED_PREFIX_SUCCESSOR", id="after", parent="before")) + "\n") + view.transcript.dirty = view.transcript.commit_required = True + await view.transcript.publish(CommitPublication) + await ui.until_shows("CLOSED_PREFIX_SUCCESSOR") + assert "CLOSED_PREFIX_MUST_SURVIVE" in ui.painted(view.window) + assert view.transcript.binding.history is history + assert ui.failure() is None + + +@pytest.mark.parametrize("pressure", (False, True)) +async def test_prune_pressure_counts_live_blocks_not_saved_history(workspace, pressure): + from dataclasses import replace + from toad.preferences import SETTINGS + + thread = workspace.thread("pressure-source") + journal = write_journal(thread.project.path / "pressure.jsonl", + assistant_entry("SAVED_PRESSURE_ROW_ONE\n\nSAVED_PRESSURE_ROW_TWO", id="saved")) + workspace.comms.registry.declare(replace( + thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) + async with workspace.app(resume="pressure-source") as ui: + await ui.until_shows("SAVED_PRESSURE_ROW_TWO") + view = ui.conversation + await ui.app.coordination_access.stop_observing() + await view.post(AgentResponse("\n\n".join( + f"Actual live response paragraph {number}" for number in range(128)))) + await ui.until(lambda: view.contents.virtual_size.height + - view.transcript.binding.history.outer_size.height > 100, + why="the real live response to have its laid-out height") + await view.transcript.suspend() + history = view.transcript.binding.history + live_height = view.contents.virtual_size.height - history.outer_size.height + threshold = live_height + 1 - 2 * int(pressure) + assert threshold >= 100 and history.outer_size.height > 2 + assert view.contents.virtual_size.height > threshold + ui.app.saved_settings.set(SETTINGS.ui.prune_low_mark, threshold) + ui.app.saved_settings.set(SETTINGS.ui.prune_excess, 0) + view.transcript.dirty = view.transcript.commit_required = False + view._require_check_prune = True + await view.check_prune() + assert view.transcript.commit_required is pressure + assert ui.failure() is None + + async def test_closing_the_transcript_mid_application_paints_nothing(workspace): async with workspace.app() as ui: view = ui.conversation From 01755b3fd077adf713d1effda9f0a074a7eb268f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 15:32:52 -0500 Subject: [PATCH 12/35] Use the declared thread project path in append controls --- tests/textual/test_transcript_presentation_fencing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 282f43113..3538f4090 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -160,7 +160,7 @@ async def test_followed_saved_append_keeps_painted_page_and_fragments(workspace) thread = workspace.thread("append-source") journal = write_journal( - thread.project.path / "append.jsonl", + thread.project_path / "append.jsonl", assistant_entry("FIRST_RETAINED_ROW", id="first"), assistant_entry("SECOND_RETAINED_ROW", id="second", parent="first"), assistant_entry("THIRD_RETAINED_ROW", id="third", parent="second"), @@ -197,7 +197,7 @@ async def test_followed_commit_publishes_a_known_undrawn_gap_before_settling(wor from dataclasses import replace thread = workspace.thread("gap-source") - journal = write_journal(thread.project.path / "gap.jsonl", + journal = write_journal(thread.project_path / "gap.jsonl", assistant_entry("BEFORE_SOURCE_GAP", id="before")) workspace.comms.registry.declare(replace( thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) @@ -228,7 +228,7 @@ async def test_closed_same_source_commit_acquires_the_complete_replacement(works from dataclasses import replace thread = workspace.thread("closed-prefix") - journal = write_journal(thread.project.path / "closed.jsonl", + journal = write_journal(thread.project_path / "closed.jsonl", assistant_entry("CLOSED_PREFIX_MUST_SURVIVE", id="before")) workspace.comms.registry.declare(replace( thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) @@ -256,7 +256,7 @@ async def test_prune_pressure_counts_live_blocks_not_saved_history(workspace, pr from toad.preferences import SETTINGS thread = workspace.thread("pressure-source") - journal = write_journal(thread.project.path / "pressure.jsonl", + journal = write_journal(thread.project_path / "pressure.jsonl", assistant_entry("SAVED_PRESSURE_ROW_ONE\n\nSAVED_PRESSURE_ROW_TWO", id="saved")) workspace.comms.registry.declare(replace( thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) From 613289776595af4c07f03dff380b3e85df93d781 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 15:48:02 -0500 Subject: [PATCH 13/35] Establish commit admission before exercising the known source gap --- .../test_transcript_presentation_fencing.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 3538f4090..1cdb1170e 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -207,15 +207,23 @@ async def test_followed_commit_publishes_a_known_undrawn_gap_before_settling(wor await ui.app.coordination_access.stop_observing() await view.transcript.suspend() history = view.transcript.binding.history + await ui.until(lambda: history.source_ready, why="the original source work to finish") with journal.open("a") as output: output.write(json.dumps(assistant_entry( "AFTER_SOURCE_GAP", id="after", parent="before")) + "\n") - page = workspace.comms.transcripts.capture_page_read("gap-source").read() - # The real retained-reader operation advances access, not the drawn end. - history.retain_committed(page.after) - assert view.window.follows_tail and history.drawn_end_more view.transcript.dirty = view.transcript.commit_required = True - await view.transcript.publish(CommitPublication) + entered, release = asyncio.Event(), asyncio.Event() + pending = asyncio.create_task(view.transcript.publish(HeldSourceCase, entered, release)) + try: + # A successful publication return can also mean a guarded no-op. + # Hold the existing real read only AFTER the commit was admitted. + await ui.until(entered.is_set, why="the admitted commit to acquire its real source") + page = workspace.comms.transcripts.capture_page_read("gap-source").read() + history.retain_committed(page.after) + assert view.window.follows_tail and history.drawn_end_more + finally: + release.set() + await pending # Check the publication contract before a later lookahead can hide it. assert not history.drawn_end_more assert history.pages[-1].prepared.page.after.same_position(page.after) From 390785d9ca9cafeebd37282996593b80ed6597fe Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 16:31:56 -0500 Subject: [PATCH 14/35] Own physical observations from frontend startup, delete debugger attachment --- docs/performance/bounded-history-capture.md | 6 +- docs/performance/physical-run.md | 68 +++++++-- tests/model/test_physical_run.py | 16 +- tools/acceptance.py | 4 +- tools/performance/capture_live.py | 160 ++++---------------- tools/physical_ui.py | 84 ++++++---- tools/physical_ui_remote.py | 155 ++++++++++++------- 7 files changed, 255 insertions(+), 238 deletions(-) diff --git a/docs/performance/bounded-history-capture.md b/docs/performance/bounded-history-capture.md index 9112a4112..339eca50b 100644 --- a/docs/performance/bounded-history-capture.md +++ b/docs/performance/bounded-history-capture.md @@ -1,4 +1,8 @@ -# Root-only history/frame observation +# Historical history/frame observation — do not run + +The frozen inputs documented here remain unchanged for retained diagnosis. +Their running-process injection is forbidden by the current host permission; +the old approval is not launch authority. See [the startup replacement](physical-run.md#startup-observation-current-source-not-yet-run). This is an observation tool, not a history performance repair. Authors do not run it. Root reviews the exact head, selects the private-process controls below, and only then attaches to its explicitly selected copied-bus frontend. No synthetic app, UI input, screenshots, provider calls, saved-history decoding or service operations are involved. diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 24f3fce0b..0963df31d 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -37,6 +37,30 @@ retains the copied root, frontend-home, operation receipts, screenshots and recording on exit. The caller still owns retiring those retained files after acceptance and actual borrower checks. +## Startup observation (current source, not yet run) + +`StartOperation` runs the installed distribution's declared Toad entrypoint in +the new frontend interpreter. One private inherited pipe belongs to that exact +frontend/controller lifetime; the descriptor cannot travel into later execs. +The blocking reader receives only paths to fresh data packets under this run's +operation directory. `FieldCodec` admits `ObserveOperation` descendants, never +start/stop operations. Textual's existing `call_from_thread` applies observations +on its own loop. No executable packet, debugger attachment, ptracer grant, +signal handler, periodic wakeup or second controller is involved. + +The copied-history factory now exercises the instrumented installed **Toad CLI**, +not the separate `toad-comms` shell launcher or its executable wrapper. Those +entrypoints need their own evidence; this run cannot claim it. The operation +module stays alive while the distribution entrypoint runs so its declared wire +types do not acquire a different `__main__` namespace. `AcpRoots` is declared +once per module, not rebuilt and registered on every observation. + +The startup/shutdown programs retain the bounded immutable source snapshot. +Ordinary observations write only typed request/result data, not another copy of +the program. Closing the controller pipe and the existing child lifetime retires +this reader. Exceptional exit sends no further observation or physical input. +The new path is source work until actual fresh-prefix/physical execution proves it. + ## What the owner establishes - `CheckRoot` alone selects/imports bounded stopped history; old goals, turns, @@ -52,18 +76,18 @@ acceptance and actual borrower checks. - HOME/XDG inside `st -e env`: st itself resets HOME from passwd before exec. - Xvfb uses displayfd, not a guessed display number. Xlib checks its actual local peer UID/process; the run verifies the allocated server incarnation. -- Child exec grants debugging only to the recorded controller PID and its - descendants. It changes no global ptrace policy and needs no sudo. The - controller owns one bounded operation-source snapshot for its entire Python - lifetime; editing the checkout cannot change an already started run's wire. +- Startup retains the recorded controller/frontend births and its inherited + private connection. It grants no debugging permission and performs no running- + process injection. The controller owns one bounded operation-source snapshot; + editing the checkout cannot change an already started run's wire. - Before physical input: the actual frontend incarnation, runtime and loaded Toad/Core/Textual module locations, HOME/XDG/database, selected/opened source, every saved entry's source and every borrowed ACP process's creator/session agent coordination roots. Pending/foreign sources refuse input. - The read-only observation asks `ToadApp.running_app()` and the actual screen - owner. A window caption is not readiness. `remote_exec` receives an explicit - module filename binding and an adjacent FieldCodec request; a missing or - timed-out result is not a successful observation. + owner on Textual's loop. A window caption is not readiness. The startup reader + accepts an adjacent FieldCodec data request; a missing or timed-out result is + not a successful observation. - st text travels with the actually published history viewport geometry. `ScreenCapture` clips through Textual/Rich's cell geometry, not sidebar separators that disappear when a sidebar closes or occur inside code. @@ -166,13 +190,29 @@ its compilation/dispatch cost. Do not report that overhead as production frame cost or quietly subtract it from a user-facing latency claim. Correlate complete controller/call spans and the recording. -The separate reviewed `capture_history.py` path retains its own explicit -approval, identity, size/time budgets and completion token. This entry point -neither starts that capture nor authorizes heap, GC, repaint, remeasure or -arbitrary trace experiments. The legacy `capture_live.py` diagnostic options do -not become authorized merely because they import the shared display resource. - -## Current qualification +The frozen `capture_history.py`, `history_capture_remote.py` and `frame_meter.py` +inputs remain unchanged and unrun. Their old injection path is forbidden by the +current host permission, notwithstanding historical approval. Replacement of +that capture needs its proper source/execution admission. No heap, GC, repaint, +remeasure or arbitrary trace experiment is authorized. `capture_live.py` now +exposes only read-only py-spy profiling; its injection and mutation options are +deleted, not retained behind a permission flag. + +## Append/publication app controls + +The authorized five-case selection on application source `26a16fe2` passed: +**5 collected, 5 passed, 71.91 s, exit 0**. It covers retained page/fragment paint, +the known-gap destination immediately after a witnessed admitted real read, +complete closed-prefix replacement, and both live-pressure cases. Setup and +teardown completed through the existing `pytest_session`/`TestLease`, using the +actual ceeed interpreter/Core/Textual and this checkout's Toad source. Evidence: +`pi-ui-performance/dm-app-controls-03/{origins.log,pytest.log,pytest.rc}` under +`/home/ts/.cache/agent-scratch/`. The earlier four-pass/one-fail run remains failed; +the new fixture witnesses admission instead of assuming a successful return +means work ran. These are headless app controls before today's-main rebase, +not incoming native-tool, startup-transport, shell-launcher or physical proof. + +## Historical qualification — not the startup replacement Core #3821 is installed in 7d0d; the merger reports 23 selected checks passed. Fresh runs10 and12 used that installed source interpreter with `-I` and no Core diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py index 0aa458580..af47d7d91 100644 --- a/tests/model/test_physical_run.py +++ b/tests/model/test_physical_run.py @@ -8,8 +8,10 @@ from agent_comms.installer.probes.interpreter import InterpreterBuildProbe from agent_comms.installer.runtime_resolution import RuntimeResolution from agent_comms.storage_rules import StorageRule +from agent_comms.nominal.field_codec import FieldCodec from tools.physical_ui_remote import ( - AdmittedResult, FrontendIdentity, ObservedResult, PendingRun, RunObservation, RunTarget, + AdmitOperation, AdmittedResult, FrontendIdentity, ObserveOperation, ObservedResult, + PendingRun, RunObservation, RunTarget, StartOperation, StopRootOperation, ) @@ -49,6 +51,18 @@ def physical_admission( return AdmittedResult(physical_observation, physical_target.build).admitted(physical_target) +def test_startup_connection_decodes_only_observation_operations(physical_target: RunTarget) -> None: + identity = ProcessIdentity.current() + tools = Path(__file__).resolve().parents[2] / "tools" + for operation in (ObserveOperation(physical_target, identity, tools), + AdmitOperation(physical_target, identity, tools)): + assert FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) == operation + start = StartOperation(physical_target, identity, (str(physical_target.runtime / "bin/toad"),), 9) + for operation in (start, StopRootOperation(physical_target)): + with pytest.raises(ValueError): + FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) + + def test_current_observation_cannot_substitute_for_build_admission( physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, ) -> None: diff --git a/tools/acceptance.py b/tools/acceptance.py index 67210103d..9c69628b2 100644 --- a/tools/acceptance.py +++ b/tools/acceptance.py @@ -5,7 +5,7 @@ Core supplies a rehearsal root and thread. A fresh frontend home does NOT restore source-bound saved Toad tabs. Plain st and the real ACP editor share TerminalApp's -post-attachment source/runtime/persistence admission. SHOWN, SEEN and the final +startup-owned source/runtime/persistence admission. SHOWN, SEEN and the final JSON retain Core's existing protocol; failed/UNKNOWN fresh input is never replayed. """ from __future__ import annotations @@ -125,7 +125,7 @@ def main() -> None: target = TerminalApp.prepare_target(comms.root, LaunchRuntime.at_prefix(args.runtime.parent)) with OwnedDisplay(args.output) as display: command = (str(args.runtime / "toad"),) - plain = TerminalApp(command, display, args.output / "plain", target, target.home) + plain = TerminalApp((*command, str(target.home)), display, args.output / "plain", target, target.home) with display.recording(args.output / "journey.mp4", plain.environment): with plain as app: app.wait("store", lambda screen: "Toad" in screen.text or "agent" in screen.text.lower()) diff --git a/tools/performance/capture_live.py b/tools/performance/capture_live.py index e0b50273b..62c71ead0 100644 --- a/tools/performance/capture_live.py +++ b/tools/performance/capture_live.py @@ -1,170 +1,70 @@ -"""Measure an explicitly selected live Toad process on required --display. +"""Read-only py-spy capture of an explicit live PID on an isolated display. -Physical input belongs to physical_ui.TerminalApp's observed private-run lifetime. ---profile-seconds record a py-spy speedscope profile. ---frame-meter N time UI work per frame and input-to-paint for N seconds (frame_meter.py). ---targets write where sidebar thread rows and session tabs are (frame_meter.targets). +In-process observations belong to physical_ui.TerminalApp's startup lifetime. +This command never injects frame meters, callbacks, heap changes or UI input. """ import argparse -from contextlib import ExitStack import json -import os from pathlib import Path import subprocess import sys import time -import psutil +from agent_comms.child_process import ProcessIdentity +from agent_comms.nominal.field_codec import FieldCodec - -# The shared physical resource owns Xlib admission; capture never owns raw input. sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from physical_ui import XvfbDisplay def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pid", type=int, required=True) - parser.add_argument("--display", required=True, help="Explicit isolated Xvfb display shared with the input controller") + parser.add_argument("--display", required=True, help="Explicit owned isolated Xvfb display") parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--name", default="live-" + time.strftime("%Y%m%d-%H%M%S")) - parser.add_argument("--profile-seconds", type=int, default=0) + parser.add_argument("--profile-seconds", type=int, required=True) parser.add_argument("--rate", type=int, default=100) parser.add_argument("--py-spy", default="py-spy") parser.add_argument("--gil", action="store_true") - parser.add_argument("--idle", action="store_true", help="Include samples of threads blocked in calls") + parser.add_argument("--idle", action="store_true") parser.add_argument("--native", action="store_true") - parser.add_argument("--frame-meter", type=float, default=0) - parser.add_argument("--targets", action="store_true") - parser.add_argument("--profile-layout", type=int, default=0, help="Profile the next N calls of --profile-target") - parser.add_argument("--profile-target", default="textual.screen:Screen._refresh_layout", - help="module:Class.method profiled by --profile-layout") - parser.add_argument("--profile-within", default="", - help="Profile --profile-target only inside calls of this module:Class.method (frame_meter.profile_within); " - "'slow' profiles every call and keeps those over --profile-min-ms (frame_meter.profile_slow)") - parser.add_argument("--profile-min-ms", type=float, default=0.0, - help="With --profile-within: keep only scopes whose target call took this long (profiled)") - parser.add_argument("--trace-calls", default="", help="Comma-separated module:Class.method targets to time") - parser.add_argument("--trace-reflows", action="store_true", help="Record each scoped reflow (frame_meter.trace_reflows)") - parser.add_argument("--trace-rules", action="store_true", help="Record each style-rule change (frame_meter.trace_rules)") - parser.add_argument("--trace-exit", action="store_true", help="Record who asks the app to exit (frame_meter.trace_exit)") - parser.add_argument("--trace-posts", action="store_true", help="Messages posted to screens, 40 s (frame_meter.trace_posts)") - parser.add_argument("--trace-abc", action="store_true", help="Record who grows ABC caches, 40 s (frame_meter.trace_abc_growth)") - parser.add_argument("--freeze-heap", action="store_true", help="Experiment: freeze the heap now (frame_meter.freeze_heap)") - parser.add_argument("--heap-census", type=float, default=0, - help="After N seconds count the collector's tracked objects by type (frame_meter.heap_census)") - parser.add_argument("--collect-all", type=float, default=0, - help="After N seconds run one full collection, timing its scan and its frees (frame_meter.collect_all)") - parser.add_argument("--heap-components", type=float, default=0, - help="After N seconds record how the unfrozen heap hangs together (frame_meter.heap_components)") - parser.add_argument("--trace-garbage", action="store_true", - help="Count by type what each large collection found, for 12 s (frame_meter.trace_garbage)") - parser.add_argument("--remeasure", type=int, default=0, - help="Arrange the screen at N new widths and time each pass (frame_meter.remeasure)") - parser.add_argument("--repaint", type=int, default=0, - help="Repaint the history viewport N times cold and warm, timing each (frame_meter.repaint)") - parser.add_argument("--sudo", action="store_true", help="Use non-interactive sudo for read-only py-spy profiling only") + parser.add_argument("--sudo", action="store_true", help="Non-interactive sudo for read-only py-spy only") args = parser.parse_args() - if not (args.profile_seconds > 0 or args.frame_meter > 0 or args.targets or args.profile_layout or args.trace_calls - or args.trace_reflows or args.trace_rules or args.trace_exit or args.trace_garbage or args.collect_all or args.heap_components or args.heap_census or args.freeze_heap or args.trace_abc or args.trace_posts - or args.remeasure or args.repaint): - parser.error("Choose --profile-seconds, --frame-meter, --targets or --profile-layout") + if args.profile_seconds <= 0 or args.rate <= 0: + parser.error("Profile duration and sampling rate must be positive") if Path(args.name).name != args.name: parser.error("--name must be a capture basename") args.output_dir = args.output_dir.expanduser().resolve() args.output_dir.mkdir(parents=True, exist_ok=True) prefix = args.output_dir / args.name manifest_path = Path(str(prefix) + "-manifest.json") - if manifest_path.exists(): + profile_path = Path(str(prefix) + ".speedscope.json") + if manifest_path.exists() or profile_path.exists(): parser.error("Choose a fresh capture name") - process = psutil.Process(args.pid) - created = process.create_time() + identity = ProcessIdentity.capture(args.pid) + manifest = {"process": FieldCodec.encode(identity), "started_ns": time.time_ns(), + "executable": str(Path(f"/proc/{identity.pid}/exe").resolve(strict=True))} privilege = ["sudo", "-n"] if args.sudo else [] - manifest = {"pid": args.pid, "created": created, "executable": process.exe(), "started_ns": time.time_ns()} - close_display = ExitStack() - display = close_display.enter_context(XvfbDisplay(args.display)) - manifest["display"] = display.name try: - if args.profile_seconds: - result = subprocess.run([*privilege, args.py_spy, "record", "--pid", str(args.pid), - "--duration", str(args.profile_seconds), "--rate", str(args.rate), "--format", "speedscope", - "--output", str(prefix) + ".speedscope.json", *(["--gil"] if args.gil else []), - *(["--idle"] if args.idle else []), *(["--native"] if args.native else [])], - timeout=args.profile_seconds + 30) + with XvfbDisplay(args.display) as display: + manifest["display"] = display.name + result = subprocess.run( + [*privilege, args.py_spy, "record", "--pid", str(identity.pid), + "--duration", str(args.profile_seconds), "--rate", str(args.rate), + "--format", "speedscope", "--output", str(profile_path), + *(["--gil"] if args.gil else []), *(["--idle"] if args.idle else []), + *(["--native"] if args.native else [])], + timeout=args.profile_seconds + 30, + ) manifest["profile_returncode"] = result.returncode result.check_returncode() - calls = [] - if args.targets: - calls.append((str(prefix) + "-targets", "targets", "")) - if args.frame_meter > 0: - calls.append((str(prefix) + "-frames", "install", f", seconds={args.frame_meter!r}")) - if args.profile_layout and args.profile_within == "slow": - calls.append((str(prefix) + "-layout", "profile_slow", - f", calls={args.profile_layout!r}, target={args.profile_target!r}, min_ms={args.profile_min_ms!r}")) - elif args.profile_layout and args.profile_within: - calls.append((str(prefix) + "-layout", "profile_within", - f", calls={args.profile_layout!r}, target={args.profile_target!r}, within={args.profile_within!r}, min_ms={args.profile_min_ms!r}")) - elif args.profile_layout: - calls.append((str(prefix) + "-layout", "profile_layout", - f", calls={args.profile_layout!r}, target={args.profile_target!r}")) - if args.trace_calls: - calls.append((str(prefix) + "-calls", "trace_calls", - f", targets={args.trace_calls.split(',')!r}, seconds={max(args.frame_meter, 40)!r}")) - if args.remeasure: - calls.append((str(prefix) + "-remeasure", "remeasure", f", iterations={args.remeasure!r}")) - if args.repaint: - calls.append((str(prefix) + "-repaint", "repaint", f", iterations={args.repaint!r}")) - if args.trace_exit: - calls.append((str(prefix) + "-exit", "trace_exit", "")) - if args.trace_rules: - calls.append((str(prefix) + "-rules", "trace_rules", f", seconds={max(args.frame_meter, 40)!r}")) - if args.trace_posts: - calls.append((str(prefix) + "-posts", "trace_posts", "")) - if args.trace_abc: - calls.append((str(prefix) + "-abc", "trace_abc_growth", "")) - if args.freeze_heap: - calls.append((str(prefix) + "-freeze", "freeze_heap", "")) - if args.heap_census: - calls.append((str(prefix) + "-heap", "heap_census", f", after={args.heap_census!r}")) - if args.collect_all: - calls.append((str(prefix) + "-collect", "collect_all", f", after={args.collect_all!r}")) - if args.heap_components: - calls.append((str(prefix) + "-components", "heap_components", f", after={args.heap_components!r}")) - if args.trace_garbage: - calls.append((str(prefix) + "-garbage", "trace_garbage", ", seconds=150")) - if args.trace_reflows: - calls.append((str(prefix) + "-reflows", "trace_reflows", f", seconds={max(args.frame_meter, 40)!r}")) - if calls: - assert process.is_running() and process.create_time() == created, "Target process identity changed" - meter = Path(__file__).resolve().with_name("frame_meter.py") - entered = str(prefix) + "-remote-entered.json" - lines = [ - "import importlib.util as _spec_util, json as _json, os as _os, sys as _sys, time as _time", - # One meter module per process: snapshots record their pauses where the meter reads them. - "_meter = _sys.modules.get('frame_meter')", - "if _meter is None:", - f" _spec = _spec_util.spec_from_file_location('frame_meter', {str(meter)!r})", - " _meter = _sys.modules['frame_meter'] = _spec_util.module_from_spec(_spec)", - " _spec.loader.exec_module(_meter)", - *(f"_meter.{call}(expected_pid={args.pid}, output={output + '.json'!r}{extra})" - for output, call, extra in calls), - f"with open({entered!r}, 'w') as _out: _json.dump({{'pid': _os.getpid(), 'entered_ns': _time.time_ns()}}, _out)", - ] - script = Path(str(prefix) + "-remote.py") - script.write_text("\n".join(lines) + "\n") - subprocess.run([process.exe(), "-c", - f"import sys; sys.remote_exec({args.pid}, {str(script)!r})"], check=True, timeout=15) - deadline = time.monotonic() + max(args.frame_meter, 60 if args.profile_layout or args.remeasure or args.repaint else 0) + 20 - outputs = [output + ".json" for output, _call, _extra in calls] - while time.monotonic() < deadline and not all(Path(path).exists() for path in outputs): - time.sleep(.1) - manifest["outputs"] = {path: Path(path).exists() for path in outputs} - manifest["entered"] = Path(entered).exists() + if not identity.alive(): + raise RuntimeError("The selected process birth ended during profiling") finally: - close_display.close() manifest["finished_ns"] = time.time_ns() - manifest_path.write_text(json.dumps(manifest, indent=2)) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") print(manifest_path) diff --git a/tools/physical_ui.py b/tools/physical_ui.py index f762fd507..a508b1101 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -11,14 +11,16 @@ import ctypes from dataclasses import dataclass import itertools +import json +from multiprocessing import Pipe import os from pathlib import Path import re import select +import shlex import socket import struct import subprocess -import sys import time from typing import get_args, get_type_hints @@ -42,7 +44,7 @@ from ui_theme.wal import WalFile from physical_ui_remote import ( - AdmitOperation, CapturedRegion, ExecOperation, ObserveOperation, PendingRun, PhysicalOperation, + AdmitOperation, CapturedRegion, StartOperation, ObserveOperation, PendingRun, PhysicalOperation, ProcessRetirement, RunObservation, RunTarget, StopRootOperation, ) @@ -221,7 +223,8 @@ class TerminalApp: def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, target: RunTarget, cwd: Path, *, size: Size = Size(140, 40)): - self.command, self.display, self.output = tuple(map(str, command)), display, output + self.command, self.display = tuple(map(str, command)), display + self.output = output.expanduser().resolve() self.target, self.cwd, self.requested_size = target, cwd, size environment = LaunchRoot.environment(ThreadLaunch.strip(dict(os.environ))) for key in tuple(environment): @@ -306,13 +309,16 @@ def copied(cls, source: Comms, threads: tuple[str, ...], runtime: LaunchRuntime, output = output.expanduser().absolute() output.mkdir(mode=0o700, parents=True) # Fresh evidence, never overwrite a failed run. copied = CheckRoot(source, threads, output / "bus").create() + project = copied.registry.require(thread).project_path RootRuntime.publish(copied.root, runtime) target = cls.prepare_target(copied.root, runtime) try: with OwnedDisplay(output / "display", size=framebuffer) as display: - # Keep the real default launcher: its root/runtime succession is - # precisely what the post-attachment observation must establish. - command = (str(Path.home() / ".local/bin/toad-comms"), thread) + # Startup instrumentation runs the actual installed Toad CLI. + # This is NOT proof of the separate toad-comms shell launcher. + command = (str(runtime.interpreter.parent / "toad"), "acp", + shlex.join((str(runtime.interpreter), "-m", "agent_comms.acp")), + str(project), "--session", thread, "--title", "Agent Comms") app = cls(command, display, output / "frontend", target, target.home, size=size) with display.recording(output / "journey.mp4", app.environment), app: yield app @@ -328,11 +334,9 @@ def operation_file(cls, operation: PhysicalOperation, directory: Path) -> Path: # controller lifetime. Later source edits cannot change a run's wire. raw = cls.operation_program script = directory / "operation.py" - # remote_exec supplies no module filename. Bind only that external - # datum; the program stays in its reviewed source file and the request - # stays in FieldCodec's adjacent record, as in capture_history. - filename = f"__file__ = {str(script)!r}\n".encode() - ExclusiveAtomicFile(script).write_bytes(filename + raw) + # Only startup/shutdown executes a program. Normal Python supplies its + # filename; later observations submit data to the startup-owned reader. + ExclusiveAtomicFile(script).write_bytes(raw) PhysicalOperation.publish(script.with_suffix(".json"), operation) return script @@ -352,23 +356,37 @@ def stop_copy(cls, target: RunTarget, directory: Path) -> None: child.collect(timeout=30).check_returncode() cls.result(directory) - def _packet(self, operation: PhysicalOperation) -> Path: - return self.operation_file(operation, self.output / "operations" / f"{next(self.operations):06d}") + def _packet(self, operation: ObserveOperation) -> Path: + directory = self.output / "operations" / f"{next(self.operations):06d}" + directory.mkdir(mode=0o700, parents=True) + packet = directory / "request.json" + PhysicalOperation.publish(packet, operation) + return packet def __enter__(self): self.output.mkdir(mode=0o700, parents=True) self.dump = self.output / "st-screen.txt" self.log = (self.output / "st.log").open("wb") - script = self._packet(ExecOperation(self.target, ProcessIdentity.current(), self.command)) - self.terminal = SpawnedProcess.launch( - ("st", "-f", "DejaVu Sans Mono:pixelsize=16", - "-g", f"{self.requested_size.width}x{self.requested_size.height}", "-o", str(self.dump), "-e", "env", - f"HOME={self.target.home}", f"XDG_CONFIG_HOME={self.target.config}", - f"XDG_STATE_HOME={self.target.state}", f"XDG_DATA_HOME={self.target.data}", - str(self.target.launch_runtime.interpreter), "-I", str(script)), - cwd=self.cwd, env=self.environment, - stdio=DetachedChildStdio(stdout=self.log, stderr=self.log), - ) + frontend_requests, self.observer = Pipe(duplex=False) + try: + descriptor = frontend_requests.fileno() + script = self.operation_file(StartOperation( + self.target, ProcessIdentity.current(), self.command, descriptor), self.output / "startup") + self.terminal = SpawnedProcess.launch( + ("st", "-f", "DejaVu Sans Mono:pixelsize=16", + "-g", f"{self.requested_size.width}x{self.requested_size.height}", "-o", str(self.dump), "-e", "env", + f"HOME={self.target.home}", f"XDG_CONFIG_HOME={self.target.config}", + f"XDG_STATE_HOME={self.target.state}", f"XDG_DATA_HOME={self.target.data}", + str(self.target.launch_runtime.interpreter), "-I", str(script)), + cwd=self.cwd, env=self.environment, pass_fds=(descriptor,), + stdio=DetachedChildStdio(stdout=self.log, stderr=self.log), + ) + except BaseException: + self.observer.close() + self.log.close() + raise + finally: + frontend_requests.close() try: PhysicalOperation.publish(self.output / "st-process.json", self.terminal.identity) self.window = int(subprocess.check_output( @@ -377,12 +395,10 @@ def __enter__(self): deadline = time.monotonic() + self.startup_seconds self._await_result(script.parent, deadline) self.frontend = self.result(script.parent).process() - # Wait until the exec wrapper has left and the real interpreter has - # emitted terminal data. Neither is application/source readiness: - # that answer comes from the live Toad owner in observe() below. + # Startup stays in the same interpreter; native image and terminal + # data do not imply application/source readiness. The UI owner does. while (Path(f"/proc/{self.frontend.pid}/exe").resolve() != self.target.launch_runtime.interpreter.resolve() - or os.fsencode(script) in Path(f"/proc/{self.frontend.pid}/cmdline").read_bytes() or not self.dump.stat().st_size): self._require_starting(deadline) time.sleep(.05) @@ -415,12 +431,12 @@ def _await_result(self, directory: Path, deadline: float) -> None: def _observe(self, operation: type[ObserveOperation]): self.terminal.platform.require(self.frontend) - script = self._packet(operation(self.target, self.frontend, Path(__file__).resolve().parent)) - # This controller is an ancestor of its st/frontend. No arbitrary PID, - # helper privilege escalation, installed-file edit or source retry. - sys.remote_exec(self.frontend.pid, str(script)) - self._await_result(script.parent, time.monotonic() + PhysicalOperation.timeout) - result = self.result(script.parent) + packet = self._packet(operation(self.target, self.frontend, Path(__file__).resolve().parent)) + # Only this new frontend inherited the reader. It decodes this data + # packet and applies it through Textual's own main-loop boundary. + self.observer.send_bytes(json.dumps(FieldCodec.encode(packet)).encode()) + self._await_result(packet.parent, time.monotonic() + PhysicalOperation.timeout) + result = self.result(packet.parent) if result.observation().frontend.process != self.frontend: raise ValueError("Observation belongs to another frontend incarnation") self.terminal.platform.require(self.terminal.identity) @@ -586,6 +602,7 @@ def resize(self, columns: int, rows: int) -> None: def close(self) -> None: """Retire one child lifetime on normal exit and failed entry alike.""" try: + self.observer.close() outcome = self.terminal.stop_sync(deadline=time.monotonic() + self.stop_seconds) PhysicalOperation.publish(self.output / "st-retired.json", ProcessRetirement(self.terminal.identity, outcome)) @@ -604,6 +621,7 @@ def __exit__(self, exception_type, exception, traceback): raise ValueError("The verified runtime build changed during the physical run") PhysicalOperation.publish(self.output / "build-final.json", final) self.keys("ctrl+q") + self.observer.close() self.terminal.process.wait(timeout=5) finally: self.close() diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index 6802f31cb..1876b3874 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -1,29 +1,36 @@ """Typed operations of one owned physical run, copied beside their requests. -The exec operation grants only its recorded controller's descendants debugging -access, then execs the unchanged launcher. Observation reads live source owners; -it never opens a Comms, sends input, repairs a route or traverses the heap. +Startup owns one inherited request connection and the installed Python entrypoint. +The controller submits only typed observation packets, applied on Textual's loop. +No attachment, debugger permission, code injection or signal handler is used. """ from abc import abstractmethod -import ctypes from dataclasses import dataclass +from importlib.metadata import distribution import json +from multiprocessing.connection import Connection import os from pathlib import Path import pwd import sys import time +from threading import Thread from typing import ClassVar +from agent_comms.acp_extension import CoordinationChangedUpdate from agent_comms.child_process import ChildOutcome, ProcessIdentity from agent_comms.launch_runtime import LaunchRuntime from agent_comms.installer.probes.resolution import RuntimeResolutionProbe -from agent_comms.installer.runtime_resolution import RuntimeResolution +from agent_comms.installer.runtime_resolution import RuntimeResolution, ToadPackage from agent_comms.storage_rules import StorageRule from agent_comms.nominal.declared_family import DeclaredFamily from agent_comms.nominal.field_codec import DeclaredWireType, FieldCodec +from metaclass_registry.mro_dispatch import handles from textual.geometry import Region +from toad.acp.agent import Agent +from toad.agent_binding import NoAgentConnection +from toad.core.projection import MroProjection from ui_theme.wal import WalFile from agent_comms.store_files import ExclusiveAtomicFile @@ -78,9 +85,9 @@ def data(self) -> Path: def require_resources(self, observed): """Current UI/source resources; build evidence belongs to AdmittedResult.""" if observed.home != self.home or observed.runtime != self.runtime: - raise ValueError("Post-attachment HOME/runtime differs from the run target") + raise ValueError("Observed HOME/runtime differs from the run target") if (observed.config, observed.state, observed.data) != (self.config, self.state, self.data): - raise ValueError("Post-attachment XDG persistence differs from the private run") + raise ValueError("Observed XDG persistence differs from the private run") if observed.palette_directory != self.palette: raise ValueError("The actual theme palette is outside the private home") if not observed.database.is_relative_to(self.state): @@ -165,7 +172,7 @@ def observation(self) -> RunObservation: raise ValueError("This physical operation did not observe an application") def process(self) -> ProcessIdentity: - raise ValueError("This physical operation did not exec a frontend") + raise ValueError("This physical operation did not start a frontend") @dataclass(frozen=True) @@ -195,9 +202,9 @@ def observe(self, target: RunTarget, current: RunObservation) -> RunObservation: @dataclass(frozen=True) -class ExecResult(PhysicalResult): +class StartedResult(PhysicalResult): identity: ProcessIdentity - debugger: ProcessIdentity + controller: ProcessIdentity def process(self) -> ProcessIdentity: return self.identity @@ -270,31 +277,93 @@ def execute_file(cls, script: Path) -> None: operation.execute(directory) except PendingRun as error: cls.publish(result, PendingResult(str(error)[:2048])) - except BaseException as error: + except Exception as error: cls.publish(directory / "error.json", FailedResult(type(error).__name__, str(error)[:2048])) @dataclass(frozen=True) -class ExecOperation(PhysicalOperation): - """One new child, not an arbitrary existing trace target or global policy.""" +class StartOperation(PhysicalOperation): + """Instrument a new Python frontend at startup, never an existing process.""" target: RunTarget - debugger: ProcessIdentity + controller: ProcessIdentity arguments: tuple[str, ...] - PR_SET_PTRACER: ClassVar[int] = 0x59616D61 # Linux's declared prctl ABI. + descriptor: int def execute(self, directory: Path) -> None: - if not self.arguments or not self.debugger.alive(): - raise ValueError("Exec needs its original living controller and command") + if not self.arguments or not self.controller.alive() or self.descriptor < 3: + raise ValueError("Startup needs its living controller, entrypoint and inherited connection") if Path.home().resolve() != self.target.home: - raise ValueError("st changed HOME before frontend exec") - prctl = ctypes.CDLL(None, use_errno=True).prctl - prctl.argtypes = (ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong) - prctl.restype = ctypes.c_int - if prctl(self.PR_SET_PTRACER, self.debugger.pid, 0, 0, 0) != 0: - raise OSError(ctypes.get_errno(), "Cannot admit this controller's debugger descendants") - self.publish(directory / "result.json", ExecResult(ProcessIdentity.current(), self.debugger)) - os.execvpe(self.arguments[0], self.arguments, os.environ) + raise ValueError("st changed HOME before frontend startup") + if LaunchRuntime.running().prefix != self.target.runtime: + raise ValueError("Frontend startup must use its declared runtime") + entrypoint = Path(self.arguments[0]).resolve(strict=True) + if not entrypoint.is_relative_to(self.target.runtime): + raise ValueError("The instrumented entrypoint is outside the declared runtime") + (declared,) = distribution(ToadPackage.distribution).entry_points.select( + group="console_scripts", name=entrypoint.name, + ) + os.set_inheritable(self.descriptor, False) + with Connection(self.descriptor, writable=False) as requests: + # This thread owns no UI state and never wakes periodically. Its + # private pipe closes with this frontend/controller lifetime. + reader = Thread(target=self.serve, args=(requests, directory.parent), daemon=True) + reader.start() + self.publish(directory / "result.json", StartedResult(ProcessIdentity.current(), self.controller)) + sys.argv = list(self.arguments) + # Keep this declaration module alive while the installed entrypoint + # runs; replacing __main__ would change the codec's type namespace. + declared.load()() + + def serve(self, requests: Connection, output: Path) -> None: + while True: + try: + raw = requests.recv_bytes(self.metadata_bytes) + except EOFError: + return + packet = FieldCodec.decode(Path, FieldCodec.parse(raw.decode())) + if (not packet.is_absolute() or packet != packet.resolve(strict=True) + or not packet.is_relative_to(output / "operations") + or packet.name != "request.json"): + raise ValueError("Observation packet is outside this startup's operation directory") + from textual._context import NoActiveAppError + from toad.app import ToadApp + + try: + app = ToadApp.running_app() + if not app.is_running: + raise PendingRun("The frontend message loop has not started") + # The existing decoder admits only ObserveOperation descendants; + # a stop/start packet cannot execute through this read-only pipe. + app.call_from_thread(ObserveOperation.execute_file, packet) + except (NoActiveAppError, PendingRun) as error: + self.publish(packet.parent / "result.json", PendingResult(str(error)[:2048])) + except Exception as error: + self.publish(packet.parent / "error.json", FailedResult(type(error).__name__, str(error)[:2048])) + + +class AcpRoots(MroProjection[tuple[Path, ...]]): + """One foreign-connection decoder, not a new handler class per observation.""" + + @handles(object) + def unknown(self, value): + raise ValueError(f"Unadmitted ACP metadata type: {type(value).__name__}") + + @handles(type(None)) + def pending(self, value): + raise PendingRun("A bound ACP has not published its coordination root") + + @handles(NoAgentConnection) + def unbound(self, value): + return () + + @handles(Agent) + def bound(self, value): + return self.project_value(value.coordination) + + @handles(CoordinationChangedUpdate) + def coordinated(self, value): + return (Path(value.wire_root).resolve(),) @dataclass(frozen=True) @@ -309,9 +378,9 @@ def observed_result(self, observed: RunObservation) -> PhysicalResult: def execute(self, directory: Path) -> None: started = time.monotonic_ns() if ProcessIdentity.current() != self.identity: - raise ValueError("Post-attachment frontend incarnation changed") + raise ValueError("Observed frontend incarnation changed") if Path.home().resolve() != self.target.home: - raise ValueError("Post-attachment HOME is not private") + raise ValueError("Observed HOME is not private") # The existing reviewed validator remains the single module-location policy. # The temporary tool import path is retired before inspecting application state. previous = sys.path[:] @@ -323,40 +392,12 @@ def execute(self, directory: Path) -> None: import agent_comms import textual import toad - from agent_comms.acp_extension import CoordinationChangedUpdate - from metaclass_registry.mro_dispatch import handles - from toad.acp.agent import Agent - from toad.agent_binding import NoAgentConnection from toad.app import ToadApp from textual._context import NoActiveAppError from textual.app import ScreenStackError - from toad.core.projection import MroProjection from toad.widgets.history_anchor import HistoryWindow from xdg_base_dirs import xdg_config_home, xdg_data_home, xdg_state_home - class AcpRoots(MroProjection[tuple[Path, ...]]): - """Decode foreign running-UI connections once, without guessing readiness.""" - - @handles(object) - def unknown(self, value): - raise ValueError(f"Unadmitted ACP metadata type: {type(value).__name__}") - - @handles(type(None)) - def pending(self, value): - raise PendingRun("A bound ACP has not published its coordination root") - - @handles(NoAgentConnection) - def unbound(self, value): - return () - - @handles(Agent) - def bound(self, value): - return self.project_value(value.coordination) - - @handles(CoordinationChangedUpdate) - def coordinated(self, value): - return (Path(value.wire_root).resolve(),) - try: app = ToadApp.running_app() screen = app.screen @@ -427,8 +468,8 @@ def execute(self, directory: Path) -> None: self.publish(directory / "result.json", StoppedResult(self.target.root)) -# Normal imports define the family. A controller-bound module filename selects -# only that immutable copy's adjacent request, including under remote_exec. +# Normal imports define the family. A startup/shutdown program selects only +# its immutable adjacent request; observations decode data, never program text. _source = Path(__file__) if _source.with_suffix(".json").is_file(): PhysicalOperation.execute_file(_source) From 21fa51b7971c42856c6c29d496f17a7ea8dc9f19 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 17:41:15 -0500 Subject: [PATCH 15/35] Execute physical operations only at the explicit module entrypoint --- tools/physical_ui_remote.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index 1876b3874..eb27bbeb3 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -468,8 +468,7 @@ def execute(self, directory: Path) -> None: self.publish(directory / "result.json", StoppedResult(self.target.root)) -# Normal imports define the family. A startup/shutdown program selects only -# its immutable adjacent request; observations decode data, never program text. -_source = Path(__file__) -if _source.with_suffix(".json").is_file(): - PhysicalOperation.execute_file(_source) +# Imports, including multiprocessing's __mp_main__, only declare the family. +# Explicit startup/shutdown execution owns its immutable adjacent request. +if __name__ == "__main__": + PhysicalOperation.execute_file(Path(__file__)) From 09108191e389f870d3fb84d12d6886af6c031c05 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 18:43:22 -0500 Subject: [PATCH 16/35] Observe saved publication instead of demanding a running agent --- tests/model/test_physical_run.py | 2 +- tests/textual/test_snapshot_cancellation.py | 49 ++++++++------------- tools/physical_ui.py | 14 +++--- tools/physical_ui_remote.py | 10 ++++- 4 files changed, 35 insertions(+), 40 deletions(-) diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py index af47d7d91..ab33b0c13 100644 --- a/tests/model/test_physical_run.py +++ b/tests/model/test_physical_run.py @@ -40,7 +40,7 @@ def physical_observation(physical_target: RunTarget) -> RunObservation: database=target.state / "toad.db", palette_directory=target.palette, root=target.root, opened=True, storage_rule=StorageRule().within(target.root), saved_roots=(target.root,), - acp_roots=(target.root,), history_regions=(), modules_checked=3, + acp_roots=(target.root,), history_regions=(), history_published=False, modules_checked=3, ) diff --git a/tests/textual/test_snapshot_cancellation.py b/tests/textual/test_snapshot_cancellation.py index c859d7b2c..1cc7ebcae 100644 --- a/tests/textual/test_snapshot_cancellation.py +++ b/tests/textual/test_snapshot_cancellation.py @@ -7,7 +7,6 @@ import asyncio from dataclasses import replace import os -from unittest.mock import patch from agent_comms.acp_extension import CoordinationChangedUpdate from agent_comms.child_process import ProcessIdentity @@ -53,36 +52,24 @@ async def test_cancelled_commit_with_the_output_owner_busy_retires_the_old_histo block = await view.output.append(ResponseStream(), new_page.events[0].text) view.window.anchor() view.prompt.focus() - entered = asyncio.Event() - retire = view.output.retire_presentations - covered = {} - - async def observe_retirement(candidates): - covered["old"] = old in candidates - covered["stream"] = block in candidates - entered.set() - await retire(candidates) - - with patch.object(view.output, "retire_presentations", observe_retirement): - async with view.output.lock: - # The transcript owner runs its own commit; a second one built here would race it. - view.transcript.require_commit() - await asyncio.wait_for(entered.wait(), 5) - # While the output owner is held, finishing must still be able to take the source and tree locks. - async with asyncio.timeout(5): - async with view.window.history_lock: - async with view.window.lock: - pass - assert not old.display - # The owner may have restarted its commit; the one retiring now is its current worker. - worker = view.transcript.worker - worker.cancel() - await ui.step(0.05) - with pytest.raises(WorkerCancelled): - await asyncio.wait_for(worker.wait(), 5) - await ui.step(0.1) - assert covered["old"], "The source did not cover the old history" - assert covered["stream"], "Source did not cover the live stream" + async with view.output.lock: + # The transcript owner runs its own commit; a second one built here would race it. + view.transcript.require_commit() + await ui.until(lambda: not old.display and not block.display, + why="accepted source retirement to withdraw both original resources") + # Withdrawal is synchronous; the real output lock still prevents teardown. + assert old.is_attached and block.is_attached + # Joining retirement must not retain the source and tree locks. + async with asyncio.timeout(5): + async with view.window.history_lock: + async with view.window.lock: + pass + worker = view.transcript.worker + worker.cancel() + await ui.step(0.05) + with pytest.raises(WorkerCancelled): + await asyncio.wait_for(worker.wait(), 5) + await ui.step(0.1) assert not old.is_attached, "Accepted cancellation abandoned the old history" assert old not in view.window.histories, "Accepted cancellation kept two histories" assert not block.is_attached, "The covered live stream block survived retirement" diff --git a/tools/physical_ui.py b/tools/physical_ui.py index a508b1101..49b7c6622 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -44,7 +44,7 @@ from ui_theme.wal import WalFile from physical_ui_remote import ( - AdmitOperation, CapturedRegion, StartOperation, ObserveOperation, PendingRun, PhysicalOperation, + AdmitOperation, StartOperation, ObserveOperation, PendingRun, PhysicalOperation, ProcessRetirement, RunObservation, RunTarget, StopRootOperation, ) @@ -176,14 +176,14 @@ def __exit__(self, *exception): @dataclass(frozen=True) class ScreenCapture: - """Real st text paired with the published viewport geometry observed for it.""" + """Real st text paired with its original application observation.""" text: str - history_regions: tuple[CapturedRegion, ...] + observation: RunObservation def message_rows(self): lines = self.text.splitlines() - for captured in self.history_regions: + for captured in self.observation.history_regions: region = captured.region for row in range(max(0, region.y), min(len(lines), region.bottom)): yield row, Strip([Segment(lines[row])]).crop(region.x, region.right).text @@ -486,7 +486,7 @@ def capture(self): stream.seek(offset) data = stream.read() if data.count(b"\n") >= self.rows: - return ScreenCapture(data.decode("utf-8"), observed.history_regions) + return ScreenCapture(data.decode("utf-8"), observed) if self.terminal.process.poll() is not None or time.monotonic() >= deadline: raise RuntimeError("st did not finish its screen dump") time.sleep(.02) @@ -531,8 +531,8 @@ def wait(self, name, predicate, seconds=60): time.sleep(.2) def show_thread(self, thread): - self.wait("ready", lambda screen: thread in screen.text - and "▶ Ready" in "\n".join(screen.text.splitlines()[-8:])) + self.wait("history-published", lambda screen: thread in screen.text + and screen.observation.history_published) self.click_cell(self.columns - 1, self.rows // 2) return self.wait("shown", lambda screen: "User messages" in screen.text and all( "Indexing" not in line.split("│", 1)[0] for line in screen.text.splitlines())) diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index eb27bbeb3..de2b10cc5 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -150,6 +150,7 @@ class RunObservation: saved_roots: tuple[Path, ...] acp_roots: tuple[Path, ...] history_regions: tuple[CapturedRegion, ...] + history_published: bool modules_checked: int @@ -422,6 +423,13 @@ def execute(self, directory: Path) -> None: # from sidebar separators that may also occur inside a user's message. regions = tuple(CapturedRegion(region) for window in screen.query(HistoryWindow) for region in (window.published_content_region,) if region is not None) + histories = tuple(history for window in screen.query(HistoryWindow) + if window.published_content_region is not None + for history in window.histories) + # The pager's original publication state also supplies the footer's + # saved-history availability. Agent status is an independent fact. + history_published = bool(histories) and all( + history.state.reports_coverage for history in histories) frontend = FrontendIdentity( self.identity, Path(toad.__file__).resolve(), Path(agent_comms.__file__).resolve(), Path(textual.__file__).resolve(), @@ -431,7 +439,7 @@ def execute(self, directory: Path) -> None: Path.home().resolve(), xdg_config_home().resolve(), xdg_state_home().resolve(), xdg_data_home().resolve(), app.tab_order._db.path.resolve(), app.colours.wal_directory.resolve(), root, - opened, storage_rule, saved, acp, regions, checked, + opened, storage_rule, saved, acp, regions, history_published, checked, ) self.target.require_resources(observed) self.publish(directory / "result.json", self.observed_result(observed)) From 39be5382aae1d54b668204d11cb1462bb6128785 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 18:27:26 -0500 Subject: [PATCH 17/35] Keep failed append acquisition coupled to replacement publication --- src/toad/widgets/committed_presentation.py | 48 ++++++++------- .../test_transcript_presentation_fencing.py | 58 ++++++++++++++++++- 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/toad/widgets/committed_presentation.py b/src/toad/widgets/committed_presentation.py index 05aa6a607..255ad6540 100644 --- a/src/toad/widgets/committed_presentation.py +++ b/src/toad/widgets/committed_presentation.py @@ -246,6 +246,12 @@ class PreparedCommit: history: CommittedHistory sequences: frozenset[int] + @classmethod + async def capture(cls, view: Conversation, history: TranscriptHistory, page: TranscriptPage, + sequences: frozenset[int]) -> Self: + """A retained reader accepts coverage without preparing replacement paint.""" + return cls(history, sequences) + @property def outgoing(self) -> frozenset[OutgoingMessageReference]: return self.history.outgoing_coverage @@ -305,23 +311,34 @@ class PreparedAppendCommit(PreparedPageCommit): async def read_snapshot[Snapshot]( cls, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, ) -> Snapshot: - try: - return await read(position=history.page_source.through.later_page()) - except TranscriptChanged: - return await read() + return await read(position=history.page_source.through.later_page()) async def publish(self, cursor: TranscriptCursor, is_current: Callable[[], bool]) -> bool: return await self.history.publish_append(cursor, publication_current=is_current) class CommitPlan(ABC): - """The captured viewport intent owns source acquisition and publication.""" + """Capture source acquisition and viewport publication together for one read.""" + + preparation: type[PreparedCommit] + + def preparation_for_source(self, source: type[PreparedPageCommit]) -> type[PreparedCommit]: + """A held viewport retains its original paint after a successful read.""" + return PreparedCommit async def read_snapshot[Snapshot]( self, read: Callable[..., Awaitable[Snapshot]], history: CommittedHistory, ) -> Snapshot: - """The source's declared commit member owns acquisition and publication.""" - return await history.page_source.commit_preparation.read_snapshot(read, history) + """The source declares acquisition; its result fixes this read's publication.""" + source = history.page_source.commit_preparation + self.preparation = self.preparation_for_source(source) + try: + return await source.read_snapshot(read, history) + except TranscriptChanged: + # A full retry is a replacement, even when the file path is reused + # and its new cursor contains the old byte offset. + self.preparation = PreparedPageCommit + return await PreparedPageCommit.read_snapshot(read, history) @abstractmethod def current(self, window: HistoryWindow) -> bool: @@ -342,14 +359,9 @@ async def prepare(self, view: Conversation, history: TranscriptHistory, sequences |= history.covered_sequences(required) | frozenset( sequence for sequence in required if page.after.covers_incoming(sequence) ) - return await self.extend(view, history, page, sequences) + return await self.preparation.capture(view, history, page, sequences) return await PreparedPageCommit.capture(view, history, page, sequences) - @abstractmethod - async def extend(self, view: Conversation, history: TranscriptHistory, page: TranscriptPage, - sequences: frozenset[int]) -> PreparedCommit: - """Prepare publication within the history's existing source.""" - def commit(self, prepared: PreparedCommit, cursor: TranscriptCursor) -> None: """Apply source metadata only after the final retirement checks.""" @@ -368,9 +380,9 @@ def __init__(self, window: HistoryWindow) -> None: def current(self, window: HistoryWindow) -> bool: return window.follows_tail and window.scroll_revision == self.revision - async def extend(self, view: Conversation, history: TranscriptHistory, page: TranscriptPage, - sequences: frozenset[int]) -> PreparedPageCommit: - return await history.page_source.commit_preparation.capture(view, history, page, sequences) + def preparation_for_source(self, source: type[PreparedPageCommit]) -> type[PreparedPageCommit]: + """A following viewport publishes through the same member that acquired it.""" + return source def finish(self, view, cursor) -> None: view.transcript.binding.painted(view.transcript, cursor) @@ -403,10 +415,6 @@ def permits(self, view, candidates) -> bool: return not any(widget in visible and visible[widget][0].overlaps(viewport) for widget in candidates) - async def extend(self, view: Conversation, history: TranscriptHistory, page: TranscriptPage, - sequences: frozenset[int]) -> PreparedCommit: - return PreparedCommit(history, sequences) - def commit(self, prepared, cursor) -> None: prepared.history.retain_committed(cursor) diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 1cdb1170e..0590fcce2 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -231,7 +231,8 @@ async def test_followed_commit_publishes_a_known_undrawn_gap_before_settling(wor assert ui.failure() is None -async def test_closed_same_source_commit_acquires_the_complete_replacement(workspace): +@pytest.mark.parametrize("following", (False, True)) +async def test_closed_same_source_commit_acquires_complete_coverage(workspace, following): import json from dataclasses import replace @@ -247,17 +248,70 @@ async def test_closed_same_source_commit_acquires_the_complete_replacement(works await view.transcript.suspend() history = view.transcript.binding.history history.close() + if not following: + view.window.release_anchor() + pages = tuple(history.pages) + fragments = tuple(history.fragments_in_range) + position = view.window.reader_position with journal.open("a") as output: output.write(json.dumps(assistant_entry( "CLOSED_PREFIX_SUCCESSOR", id="after", parent="before")) + "\n") view.transcript.dirty = view.transcript.commit_required = True await view.transcript.publish(CommitPublication) - await ui.until_shows("CLOSED_PREFIX_SUCCESSOR") + if following: + await ui.until_shows("CLOSED_PREFIX_SUCCESSOR") + else: + await ui.step() + assert tuple(history.pages) == pages + assert all(actual is original for actual, original in + zip(history.fragments_in_range, fragments, strict=True)) + assert view.window.reader_position == position + assert "CLOSED_PREFIX_SUCCESSOR" not in ui.painted(view.window) assert "CLOSED_PREFIX_MUST_SURVIVE" in ui.painted(view.window) assert view.transcript.binding.history is history assert ui.failure() is None +@pytest.mark.parametrize("following", (False, True)) +async def test_rewritten_saved_file_replaces_paint_even_when_its_cursor_grew(workspace, following): + from dataclasses import replace + from agent_comms.errors import TranscriptChanged + + thread = workspace.thread("rewritten-source") + journal = write_journal(thread.project_path / "rewritten.jsonl", + assistant_entry("SUPERSEDED_SOURCE_ROW", id="old")) + workspace.comms.registry.declare(replace( + thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) + async with workspace.app(resume="rewritten-source") as ui: + await ui.until_shows("SUPERSEDED_SOURCE_ROW") + view = ui.conversation + await ui.app.coordination_access.stop_observing() + await view.transcript.suspend() + history = view.transcript.binding.history + original = history.page_source.through + if not following: + view.window.release_anchor() + # Reuse the path, but replace its native bytes with a longer record. + # Filename/offset containment alone cannot witness this old prefix. + write_journal(journal, assistant_entry( + "REPLACEMENT_SOURCE_ROW " * 80, id="new")) + + def read_replacement(): + read = workspace.comms.transcripts.capture_page_read("rewritten-source") + with pytest.raises(TranscriptChanged): + read.read(position=original.later_page()) + return read.read() + + replacement = await asyncio.to_thread(read_replacement) + assert replacement.after.contains(original) + view.transcript.dirty = view.transcript.commit_required = True + await view.transcript.publish(CommitPublication) + await ui.until_shows("REPLACEMENT_SOURCE_ROW") + assert "SUPERSEDED_SOURCE_ROW" not in ui.painted(view.window) + assert view.transcript.binding.history is history + assert ui.failure() is None + + @pytest.mark.parametrize("pressure", (False, True)) async def test_prune_pressure_counts_live_blocks_not_saved_history(workspace, pressure): from dataclasses import replace From 33059e87099e06170c955e684cea35ef02ea1db3 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 19:20:50 -0500 Subject: [PATCH 18/35] Wait for source publication at the observation boundary --- tools/physical_ui.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tools/physical_ui.py b/tools/physical_ui.py index 49b7c6622..d5ba30af6 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -403,14 +403,8 @@ def __enter__(self): self._require_starting(deadline) time.sleep(.05) self.read_grid() - while True: - try: - self.admission = self._observe(AdmitOperation).admitted(self.target) - PhysicalOperation.publish(self.output / "build-admitted.json", self.admission) - break - except PendingRun: - self._require_starting(deadline) - time.sleep(.1) + self.admission = self._observe(AdmitOperation).admitted(self.target) + PhysicalOperation.publish(self.output / "build-admitted.json", self.admission) # -o enables st's raw printer too. Even these printer keys pass the # same source admission as every later physical key/click. self.keys("ctrl+Print") @@ -430,13 +424,21 @@ def _await_result(self, directory: Path, deadline: float) -> None: time.sleep(.02) def _observe(self, operation: type[ObserveOperation]): - self.terminal.platform.require(self.frontend) - packet = self._packet(operation(self.target, self.frontend, Path(__file__).resolve().parent)) - # Only this new frontend inherited the reader. It decodes this data - # packet and applies it through Textual's own main-loop boundary. - self.observer.send_bytes(json.dumps(FieldCodec.encode(packet)).encode()) - self._await_result(packet.parent, time.monotonic() + PhysicalOperation.timeout) - result = self.result(packet.parent) + deadline = time.monotonic() + PhysicalOperation.timeout + while True: + self.terminal.platform.require(self.frontend) + packet = self._packet(operation(self.target, self.frontend, Path(__file__).resolve().parent)) + # Fresh read-only observations may see an ACP which has not yet + # published its root. No physical input is sent until it does; + # foreign resources and failed input are never retried here. + self.observer.send_bytes(json.dumps(FieldCodec.encode(packet)).encode()) + self._await_result(packet.parent, deadline) + try: + result = self.result(packet.parent) + break + except PendingRun: + self._require_starting(deadline) + time.sleep(.1) if result.observation().frontend.process != self.frontend: raise ValueError("Observation belongs to another frontend incarnation") self.terminal.platform.require(self.terminal.identity) From 50f340c3e02894cfda1702efdb7f6567436d5895 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sun, 1 Nov 2026 19:31:13 -0500 Subject: [PATCH 19/35] Enter prompt insertion before physical text input --- tools/acceptance.py | 2 +- tools/physical_ui.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/acceptance.py b/tools/acceptance.py index 9c69628b2..fe68f5a3b 100644 --- a/tools/acceptance.py +++ b/tools/acceptance.py @@ -43,7 +43,7 @@ def check_typed_reply(app: TerminalApp, thread: str, output: Path) -> None: attempt = {"status": "UNKNOWN", "replay": False, "thread": thread, "token": token} marker = output / "typed-input.json" app.click_cell(app.columns // 2, app.rows - 4) - app.type_text("Reply with " + token + ". Do not use tools.") + app.type_prompt("Reply with " + token + ". Do not use tools.") marker.write_text(json.dumps(attempt)) started = time.monotonic() app.keys("Return") diff --git a/tools/physical_ui.py b/tools/physical_ui.py index d5ba30af6..657061df4 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -475,7 +475,9 @@ def server_identity(self) -> None: def keys(self, *keys: str) -> RunObservation: return self.input("key", "--clearmodifiers", *keys) - def type_text(self, text: str) -> None: + def type_prompt(self, text: str) -> None: + """Use the prompt's normal-mode `i` action before sending literal text.""" + self.keys("Escape", "i") self.input("type", "--clearmodifiers", "--delay", "1", text) def capture(self): From cfec52a0886d05c066bc3f5bd63466a5dff2218e Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 00:13:18 -0500 Subject: [PATCH 20/35] Separate installed resource observation from candidate build admission --- docs/performance/physical-run.md | 70 +++++++++++--- tests/model/test_physical_run.py | 47 ++++++++-- tools/acceptance.py | 7 +- tools/performance/capture_live.py | 6 +- tools/performance/frame_meter.py | 2 +- tools/performance/scroll_scenario.py | 7 +- tools/physical_ui.py | 87 +++++++++++------- tools/physical_ui_admission.py | 73 +++++++++++++++ tools/physical_ui_candidate.py | 31 +++++++ tools/physical_ui_remote.py | 133 ++++++++++++++++++--------- 10 files changed, 355 insertions(+), 108 deletions(-) create mode 100644 tools/physical_ui_admission.py create mode 100644 tools/physical_ui_candidate.py diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 0963df31d..84afefddf 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -2,12 +2,53 @@ The shared implementation is `tools/physical_ui.py`: the existing `XvfbDisplay → OwnedDisplay` family owns Xlib/display allocation and retirement; -`TerminalApp` owns st, the frontend, source observations and all physical input. -`tools/acceptance.py` and the scrolling scenario consume it. There is no public -raw-input method on a display. - -## Entry point - +`ObservedTerminalApp` owns st, the frontend, source observations and all physical +input. `tools/physical_ui_candidate.py::TerminalApp` extends that same lifetime +with strict candidate-build verification. `tools/acceptance.py` and the existing +stopped-history CLI use the strict member. There is no public raw-input method +on a display. + +## Installed observation, without candidate admission + +Import `tools.physical_ui.ObservedTerminalApp` from the published source checkpoint +with the installed interpreter. Its existing `copied(source, threads, runtime, +output, thread=...)` factory owns the same `CheckRoot`, private home, lease/display, +startup connection and input checks. Use `LaunchRuntime.installed()` for installed +bytes. This does not require an interpreter observation in old build metadata. +Wait for both `screen.observation.history_published` and +`screen.observation.agent_ready` before the readiness-dependent journey; an active +capture additionally needs `active_response` and actual response text/focus. + +Entry/final `InstalledOperation` records the actual Python/native executable +paths and the installed runtime's package-origin answer through its existing +`installed_packages` wire operation. There is no duplicate metadata parser or +fallback probe import. Its `InstalledResult` checks origin continuity at normal +exit and retains `resources-observed.json` / `resources-final.json`. These are +origin observations, not candidate byte/interpreter proof. Per-input observations +still check the actual frontend birth, module locations and current private +resources; package inventory is not rescanned for every key. + +The qualified module namespace is `tools.physical_ui*` for controllers, consumers +and staged dependencies. Program source is captured only when execution starts, +not during ordinary imports. Candidate operations stage their required sibling +module under that same namespace. No live-process alias or import fallback exists. + +Data/source qualification: `observer-installed-06.log` decoded three staged +requests on the installed interpreter without loading the new candidate probe; +`observer-candidate-06.log` decoded three strict-member requests on the existing +immutable0402/3475 interpreter. Neither executes a frontend, build probe or input. +The source map covers Core src, Toad src/tools and the physical-run consumer: +971 modules, 8,192 raw classes, 8,184 canonical projections, zero parse omissions. +Dynamic imports/aliases, descriptor/effect behavior, foreign platforms, physical +movement and mounted readiness remain OPEN. A separate `observer-result-07.log` +on installed Core038c checks the result codec/origin continuity and changed- +executable refusal using an explicitly synthetic UI record and the real package +answer. It is not a mounted observation. No author pytest ran. + +## Candidate/build-verifying entry point + +This member requires Core3825's recorded-interpreter/probe APIs; that requirement +belongs here, not to installed-only observation. Run with the **installed source runtime's Python**, with Core's retained `CheckRoot.create()` and `SuppliedRuntimeHome` APIs available (Core PR #3821). Do not put the candidate Core checkout on `PYTHONPATH` and call that installed @@ -31,7 +72,8 @@ sizes are **pixels**. For the mobile48×48 →140×40 journey, supply `framebuffer=Size(1600, 1200)` to the callable factory. The owner checks actual native window bounds and rejects clipping instead of recording half a terminal. -The callable entry point is `TerminalApp.copied(source, threads, runtime, +The strict callable entry point is +`tools.physical_ui_candidate.TerminalApp.copied(source, threads, runtime, output, thread=..., size=Size(...), framebuffer=Size(...))`. Its context yields the admitted app and retains the copied root, frontend-home, operation receipts, screenshots and recording on exit. The caller still owns retiring those retained files after @@ -52,8 +94,8 @@ The copied-history factory now exercises the instrumented installed **Toad CLI** not the separate `toad-comms` shell launcher or its executable wrapper. Those entrypoints need their own evidence; this run cannot claim it. The operation module stays alive while the distribution entrypoint runs so its declared wire -types do not acquire a different `__main__` namespace. `AcpRoots` is declared -once per module, not rebuilt and registered on every observation. +types do not acquire a different `__main__` namespace. `AcpObservationDecoder` +is declared once per module, not rebuilt and registered on every observation. The startup/shutdown programs retain the bounded immutable source snapshot. Ordinary observations write only typed request/result data, not another copy of @@ -97,8 +139,8 @@ The new path is source work until actual fresh-prefix/physical execution proves ## Runtime input identity (Core #3825 dependency) -The current source requires a `RuntimeResolution` with a recorded interpreter -observation, not just a prefix and three module locations. `RunTarget` retains +Only the strict candidate member requires a `RuntimeResolution` with a recorded +interpreter observation, not just a prefix and three module locations. `RunTarget` retains that intended build. `AdmitOperation(ObserveOperation)` obtains the actual process's resolution once at frontend admission through the target-owned probe. Its `AdmittedResult` belongs to that exact frontend birth; a plain @@ -129,7 +171,7 @@ content hashing or complete loaded-code equivalence. Process/root/module-origin and saved/ACP-root invalidation remain current before every key. Current observations also carry the actual typed `StorageRule` from the already -observing service's `Registration`. `RunTarget` calls that rule's pure +observing service's `Registration`. Shared `RunResources` calls that rule's pure `require_isolated(root)` observation contract; it no longer repeats the owner's projection/equality decision. Observation neither constructs a Comms nor writes a policy. The paired Core must include this API: its `ScopedDirectory` field @@ -196,7 +238,9 @@ current host permission, notwithstanding historical approval. Replacement of that capture needs its proper source/execution admission. No heap, GC, repaint, remeasure or arbitrary trace experiment is authorized. `capture_live.py` now exposes only read-only py-spy profiling; its injection and mutation options are -deleted, not retained behind a permission flag. +deleted, not retained behind a permission flag. The mutable source mapper now +reads the declared `TranscriptReadIdentity.page_bound`; frozen inputs are not +edited and that caller correction authorizes no injection experiment. ## Append/publication app controls diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py index ab33b0c13..49a7a0f49 100644 --- a/tests/model/test_physical_run.py +++ b/tests/model/test_physical_run.py @@ -9,22 +9,30 @@ from agent_comms.installer.runtime_resolution import RuntimeResolution from agent_comms.storage_rules import StorageRule from agent_comms.nominal.field_codec import FieldCodec +from tools.physical_ui_admission import ( + AdmitOperation, AdmittedResult, RunTarget, StartAdmittedOperation, +) from tools.physical_ui_remote import ( - AdmitOperation, AdmittedResult, FrontendIdentity, ObserveOperation, ObservedResult, - PendingRun, RunObservation, RunTarget, StartOperation, StopRootOperation, + FrontendIdentity, InstalledResult, ObserveOperation, + ObservedResult, PendingRun, RunObservation, RunResources, StartOperation, StopRootOperation, ) @pytest.fixture -def physical_target(tmp_path: Path) -> RunTarget: - """Synthetic resource paths and metadata, not an application admission proof.""" +def physical_resources(tmp_path: Path) -> RunResources: root, home, runtime = (tmp_path / name for name in ("bus", "frontend-home", "runtime")) for path in (root, home, runtime): path.mkdir() + return RunResources(root, home, runtime) + + +@pytest.fixture +def physical_target(physical_resources: RunResources) -> RunTarget: + """Synthetic resource paths and metadata, not an application admission proof.""" build = RuntimeResolution( core_commit="a" * 40, packages={}, interpreter=InterpreterBuildProbe().run(), ) - return RunTarget(root, home, runtime, build) + return physical_resources.extended(RunTarget, build=build) @pytest.fixture @@ -51,14 +59,19 @@ def physical_admission( return AdmittedResult(physical_observation, physical_target.build).admitted(physical_target) -def test_startup_connection_decodes_only_observation_operations(physical_target: RunTarget) -> None: +def test_startup_connection_decodes_only_observation_operations( + physical_target: RunTarget, physical_resources: RunResources, +) -> None: identity = ProcessIdentity.current() tools = Path(__file__).resolve().parents[2] / "tools" - for operation in (ObserveOperation(physical_target, identity, tools), + for operation in (ObserveOperation(physical_resources, identity, tools), AdmitOperation(physical_target, identity, tools)): assert FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) == operation - start = StartOperation(physical_target, identity, (str(physical_target.runtime / "bin/toad"),), 9) - for operation in (start, StopRootOperation(physical_target)): + start = StartOperation(physical_resources, identity, (str(physical_target.runtime / "bin/toad"),), 9) + strict = StartAdmittedOperation( + physical_target, identity, (str(physical_target.runtime / "bin/toad"),), 9, + ) + for operation in (start, strict, StopRootOperation(physical_resources)): with pytest.raises(ValueError): FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) @@ -71,6 +84,22 @@ def test_current_observation_cannot_substitute_for_build_admission( assert physical_admission.observe(physical_target, physical_observation) == physical_observation +def test_installed_origins_are_not_candidate_proof( + physical_resources: RunResources, physical_observation: RunObservation, +) -> None: + result = InstalledResult( + physical_observation, physical_resources.runtime / "bin/python", + physical_resources.runtime / "python-image", {"sample": {"version": "1"}}, + ) + result.finish(physical_resources, result) + with pytest.raises(ValueError, match="did not verify the runtime build"): + result.admitted(physical_resources) + with pytest.raises(ValueError, match="origins changed"): + result.finish(physical_resources, replace(result, packages={"sample": {"version": "2"}})) + with pytest.raises(ValueError, match="origins changed"): + result.finish(physical_resources, replace(result, native_executable=Path("/foreign/python"))) + + def test_admission_refuses_different_build_and_frontend_birth( physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, ) -> None: diff --git a/tools/acceptance.py b/tools/acceptance.py index fe68f5a3b..5a3619aaf 100644 --- a/tools/acceptance.py +++ b/tools/acceptance.py @@ -25,9 +25,10 @@ from agent_comms.ui_model.message_category import InboundCategory, all_categories from toad.widgets.side_bar import BackTabButton -# Isolated Python omits the script directory; this tool owns its sibling imports. -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from physical_ui import OwnedDisplay, TerminalApp +# Isolated Python omits the project directory; admit this tool package only. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from tools.physical_ui import OwnedDisplay +from tools.physical_ui_candidate import TerminalApp def any_channel(screen): diff --git a/tools/performance/capture_live.py b/tools/performance/capture_live.py index 62c71ead0..3bb5346de 100644 --- a/tools/performance/capture_live.py +++ b/tools/performance/capture_live.py @@ -1,6 +1,6 @@ """Read-only py-spy capture of an explicit live PID on an isolated display. -In-process observations belong to physical_ui.TerminalApp's startup lifetime. +In-process observations belong to physical_ui.ObservedTerminalApp's startup lifetime. This command never injects frame meters, callbacks, heap changes or UI input. """ @@ -14,8 +14,8 @@ from agent_comms.child_process import ProcessIdentity from agent_comms.nominal.field_codec import FieldCodec -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from physical_ui import XvfbDisplay +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from tools.physical_ui import XvfbDisplay def main(): diff --git a/tools/performance/frame_meter.py b/tools/performance/frame_meter.py index b6f91ec54..44fe92032 100644 --- a/tools/performance/frame_meter.py +++ b/tools/performance/frame_meter.py @@ -426,7 +426,7 @@ def native_page(self, page, capture): before=self.project_value(page.before, capture), after=self.project_value(page.after, capture), root=witness.root[:384], thread=witness.requested_name[:160], - through=self.project_value(witness.through, capture), + page_bound=self.project_value(witness.page_bound, capture), thread_birth=witness.thread.incarnation.wire_birth, snapshot=capture.label(witness)) diff --git a/tools/performance/scroll_scenario.py b/tools/performance/scroll_scenario.py index 3046f594b..25683b86f 100644 --- a/tools/performance/scroll_scenario.py +++ b/tools/performance/scroll_scenario.py @@ -11,13 +11,14 @@ from pathlib import Path import sys -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from agent_comms.active_route import ActiveRoute from agent_comms.comms import wire from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime -from physical_ui import OwnedDisplay, TerminalApp -from physical_ui_remote import ObservedResult, PhysicalOperation +from tools.physical_ui import OwnedDisplay +from tools.physical_ui_candidate import TerminalApp +from tools.physical_ui_remote import ObservedResult, PhysicalOperation from textual.geometry import Size diff --git a/tools/physical_ui.py b/tools/physical_ui.py index 657061df4..963b002b6 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -10,6 +10,7 @@ from contextlib import AbstractContextManager, ExitStack, contextmanager import ctypes from dataclasses import dataclass +from functools import cache import itertools import json from multiprocessing import Pipe @@ -30,7 +31,6 @@ from agent_comms.child_process import DetachedChildStdio, ProcessIdentity, SpawnedProcess from agent_comms.comms import Comms from agent_comms.installer.practice import SuppliedRuntimeHome -from agent_comms.installer.runtime_resolution import RuntimeResolution from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime, ThreadLaunch from agent_comms.nominal.field_codec import FieldCodec @@ -43,16 +43,17 @@ from textual.strip import Strip from ui_theme.wal import WalFile -from physical_ui_remote import ( - AdmitOperation, StartOperation, ObserveOperation, PendingRun, PhysicalOperation, - ProcessRetirement, RunObservation, RunTarget, StopRootOperation, +from tools import physical_ui_remote +from tools.physical_ui_remote import ( + StartOperation, ObserveOperation, InstalledOperation, PendingRun, PhysicalOperation, + ProcessRetirement, RunObservation, RunResources, StopRootOperation, ) class XvfbDisplay(AbstractContextManager): """Own a connection to a local framebuffer, resolving aliases at Xlib. - This resource has no raw-input entrypoint. Only TerminalApp may drive its + This resource has no raw-input entrypoint. Only ObservedTerminalApp may drive its window after checking the actual application/source on that incarnation. """ @@ -209,20 +210,38 @@ def reader_position(self): return () -class TerminalApp: +@dataclass(frozen=True) +class OperationProgram: + path: Path + source: bytes + + @classmethod + def capture(cls, path: Path): + with path.open("rb") as source: + raw = source.read(PhysicalOperation.source_bytes + 1) + if len(raw) > PhysicalOperation.source_bytes: + raise ValueError("Physical operation source exceeds its bound") + return cls(path, raw) + + +class ObservedTerminalApp: """Own one isolated st window, actual-source checks and all physical input.""" startup_seconds = 60.0 stop_seconds = 10.0 - # One source snapshot belongs to the controller's Python module lifetime, - # not to each callback executing inside a foreign running application. - with Path(__file__).with_name("physical_ui_remote.py").open("rb") as _source: - operation_program = _source.read(PhysicalOperation.source_bytes + 1) - if len(operation_program) > PhysicalOperation.source_bytes: - raise ValueError("Physical operation source exceeds its bound") + start_operation = StartOperation + entry_operation = InstalledOperation + entry_record = "resources-observed.json" + final_record = "resources-final.json" + + @classmethod + @cache + def operation_programs(cls): + # Capture once when execution starts, never as an import side effect. + return (OperationProgram.capture(Path(physical_ui_remote.__file__)),) def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, - target: RunTarget, cwd: Path, *, size: Size = Size(140, 40)): + target: RunResources, cwd: Path, *, size: Size = Size(140, 40)): self.command, self.display = tuple(map(str, command)), display self.output = output.expanduser().resolve() self.target, self.cwd, self.requested_size = target, cwd, size @@ -248,8 +267,8 @@ def columns(self) -> int: def rows(self) -> int: return self.grid_size.height - @staticmethod - def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: + @classmethod + def prepare_target(cls, root: Path, runtime: LaunchRuntime) -> RunResources: """Prepare a fresh frontend home beside an already owned private root. Only presentation settings travel. Saved Toad databases/tabs never do. @@ -258,10 +277,9 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: from toad.preferences import INTERFACE, SETTINGS, SavedSettings, ToadSettings root = root.resolve(strict=True) - build = RuntimeResolution.at(runtime.prefix) - build.interpreter.require_recorded() + runtime.require_available() home = root.parent / "frontend-home" - # Validate the intended paths before touching the home. RunTarget also + # Validate the intended paths before touching the home. RunResources also # rejects the real user's home/workspace and an unrelated bus. if home.exists(): raise FileExistsError(f"A physical run needs fresh frontend persistence: {home}") @@ -273,7 +291,7 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: raise ValueError("Refusing frontend setup beside a live user root") home.mkdir(mode=0o700) SuppliedRuntimeHome(home, runtime).prepare() - target = RunTarget(root, home.resolve(), runtime.prefix, build) + target = RunResources(root, home.resolve(), runtime.prefix) for directory in (target.config, target.state, target.data): directory.mkdir(mode=0o700) settings = SavedSettings.path() @@ -304,7 +322,7 @@ def prepare_target(root: Path, runtime: LaunchRuntime) -> RunTarget: @contextmanager def copied(cls, source: Comms, threads: tuple[str, ...], runtime: LaunchRuntime, output: Path, *, thread: str, size: Size = Size(140, 40), - framebuffer: Size = OwnedDisplay.default_size) -> Iterator[TerminalApp]: + framebuffer: Size = OwnedDisplay.default_size) -> Iterator[ObservedTerminalApp]: """Create under the source owner, bind only its copy, retain run evidence.""" output = output.expanduser().absolute() output.mkdir(mode=0o700, parents=True) # Fresh evidence, never overwrite a failed run. @@ -332,7 +350,12 @@ def operation_file(cls, operation: PhysicalOperation, directory: Path) -> Path: directory.mkdir(mode=0o700, parents=True) # The declaration module owns one bounded program snapshot for this # controller lifetime. Later source edits cannot change a run's wire. - raw = cls.operation_program + programs = cls.operation_programs() + for program in programs[:-1]: + package = directory / program.path.parent.name + package.mkdir(mode=0o700, exist_ok=True) + ExclusiveAtomicFile(package / program.path.name).write_bytes(program.source) + raw = programs[-1].source script = directory / "operation.py" # Only startup/shutdown executes a program. Normal Python supplies its # filename; later observations submit data to the startup-owned reader. @@ -347,7 +370,7 @@ def result(directory: Path): return PhysicalOperation.read_result(path).require() @classmethod - def stop_copy(cls, target: RunTarget, directory: Path) -> None: + def stop_copy(cls, target: RunResources, directory: Path) -> None: script = cls.operation_file(StopRootOperation(target), directory) with SpawnedProcess.launch( (str(target.launch_runtime.interpreter), "-I", str(script)), @@ -370,7 +393,7 @@ def __enter__(self): frontend_requests, self.observer = Pipe(duplex=False) try: descriptor = frontend_requests.fileno() - script = self.operation_file(StartOperation( + script = self.operation_file(self.start_operation( self.target, ProcessIdentity.current(), self.command, descriptor), self.output / "startup") self.terminal = SpawnedProcess.launch( ("st", "-f", "DejaVu Sans Mono:pixelsize=16", @@ -403,8 +426,8 @@ def __enter__(self): self._require_starting(deadline) time.sleep(.05) self.read_grid() - self.admission = self._observe(AdmitOperation).admitted(self.target) - PhysicalOperation.publish(self.output / "build-admitted.json", self.admission) + self.entry = self.check_entry(self._observe(self.entry_operation)) + PhysicalOperation.publish(self.output / self.entry_record, self.entry) # -o enables st's raw printer too. Even these printer keys pass the # same source admission as every later physical key/click. self.keys("ctrl+Print") @@ -414,6 +437,10 @@ def __enter__(self): self.close() raise + def check_entry(self, result): + self.target.require_resources(result.observation()) + return result + def _require_starting(self, deadline: float) -> None: if self.terminal.process.poll() is not None or time.monotonic() >= deadline: raise TimeoutError(f"Physical frontend did not become ready; see {self.output}") @@ -454,7 +481,7 @@ def _observe(self, operation: type[ObserveOperation]): def observe(self) -> RunObservation: current = self._observe(ObserveOperation).observation() - return self.admission.observe(self.target, current) + return self.entry.observe(self.target, current) def input(self, *commands: str) -> RunObservation: observed = self.observe() @@ -619,11 +646,9 @@ def __exit__(self, exception_type, exception, traceback): # An exceptional/uncertain operation is not permission for another # attachment or input. Retire only the already owned child instead. if exception_type is None and self.terminal.process.poll() is None: - final = self._observe(AdmitOperation).admitted(self.target) - self.admission.observe(self.target, final.observation()) - if final.build != self.admission.build: - raise ValueError("The verified runtime build changed during the physical run") - PhysicalOperation.publish(self.output / "build-final.json", final) + final = self.check_entry(self._observe(self.entry_operation)) + self.entry.finish(self.target, final) + PhysicalOperation.publish(self.output / self.final_record, final) self.keys("ctrl+q") self.observer.close() self.terminal.process.wait(timeout=5) diff --git a/tools/physical_ui_admission.py b/tools/physical_ui_admission.py new file mode 100644 index 000000000..1488197af --- /dev/null +++ b/tools/physical_ui_admission.py @@ -0,0 +1,73 @@ +"""Strict candidate-build operations, separate from installed resource observation.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import sys + +# Only explicit execution admits this frozen tool dependency, never installed packages. +if __name__ == "__main__": + sys.path.insert(0, str(Path(__file__).parent)) + +from agent_comms.installer.probes.resolution import RuntimeResolutionProbe +from agent_comms.installer.runtime_resolution import RuntimeResolution +from tools.physical_ui_remote import ( + ObserveOperation, ObservedResult, PhysicalOperation, PhysicalResult, + RunObservation, RunResources, StartOperation, +) + + +@dataclass(frozen=True) +class RunTarget(RunResources): + build: RuntimeResolution + + def __post_init__(self): + super().__post_init__() + self.build.interpreter.require_recorded() + + def require_build(self, actual: RuntimeResolution) -> None: + self.build.require_runtime(actual) + + +@dataclass(frozen=True) +class AdmittedResult(ObservedResult): + """Build evidence for one frontend birth, not a fresh hash on every UI read.""" + + build: RuntimeResolution + + def admitted(self, target: RunResources) -> AdmittedResult: + target.require_build(self.build) + target.require_resources(self.value) + return self + + def require_build(self) -> RuntimeResolution: + return self.build + + def finish(self, target: RunResources, current: PhysicalResult) -> None: + super().finish(target, current) + actual = current.require_build() + target.require_build(actual) + if actual != self.build: + raise ValueError("The verified runtime build changed during the physical run") + + +@dataclass(frozen=True) +class StartAdmittedOperation(StartOperation): + target: RunTarget + + +@dataclass(frozen=True) +class AdmitOperation(ObserveOperation): + """Full disk-artifact verification at entry and successful retirement only.""" + + target: RunTarget + + def observed_result(self, observed: RunObservation, directory: Path) -> PhysicalResult: + build = RuntimeResolutionProbe( + self.target.build.core_commit, self.target.build.wheel_inputs() + ).run() + return AdmittedResult(observed, build).admitted(self.target) + + +if __name__ == "__main__": + PhysicalOperation.execute_file(Path(__file__)) diff --git a/tools/physical_ui_candidate.py b/tools/physical_ui_candidate.py new file mode 100644 index 000000000..8902fda0f --- /dev/null +++ b/tools/physical_ui_candidate.py @@ -0,0 +1,31 @@ +"""Candidate verification extends the installed terminal's resource lifetime.""" +from functools import cache +from pathlib import Path + +from agent_comms.installer.runtime_resolution import RuntimeResolution +from tools import physical_ui_admission +from tools.physical_ui import ObservedTerminalApp, OperationProgram +from tools.physical_ui_admission import AdmitOperation, RunTarget, StartAdmittedOperation + + +class TerminalApp(ObservedTerminalApp): + start_operation = StartAdmittedOperation + entry_operation = AdmitOperation + entry_record = "build-admitted.json" + final_record = "build-final.json" + + @classmethod + @cache + def operation_programs(cls): + return (*super().operation_programs(), + OperationProgram.capture(Path(physical_ui_admission.__file__))) + + @classmethod + def prepare_target(cls, root, runtime) -> RunTarget: + build = RuntimeResolution.at(runtime.prefix) + build.interpreter.require_recorded() + resources = super().prepare_target(root, runtime) + return resources.extended(RunTarget, build=build) + + def check_entry(self, result): + return result.admitted(self.target) diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index de2b10cc5..bbfadf4a1 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -6,6 +6,7 @@ """ from abc import abstractmethod +from contextlib import contextmanager from dataclasses import dataclass from importlib.metadata import distribution import json @@ -16,16 +17,18 @@ import sys import time from threading import Thread -from typing import ClassVar +from typing import ClassVar, Self from agent_comms.acp_extension import CoordinationChangedUpdate from agent_comms.child_process import ChildOutcome, ProcessIdentity from agent_comms.launch_runtime import LaunchRuntime -from agent_comms.installer.probes.resolution import RuntimeResolutionProbe +from agent_comms.installer.interpreter_operation import RuntimeProbe from agent_comms.installer.runtime_resolution import RuntimeResolution, ToadPackage +from agent_comms.installer.runtime_stack import Extendable from agent_comms.storage_rules import StorageRule from agent_comms.nominal.declared_family import DeclaredFamily from agent_comms.nominal.field_codec import DeclaredWireType, FieldCodec +from pydantic import JsonValue from metaclass_registry.mro_dispatch import handles from textual.geometry import Region from toad.acp.agent import Agent @@ -40,16 +43,14 @@ class PendingRun(RuntimeError): @dataclass(frozen=True) -class RunTarget: +class RunResources(Extendable): """Intended private resources, distinct from an observation of the app.""" root: Path home: Path runtime: Path - build: RuntimeResolution def __post_init__(self): - self.build.interpreter.require_recorded() for path in (self.root, self.home, self.runtime): if path != path.resolve(strict=True): raise ValueError(f"A physical run needs canonical existing paths: {path}") @@ -82,8 +83,11 @@ def state(self) -> Path: def data(self) -> Path: return self.home / "data" + def require_build(self, actual: RuntimeResolution) -> None: + raise ValueError("An installed resource observation does not certify a candidate build") + def require_resources(self, observed): - """Current UI/source resources; build evidence belongs to AdmittedResult.""" + """Current UI/source resources, independent of candidate build evidence.""" if observed.home != self.home or observed.runtime != self.runtime: raise ValueError("Observed HOME/runtime differs from the run target") if (observed.config, observed.state, observed.data) != (self.config, self.state, self.data): @@ -152,6 +156,8 @@ class RunObservation: history_regions: tuple[CapturedRegion, ...] history_published: bool modules_checked: int + agent_ready: bool = False + active_response: bool = False @dataclass(frozen=True) @@ -166,9 +172,15 @@ class PhysicalResult(DeclaredFamily, affix="Result"): def require(self): return self - def admitted(self, target: RunTarget) -> AdmittedResult: + def admitted(self, target: RunResources) -> Self: raise ValueError("This physical operation did not verify the runtime build") + def require_build(self) -> RuntimeResolution: + raise ValueError("This observation carries no candidate build proof") + + def require_origins(self, original: InstalledResult) -> None: + raise ValueError("This result did not observe installed executable and package origins") + def observation(self) -> RunObservation: raise ValueError("This physical operation did not observe an application") @@ -183,23 +195,33 @@ class ObservedResult(PhysicalResult): def observation(self) -> RunObservation: return self.value + def observe(self, target: RunResources, current: RunObservation) -> RunObservation: + if current.frontend != self.value.frontend: + raise ValueError("The admitted frontend process or module-file locations changed") + # Dynamic persistence and every saved/ACP root still precede each input. + return target.require_resources(current) + + def finish(self, target: RunResources, current: PhysicalResult) -> None: + self.observe(target, current.observation()) + @dataclass(frozen=True) -class AdmittedResult(ObservedResult): - """Build evidence for one frontend birth, not a fresh hash on every UI read.""" +class InstalledResult(ObservedResult): + """Actual installed origins, not interpreter or candidate byte verification.""" - build: RuntimeResolution + python_executable: Path + native_executable: Path + packages: JsonValue - def admitted(self, target: RunTarget) -> AdmittedResult: - target.build.require_runtime(self.build) - target.require_resources(self.value) - return self + def finish(self, target: RunResources, current: PhysicalResult) -> None: + super().finish(target, current) + current.require_origins(self) - def observe(self, target: RunTarget, current: RunObservation) -> RunObservation: - if current.frontend != self.value.frontend: - raise ValueError("The admitted frontend process or module-file locations changed") - # Dynamic persistence and every saved/ACP root still precede each input. - return target.require_resources(current) + def require_origins(self, original: InstalledResult) -> None: + if (self.python_executable != original.python_executable + or self.native_executable != original.native_executable + or self.packages != original.packages): + raise ValueError("Installed executable or package origins changed during the run") @dataclass(frozen=True) @@ -286,7 +308,7 @@ def execute_file(cls, script: Path) -> None: class StartOperation(PhysicalOperation): """Instrument a new Python frontend at startup, never an existing process.""" - target: RunTarget + target: RunResources controller: ProcessIdentity arguments: tuple[str, ...] descriptor: int @@ -343,7 +365,14 @@ def serve(self, requests: Connection, output: Path) -> None: self.publish(packet.parent / "error.json", FailedResult(type(error).__name__, str(error)[:2048])) -class AcpRoots(MroProjection[tuple[Path, ...]]): +@dataclass(frozen=True) +class AcpObservation: + roots: tuple[Path, ...] + ready: bool = False + active: bool = False + + +class AcpObservationDecoder(MroProjection[AcpObservation]): """One foreign-connection decoder, not a new handler class per observation.""" @handles(object) @@ -356,24 +385,34 @@ def pending(self, value): @handles(NoAgentConnection) def unbound(self, value): - return () + return AcpObservation(()) @handles(Agent) def bound(self, value): - return self.project_value(value.coordination) + source = self.project_value(value.coordination) + return AcpObservation(source.roots, value.ready, value.current_turn.busy) @handles(CoordinationChangedUpdate) def coordinated(self, value): - return (Path(value.wire_root).resolve(),) + return AcpObservation((Path(value.wire_root).resolve(),)) @dataclass(frozen=True) class ObserveOperation(PhysicalOperation): - target: RunTarget + target: RunResources identity: ProcessIdentity tools: Path - def observed_result(self, observed: RunObservation) -> PhysicalResult: + @contextmanager + def tool_imports(self): + previous = sys.path[:] + try: + sys.path.insert(0, str(self.tools)) + yield + finally: + sys.path[:] = previous + + def observed_result(self, observed: RunObservation, directory: Path) -> PhysicalResult: return ObservedResult(observed) def execute(self, directory: Path) -> None: @@ -384,12 +423,8 @@ def execute(self, directory: Path) -> None: raise ValueError("Observed HOME is not private") # The existing reviewed validator remains the single module-location policy. # The temporary tool import path is retired before inspecting application state. - previous = sys.path[:] - try: - sys.path.insert(0, str(self.tools)) + with self.tool_imports(): from performance.history_capture_remote import require_modules - finally: - sys.path[:] = previous import agent_comms import textual import toad @@ -415,10 +450,11 @@ def execute(self, directory: Path) -> None: sessions = app.session_navigation saved = tuple(Path(source).resolve() for entry in sessions.members for source, _identity in (*entry.original_threads(sessions), *entry.original_channels())) - decoder = AcpRoots() - acp = tuple(source for process in dict.fromkeys(sessions.processes) - for agent in (process.agent, *(session.agent for session in process.sessions)) - for source in decoder.project_value(agent)) + decoder = AcpObservationDecoder() + acp_states = tuple(decoder.project_value(agent) + for process in dict.fromkeys(sessions.processes) + for agent in (process.agent, *(session.agent for session in process.sessions))) + acp = tuple(source for state in acp_states for source in state.roots) # This is geometry of actually published history windows, not a guess # from sidebar separators that may also occur inside a user's message. regions = tuple(CapturedRegion(region) for window in screen.query(HistoryWindow) @@ -440,27 +476,34 @@ def execute(self, directory: Path) -> None: xdg_data_home().resolve(), app.tab_order._db.path.resolve(), app.colours.wal_directory.resolve(), root, opened, storage_rule, saved, acp, regions, history_published, checked, + agent_ready=bool(acp_states) and all(state.ready for state in acp_states), + active_response=any(state.active for state in acp_states), ) self.target.require_resources(observed) - self.publish(directory / "result.json", self.observed_result(observed)) + self.publish(directory / "result.json", self.observed_result(observed, directory)) @dataclass(frozen=True) -class AdmitOperation(ObserveOperation): - """Full disk-artifact verification at entry and successful retirement only.""" - - def observed_result(self, observed: RunObservation) -> PhysicalResult: - build = RuntimeResolutionProbe( - self.target.build.core_commit, self.target.build.wheel_inputs() - ).run() - return AdmittedResult(observed, build).admitted(self.target) +class InstalledOperation(ObserveOperation): + def observed_result(self, observed: RunObservation, directory: Path) -> PhysicalResult: + recorded = RuntimeResolution.at(self.target.runtime) + # This is the installed runtime's stable wire operation, not an import + # of a concrete probe from a possibly different source-module layout. + probe = FieldCodec.decode(RuntimeProbe, { + "kind": "installed_packages", "core": recorded.core_commit, + }) + packages = FieldCodec.encode(probe.run(), probe.answer_type()) + return InstalledResult( + observed, Path(sys.executable).resolve(strict=True), + Path("/proc/self/exe").resolve(strict=True), packages, + ) @dataclass(frozen=True) class StopRootOperation(PhysicalOperation): """Stop only the private copy, in its own declared runtime; retain its files.""" - target: RunTarget + target: RunResources def execute(self, directory: Path) -> None: if LaunchRuntime.running().prefix != self.target.runtime: From 5bbc58e9ec3526a2e1bc7fe1a7f5fdf0e8234b0f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 03:01:34 -0500 Subject: [PATCH 21/35] Record installed capture08/09 limits and caller-owned lease --- docs/performance/physical-run.md | 6 +++-- docs/performance/scroll-series.md | 42 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 84afefddf..3273bcb08 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -12,8 +12,10 @@ on a display. Import `tools.physical_ui.ObservedTerminalApp` from the published source checkpoint with the installed interpreter. Its existing `copied(source, threads, runtime, -output, thread=...)` factory owns the same `CheckRoot`, private home, lease/display, -startup connection and input checks. Use `LaunchRuntime.installed()` for installed +output, thread=...)` factory owns the same `CheckRoot`, private home, display, +startup connection and input checks. The caller holds `TestLease.current().held()` +around that factory; the factory does not acquire the host lease itself. +Use `LaunchRuntime.installed()` for installed bytes. This does not require an interpreter observation in old build metadata. Wait for both `screen.observation.history_published` and `screen.observation.agent_ready` before the readiness-dependent journey; an active diff --git a/docs/performance/scroll-series.md b/docs/performance/scroll-series.md index 6f00bb0fa..2feda9f87 100644 --- a/docs/performance/scroll-series.md +++ b/docs/performance/scroll-series.md @@ -5,6 +5,48 @@ source/traffic scopes. The unsafe shell controller and raw display-input API are retired. Use the [shared isolated physical-run owner](physical-run.md) for new work; its stopped-history scenario is not the old active-response benchmark. +## 2026-11-02 installed observation attempts 08/09 — not scrolling results + +Both attempts used the read-only `observer-head-53e04c515` tool archive and +installed Core `038c8f657`, not candidate package overlays. The command launched +the installed Toad CLI; it did not exercise the `toad-comms` shell launcher. +The caller, not the frontend factory, held the existing host `TestLease`. + +- **08:** the copied history's project was not a Git checkout. + `LaunchCheckout.checked` refused before any frontend, fork, input or profiling. + The private copy retired; no guard was weakened or repository manufactured. +- **09:** the existing goal-owned prepared Toad checkout passed qualification. + A fresh private fork reached visibly loaded history and Ready. Its new prompt + was submitted once: input `fc3ca84a66414cf595de16f8da929a65`, turn + `fb0955e420bc49148a352628315ac2ba`. ACP recorded `identified_input_started`, + and the retained native session contains that user message. No replay occurred. + The provider WebSocket connected at 347 ms and its first stream event arrived + at 742 ms **from native request preparation**, not from physical input or paint. + These events establish request progress, not response text. The retained ACP + log and 164-record session contain no new assistant text; the active-response + predicate failed after 75 s, before any scroll gestures or profiler launch. + The missing response's cause remains unestablished. Retirement also logged a + cancellation followed by `RelationViolationError: An authorized original turn + is required` at `owned_turn.py:547`; that separate cleanup failure does not + establish why no answer appeared before the deadline. + +For09, retained process-birth records show st stopped by TERM, Xvfb exited0 and +recording exited0; `result.json` records private-copy retirement. The controller's +lease ended with its context; this does not prove global lease availability. +Video/screen snapshots are retained, but there is **no frame/input-to-paint +latency distribution, held-scroll acceptance or representative CPU result**. +Input-start evidence does not establish a completed response; its final outcome +remains uncertain and the original input is not eligible for replay. + +Artifacts: `~/.cache/agent-scratch/pi-ui-performance/installed-active-038-{08,09}/` +and adjacent controller logs/return codes.09 includes `frontend/loaded-ready.png`, +`frontend/active-response-failed.png`, the complete ACP log under +`frontend-home/state/toad/logs/`, and native request/session records under `bus/`. +The immutable archive remains borrowed by UI; these records do not replace the +configured active-answer/held-PageUp/default-launcher journey still owed. + +## Historical series + One line per change, from `tools/performance/scroll_scenario.sh` (real `toad` in `st` on display :121, a fork of `perf-base-20261009`, which is a fork of the saved `agent-comms-ux` session; an active response, held PageUp from the input box, wheel bursts with reversal, End, tab return). Frame work is UI-thread time per frame handed to the driver; input-to-paint runs from the driver posting an event to the first frame after its last handler returns. Input-to-paint pairs each input with the first frame after it is handled (fixed 2026-10-09; earlier summaries paired out of order). Raw results are under `~/.cache/agent-scratch/scroll-series//`. | Date | Change | Total frame median (ms) | Total frame p95 (ms) | Input-to-paint median (ms) | Input-to-paint p95 (ms) | Scroll-path lines +/- | What the screen showed | From 3f123d4a7457d2f2651e6cee8765186216e1afdb Mon Sep 17 00:00:00 2001 From: pi-ui-performance Date: Mon, 2 Nov 2026 05:03:10 -0500 Subject: [PATCH 22/35] Qualify active replies from their source and published body --- docs/performance/physical-run.md | 6 +++++- tests/model/test_physical_run.py | 33 ++++++++++++++++++++++++++++++-- tools/physical_ui.py | 17 ++++++++++++++-- tools/physical_ui_remote.py | 32 ++++++++++++++++++++++++++++--- 4 files changed, 80 insertions(+), 8 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 3273bcb08..cff2af3aa 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -19,7 +19,11 @@ Use `LaunchRuntime.installed()` for installed bytes. This does not require an interpreter observation in old build metadata. Wait for both `screen.observation.history_published` and `screen.observation.agent_ready` before the readiness-dependent journey; an active -capture additionally needs `active_response` and actual response text/focus. +capture additionally needs `screen.has_active_reply(marker)` and composer focus. +`turn_busy` only reports the backend phase; it does not prove that an answer exists. +The reply check requires both the current `ResponseStream`'s source text and matching +real st cells inside that block's last published region. A user echo, Thinking text, +a saved row or a completed notice cannot substitute for that live reply. Entry/final `InstalledOperation` records the actual Python/native executable paths and the installed runtime's package-origin answer through its existing diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py index 49a7a0f49..b404e221d 100644 --- a/tests/model/test_physical_run.py +++ b/tests/model/test_physical_run.py @@ -3,17 +3,21 @@ from pathlib import Path import pytest +from textual.geometry import Region + +from tools.physical_ui import ScreenCapture from agent_comms.child_process import ProcessIdentity from agent_comms.installer.probes.interpreter import InterpreterBuildProbe from agent_comms.installer.runtime_resolution import RuntimeResolution +from agent_comms.installer.refusals import UpgradeRefused from agent_comms.storage_rules import StorageRule from agent_comms.nominal.field_codec import FieldCodec from tools.physical_ui_admission import ( AdmitOperation, AdmittedResult, RunTarget, StartAdmittedOperation, ) from tools.physical_ui_remote import ( - FrontendIdentity, InstalledResult, ObserveOperation, + CapturedRegion, CapturedResponse, FrontendIdentity, InstalledResult, ObserveOperation, ObservedResult, PendingRun, RunObservation, RunResources, StartOperation, StopRootOperation, ) @@ -104,7 +108,7 @@ def test_admission_refuses_different_build_and_frontend_birth( physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, ) -> None: different = physical_target.build.model_copy(update={"core_commit": "b" * 40}) - with pytest.raises(ValueError): + with pytest.raises(UpgradeRefused): AdmittedResult(physical_observation, different).admitted(physical_target) original = physical_observation.frontend replacement = replace(original, process=replace(original.process, pid=original.process.pid + 1)) @@ -154,6 +158,31 @@ def test_actual_source_policy_is_checked_after_admission( physical_admission.observe(physical_target, current) +@pytest.mark.parametrize("busy, source, painted, expected", ( + (True, None, "Thinking", False), + (True, "reply-marker", "Thinking", False), + (True, "different reply", "reply-marker", False), + (False, "reply-marker", "reply-marker", False), + (True, "reply-marker", "reply-marker", True), +)) +def test_active_reply_needs_its_own_source_and_painted_cells( + physical_observation: RunObservation, busy: bool, source: str | None, + painted: str, expected: bool, +) -> None: + responses = (() if source is None else ( + CapturedResponse(CapturedRegion(Region(0, 1, 40, 1)), source), + )) + observed = replace(physical_observation, turn_busy=busy, responses=responses) + # The user's visible echo cannot satisfy the separate reply placement. + screen = ScreenCapture("user: reply-marker\n" + painted, observed) + assert screen.has_active_reply("reply-marker") is expected + + +def test_active_reply_rejects_an_empty_marker(physical_observation: RunObservation) -> None: + with pytest.raises(ValueError, match="nonempty marker"): + ScreenCapture("", physical_observation).has_active_reply("") + + def test_source_readiness_is_current_not_remembered_admission( physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, ) -> None: diff --git a/tools/physical_ui.py b/tools/physical_ui.py index 963b002b6..21bee6d4b 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -182,13 +182,26 @@ class ScreenCapture: text: str observation: RunObservation - def message_rows(self): + def rows_in(self, regions: tuple[physical_ui_remote.CapturedRegion, ...]): lines = self.text.splitlines() - for captured in self.observation.history_regions: + for captured in regions: region = captured.region for row in range(max(0, region.y), min(len(lines), region.bottom)): yield row, Strip([Segment(lines[row])]).crop(region.x, region.right).text + def message_rows(self): + return self.rows_in(self.observation.history_regions) + + def has_active_reply(self, marker: str) -> bool: + """Require source-owned reply text and matching real st cells, not an echo.""" + if not marker: + raise ValueError("Active reply qualification requires a nonempty marker") + return self.observation.turn_busy and any( + marker in response.text + and marker in "\n".join(text for _, text in self.rows_in((response.region,))) + for response in self.observation.responses + ) + def messages(self): message = None for _, cell in self.message_rows(): diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index bbfadf4a1..18ea19002 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -136,6 +136,14 @@ class FrontendIdentity: textual_module: Path +@dataclass(frozen=True) +class CapturedResponse: + """An active reply's source text and last published native placement.""" + + region: CapturedRegion + text: str + + @dataclass(frozen=True) class RunObservation: frontend: FrontendIdentity @@ -157,7 +165,8 @@ class RunObservation: history_published: bool modules_checked: int agent_ready: bool = False - active_response: bool = False + turn_busy: bool = False + responses: tuple[CapturedResponse, ...] = () @dataclass(frozen=True) @@ -369,7 +378,7 @@ def serve(self, requests: Connection, output: Path) -> None: class AcpObservation: roots: tuple[Path, ...] ready: bool = False - active: bool = False + busy: bool = False class AcpObservationDecoder(MroProjection[AcpObservation]): @@ -431,6 +440,8 @@ def execute(self, directory: Path) -> None: from toad.app import ToadApp from textual._context import NoActiveAppError from textual.app import ScreenStackError + from toad.live_output import ResponseStream + from toad.widgets.conversation import Conversation from toad.widgets.history_anchor import HistoryWindow from xdg_base_dirs import xdg_config_home, xdg_data_home, xdg_state_home @@ -462,6 +473,20 @@ def execute(self, directory: Path) -> None: histories = tuple(history for window in screen.query(HistoryWindow) if window.published_content_region is not None for history in window.histories) + # Decode the installed frontend at this read-only boundary. Only the + # current live ResponseStream supplies reply text: not a user-input + # echo, a ThoughtStream, a saved row or an earlier completed notice. + published = screen._compositor._published_map + responses = tuple( + CapturedResponse( + CapturedRegion(placement.content_region.intersection(placement.clip)), + block.body.text, + ) + for view in screen.query(Conversation) if view.turns.owner.busy + for stream in (view.output.streams.get(ResponseStream),) if stream is not None + for block in (stream.block,) if block is not None + for placement in (published.get(block.body),) if placement is not None + ) # The pager's original publication state also supplies the footer's # saved-history availability. Agent status is an independent fact. history_published = bool(histories) and all( @@ -477,7 +502,8 @@ def execute(self, directory: Path) -> None: app.colours.wal_directory.resolve(), root, opened, storage_rule, saved, acp, regions, history_published, checked, agent_ready=bool(acp_states) and all(state.ready for state in acp_states), - active_response=any(state.active for state in acp_states), + turn_busy=any(state.busy for state in acp_states), + responses=responses, ) self.target.require_resources(observed) self.publish(directory / "result.json", self.observed_result(observed, directory)) From 7c80d9b0557ce44c40791de9ef4a5cf8394d25cf Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 05:54:49 -0500 Subject: [PATCH 23/35] Compose frame measurement with the existing startup and admission lifetime --- docs/performance/physical-run.md | 52 +++++++++++++++++++++++++++----- tests/model/test_physical_run.py | 8 ++++- tools/performance/frame_meter.py | 5 +-- tools/physical_ui.py | 24 +++++++++++++-- tools/physical_ui_admission.py | 7 ++++- tools/physical_ui_candidate.py | 12 ++++++-- tools/physical_ui_remote.py | 49 ++++++++++++++++++++++++++++-- 7 files changed, 139 insertions(+), 18 deletions(-) diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index cff2af3aa..52184b2db 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -19,7 +19,8 @@ Use `LaunchRuntime.installed()` for installed bytes. This does not require an interpreter observation in old build metadata. Wait for both `screen.observation.history_published` and `screen.observation.agent_ready` before the readiness-dependent journey; an active -capture additionally needs `screen.has_active_reply(marker)` and composer focus. +capture additionally needs `screen.has_active_reply(marker)` and the actual +`screen.observation.composer_focused` result, not an assumed click coordinate. `turn_busy` only reports the backend phase; it does not prove that an answer exists. The reply check requires both the current `ResponseStream`'s source text and matching real st cells inside that block's last published region. A user echo, Thinking text, @@ -43,14 +44,34 @@ Data/source qualification: `observer-installed-06.log` decoded three staged requests on the installed interpreter without loading the new candidate probe; `observer-candidate-06.log` decoded three strict-member requests on the existing immutable0402/3475 interpreter. Neither executes a frontend, build probe or input. -The source map covers Core src, Toad src/tools and the physical-run consumer: -971 modules, 8,192 raw classes, 8,184 canonical projections, zero parse omissions. +The historical context-free source map listed Core src, Toad src/tools and the +physical-run consumer:971 modules,8,192 raw classes and8,184 projected rows. +Those counts do not establish canonical binding closure through declared import +roots; that stronger claim is withdrawn and the rooted reconstruction stays OPEN. Dynamic imports/aliases, descriptor/effect behavior, foreign platforms, physical movement and mounted readiness remain OPEN. A separate `observer-result-07.log` on installed Core038c checks the result codec/origin continuity and changed- executable refusal using an explicitly synthetic UI record and the real package answer. It is not a mounted observation. No author pytest ran. +## Launch-time frame measurement + +`MeasuredTerminalApp(ObservedTerminalApp)` adds the existing bounded frame meter +through `ApplicationLifetime.observe_next_start` before the saved session opens. +Its `MeasuredStartOperation` owns the meter until that actual distribution +entrypoint exits. It snapshots the meter source beside the startup program and +restores its hooks through the same bounded capture's closer. The read-only +observation pipe still accepts only `ObserveOperation` descendants; measurement +cannot be installed by sending an executable packet to a running app. + +`MeasuredCandidateTerminalApp(TerminalApp, MeasuredTerminalApp)` composes the same +measurement with strict candidate admission. It does not create another frontend, +resource lifetime or admission implementation. Output is `frontend/frame-meter.json`; +record caps, censored spans and restoration status must be read before reporting +numbers. Setup/startup records are not the fixed scrolling scenario. Driver-handoff +measurements are not proof of terminal pixels; retain the physical recording and +actual response qualification with the input spans. + ## Candidate/build-verifying entry point This member requires Core3825's recorded-interpreter/probe APIs; that requirement @@ -85,7 +106,20 @@ retains the copied root, frontend-home, operation receipts, screenshots and recording on exit. The caller still owns retiring those retained files after acceptance and actual borrower checks. -## Startup observation (current source, not yet run) +## Startup observation + +Fresh qualification10 used published467 observer code with installed Core f03, +loaded the copied saved history and reached ACP Ready without provider input. +The new `turn_busy` and response-placement fields decoded in the real frontend; +its private copy, st and Xvfb retired successfully, controller rc0. + +Qualification11 added the launch-time meter on that installed application. It +recorded83 display handoffs and4 physical input arrivals (3 followed by a handoff), +restored all10 hooks, dropped/censored no records and retired the private run, +rc0. Its109,328-byte capture covers startup and three paging keys, **not** the +fixed active-answer scenario or1408. Evidence is retained under +`pi-ui-performance/installed-observer-f03-10` and `installed-meter-f03-11` in the +host scratch owner. No provider prompt was sent in either qualification. `StartOperation` runs the installed distribution's declared Toad entrypoint in the new frontend interpreter. One private inherited pipe belongs to that exact @@ -226,10 +260,12 @@ visible pixels. Neither an import nor one screenshot proves the whole journey. A run with no ACP bindings does not demonstrate the bound/pending/foreign ACP cases. -The new admission/resource-lifetime split has written model controls for build -admission versus ordinary observation, changed build/process/resources/module -paths, new saved/ACP roots, actual storage policy and source readiness after admission. These controls -are not yet executed. The source/codec checks and eventual candidate preparation +The admission/resource-lifetime split has model controls for build admission +versus ordinary observation, changed build/process/resources/module paths, new +saved/ACP roots, actual storage policy and readiness after admission. Run14 +executed25/25 Toad cases; run15 executed25/25 after adding measured-startup codec +round trips and read-only-pipe rejection. These are source-model controls, not +candidate or physical scenario acceptance. The source/codec checks and eventual candidate preparation must not be reported as physical or performance acceptance. Source-observer work is part of the measurement overhead. For example, one diff --git a/tests/model/test_physical_run.py b/tests/model/test_physical_run.py index b404e221d..98bb20e32 100644 --- a/tests/model/test_physical_run.py +++ b/tests/model/test_physical_run.py @@ -15,10 +15,12 @@ from agent_comms.nominal.field_codec import FieldCodec from tools.physical_ui_admission import ( AdmitOperation, AdmittedResult, RunTarget, StartAdmittedOperation, + MeasuredStartAdmittedOperation, ) from tools.physical_ui_remote import ( CapturedRegion, CapturedResponse, FrontendIdentity, InstalledResult, ObserveOperation, ObservedResult, PendingRun, RunObservation, RunResources, StartOperation, StopRootOperation, + MeasuredStartOperation, ) @@ -75,7 +77,11 @@ def test_startup_connection_decodes_only_observation_operations( strict = StartAdmittedOperation( physical_target, identity, (str(physical_target.runtime / "bin/toad"),), 9, ) - for operation in (start, strict, StopRootOperation(physical_resources)): + measured = MeasuredStartOperation(physical_resources, identity, start.arguments, 9) + measured_strict = MeasuredStartAdmittedOperation(physical_target, identity, strict.arguments, 9) + for operation in (start, strict, measured, measured_strict): + assert FieldCodec.decode(StartOperation, FieldCodec.encode(operation)) == operation + for operation in (start, strict, measured, measured_strict, StopRootOperation(physical_resources)): with pytest.raises(ValueError): FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) diff --git a/tools/performance/frame_meter.py b/tools/performance/frame_meter.py index 44fe92032..afd3c9d0d 100644 --- a/tools/performance/frame_meter.py +++ b/tools/performance/frame_meter.py @@ -1,6 +1,7 @@ """Total UI-thread work per frame and input-to-paint latency in a live Textual app. -``install`` runs inside the application (through ``capture_live.py --frame-meter``). +``install`` runs inside the application through its normal-user startup hook +(``physical_ui.MeasuredTerminalApp``), never through live-process injection. Every event-loop callback is timed; a frame is the UI-thread work done since the previous frame was handed to the driver. An input event is timed from the moment the driver posts it to the App until the first frame handed over after its last @@ -644,7 +645,7 @@ def counted_bindings(self): return refresh_bindings(self) def timed_run(handle): - if not capture.accepting: + if not capture.accepting or handle._loop is not capture.loop: return run(handle) state['current'] = begin = clock() cpu_begin = time.thread_time_ns() diff --git a/tools/physical_ui.py b/tools/physical_ui.py index 21bee6d4b..ab7c63151 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -25,7 +25,7 @@ import time from typing import get_args, get_type_hints -from agent_comms.active_route import ActiveRoute, active_route_path, route_new_bus +from agent_comms.active_route import ActiveRoute, active_route_path, route_new_bus, resolve_comms_route from agent_comms.bus import Bus from agent_comms.check_root import CheckRoot from agent_comms.child_process import DetachedChildStdio, ProcessIdentity, SpawnedProcess @@ -34,6 +34,7 @@ from agent_comms.launch_root import LaunchRoot from agent_comms.launch_runtime import LaunchRuntime, ThreadLaunch from agent_comms.nominal.field_codec import FieldCodec +from agent_comms.native_arguments import ConfiguredBinary, NativeArguments from agent_comms.recorded_bus import RecordedBusIdentity from agent_comms.root_runtime import RootRuntime from agent_comms.store_files import ExclusiveAtomicFile @@ -45,7 +46,7 @@ from tools import physical_ui_remote from tools.physical_ui_remote import ( - StartOperation, ObserveOperation, InstalledOperation, PendingRun, PhysicalOperation, + StartOperation, MeasuredStartOperation, ObserveOperation, InstalledOperation, PendingRun, PhysicalOperation, ProcessRetirement, RunObservation, RunResources, StopRootOperation, ) @@ -259,6 +260,12 @@ def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, self.output = output.expanduser().resolve() self.target, self.cwd, self.requested_size = target, cwd, size environment = LaunchRoot.environment(ThreadLaunch.strip(dict(os.environ))) + # The source controller's absolute launcher is not the target's launch + # identity. Retarget through the root's existing owner policy, exactly + # as a prepared owner restart does; native validation stays unchanged. + environment[NativeArguments.binary_key] = resolve_comms_route(target.root).launch().restart_entrypoint( + ConfiguredBinary().configured(environment), target.launch_runtime, + ) for key in tuple(environment): if key.startswith("XDG_") or key in ("PYTHONPATH", "PYTHONHOME", "NO_COLOR", "TOAD_VALIDATION"): del environment[key] @@ -667,3 +674,16 @@ def __exit__(self, exception_type, exception, traceback): self.terminal.process.wait(timeout=5) finally: self.close() + + +class MeasuredTerminalApp(ObservedTerminalApp): + """The same terminal/source lifetime with an opt-in startup frame meter.""" + + start_operation = MeasuredStartOperation + + @classmethod + @cache + def operation_programs(cls): + programs = super().operation_programs() + meter = OperationProgram.capture(Path(__file__).parent / "performance/frame_meter.py") + return (*programs[:-1], meter, programs[-1]) diff --git a/tools/physical_ui_admission.py b/tools/physical_ui_admission.py index 1488197af..482902cda 100644 --- a/tools/physical_ui_admission.py +++ b/tools/physical_ui_admission.py @@ -13,7 +13,7 @@ from agent_comms.installer.runtime_resolution import RuntimeResolution from tools.physical_ui_remote import ( ObserveOperation, ObservedResult, PhysicalOperation, PhysicalResult, - RunObservation, RunResources, StartOperation, + RunObservation, RunResources, StartOperation, MeasuredStartOperation, ) @@ -56,6 +56,11 @@ class StartAdmittedOperation(StartOperation): target: RunTarget +@dataclass(frozen=True) +class MeasuredStartAdmittedOperation(MeasuredStartOperation, StartAdmittedOperation): + """Compose strict candidate startup with the existing frame measurement.""" + + @dataclass(frozen=True) class AdmitOperation(ObserveOperation): """Full disk-artifact verification at entry and successful retirement only.""" diff --git a/tools/physical_ui_candidate.py b/tools/physical_ui_candidate.py index 8902fda0f..b819bdbed 100644 --- a/tools/physical_ui_candidate.py +++ b/tools/physical_ui_candidate.py @@ -4,8 +4,10 @@ from agent_comms.installer.runtime_resolution import RuntimeResolution from tools import physical_ui_admission -from tools.physical_ui import ObservedTerminalApp, OperationProgram -from tools.physical_ui_admission import AdmitOperation, RunTarget, StartAdmittedOperation +from tools.physical_ui import MeasuredTerminalApp, ObservedTerminalApp, OperationProgram +from tools.physical_ui_admission import ( + AdmitOperation, RunTarget, StartAdmittedOperation, MeasuredStartAdmittedOperation, +) class TerminalApp(ObservedTerminalApp): @@ -29,3 +31,9 @@ def prepare_target(cls, root, runtime) -> RunTarget: def check_entry(self, result): return result.admitted(self.target) + + +class MeasuredCandidateTerminalApp(TerminalApp, MeasuredTerminalApp): + """Candidate admission and measurement share one terminal/resource lifetime.""" + + start_operation = MeasuredStartAdmittedOperation diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index 18ea19002..fc6effccb 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -6,7 +6,8 @@ """ from abc import abstractmethod -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager +from functools import partial from dataclasses import dataclass from importlib.metadata import distribution import json @@ -167,6 +168,9 @@ class RunObservation: agent_ready: bool = False turn_busy: bool = False responses: tuple[CapturedResponse, ...] = () + composer_focused: bool = False + composer_regions: tuple[CapturedRegion, ...] = () + active_mode: str = "" @dataclass(frozen=True) @@ -345,7 +349,13 @@ def execute(self, directory: Path) -> None: sys.argv = list(self.arguments) # Keep this declaration module alive while the installed entrypoint # runs; replacing __main__ would change the codec's type namespace. - declared.load()() + with self.startup_observed(directory): + declared.load()() + + @contextmanager + def startup_observed(self, directory: Path): + """Optional launch-time instrumentation shares the entrypoint lifetime.""" + yield def serve(self, requests: Connection, output: Path) -> None: while True: @@ -374,6 +384,33 @@ def serve(self, requests: Connection, output: Path) -> None: self.publish(packet.parent / "error.json", FailedResult(type(error).__name__, str(error)[:2048])) +class MeasuredStartOperation(StartOperation): + """Frame measurement is installed by the application's one-shot startup hook. + + The same capability composes with installed observation and strict candidate + admission. It adds no write operation to the running observation connection. + """ + + @contextmanager + def startup_observed(self, directory: Path): + from toad.application_lifetime import ApplicationLifetime + + with super().startup_observed(directory), ExitStack() as cleanup: + ApplicationLifetime.observe_next_start(partial(self.measure, directory, cleanup)) + yield + + def measure(self, directory: Path, cleanup: ExitStack, lifetime) -> None: + from performance.frame_meter import install + + # install resolves and validates every target before changing any hook; + # its returned closer restores them and publishes bounded output. + cleanup.callback(install( + expected_pid=os.getpid(), seconds=240, + output=directory.parent / "frame-meter.json", + max_records=200000, max_bytes=16 << 20, + )) + + @dataclass(frozen=True) class AcpObservation: roots: tuple[Path, ...] @@ -443,6 +480,7 @@ def execute(self, directory: Path) -> None: from toad.live_output import ResponseStream from toad.widgets.conversation import Conversation from toad.widgets.history_anchor import HistoryWindow + from toad.widgets.prompt import PromptTextArea from xdg_base_dirs import xdg_config_home, xdg_data_home, xdg_state_home try: @@ -504,6 +542,13 @@ def execute(self, directory: Path) -> None: agent_ready=bool(acp_states) and all(state.ready for state in acp_states), turn_busy=any(state.busy for state in acp_states), responses=responses, + composer_focused=any(editor.has_focus for editor in screen.query(PromptTextArea)), + composer_regions=tuple( + CapturedRegion(placement.content_region.intersection(placement.clip)) + for editor in screen.query(PromptTextArea) + for placement in (published.get(editor),) if placement is not None + ), + active_mode=app.current_mode, ) self.target.require_resources(observed) self.publish(directory / "result.json", self.observed_result(observed, directory)) From 132aea3630da35ed2fea2d78eeb433a3710d6da1 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 07:01:55 -0500 Subject: [PATCH 24/35] Use the declared transcript exception in source rewrite controls --- tests/textual/test_transcript_presentation_fencing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 0590fcce2..bbd6b44ad 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -275,7 +275,7 @@ async def test_closed_same_source_commit_acquires_complete_coverage(workspace, f @pytest.mark.parametrize("following", (False, True)) async def test_rewritten_saved_file_replaces_paint_even_when_its_cursor_grew(workspace, following): from dataclasses import replace - from agent_comms.errors import TranscriptChanged + from agent_comms.coordination_errors import TranscriptChanged thread = workspace.thread("rewritten-source") journal = write_journal(thread.project_path / "rewritten.jsonl", From 874dc528e02425d2928eaa53f61a94b3cfc0e0db Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 07:08:44 -0500 Subject: [PATCH 25/35] Derive transcript preparation identity from the declared content view --- src/toad/transcript_preparation.py | 6 ++++++ tests/textual/test_transcript_presentation_fencing.py | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/toad/transcript_preparation.py b/src/toad/transcript_preparation.py index 9ddd1ca8f..c00e81e95 100644 --- a/src/toad/transcript_preparation.py +++ b/src/toad/transcript_preparation.py @@ -11,6 +11,7 @@ from weakref import ReferenceType, ref from agent_comms.historical_views import ConversationObservation +from agent_comms.nominal.field_codec import ContentExclude, FieldCodec from agent_comms.sent_message import OutgoingMessageReference from agent_comms.transcripts import ( TranscriptCursor, @@ -424,6 +425,11 @@ def fragments_for(self, page: PreparedTranscriptPage): class TranscriptPageRenderTask(ReusableRenderTask[PreparedTranscriptPage]): page: TranscriptPage + @property + def preparation_inputs(self) -> object: + """Read receipts and coordination activity cannot change prepared content.""" + return FieldCodec.project(self.page, ContentExclude) + def result_size(self, result: PreparedTranscriptPage) -> int: return result.retained_bytes diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index bbd6b44ad..bbcfaa1d3 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -297,10 +297,10 @@ async def test_rewritten_saved_file_replaces_paint_even_when_its_cursor_grew(wor "REPLACEMENT_SOURCE_ROW " * 80, id="new")) def read_replacement(): - read = workspace.comms.transcripts.capture_page_read("rewritten-source") with pytest.raises(TranscriptChanged): - read.read(position=original.later_page()) - return read.read() + workspace.comms.transcripts.capture_page_read( + "rewritten-source", position=original.later_page()).read() + return workspace.comms.transcripts.capture_page_read("rewritten-source").read() replacement = await asyncio.to_thread(read_replacement) assert replacement.after.contains(original) From fdafb776cfbf85baa1dbdc89eb63c39a3caf634f Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 07:23:08 -0500 Subject: [PATCH 26/35] Retain transcript suppliers through the existing preparation owner --- src/toad/render_backend.py | 9 ++++++++- src/toad/transcript_preparation.py | 8 +++++--- tests/textual/test_saved_history_coverage.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/toad/render_backend.py b/src/toad/render_backend.py index 323caab6b..b9b8ed232 100644 --- a/src/toad/render_backend.py +++ b/src/toad/render_backend.py @@ -97,7 +97,9 @@ async def preparation_identity(self, work: RenderPreparation, runtime: Preparati return WorkKey(type(work), UniqueRevision(), work.scope) -class ReusableRenderTask(SerializedStorage[ResultT], RenderTask[ResultT]): +class ContentAddressedRenderTask(RenderTask[ResultT]): + """Reusable input identity, independent of the result's storage declaration.""" + async def preparation_identity(self, work: RenderPreparation, runtime: PreparationRuntime) -> WorkKey: from dataclasses import replace @@ -105,6 +107,11 @@ async def preparation_identity(self, work: RenderPreparation, runtime: Preparati return replace(await ContentAddressedWork.identity(work, runtime), scope=work.scope) + +class ReusableRenderTask(SerializedStorage[ResultT], ContentAddressedRenderTask[ResultT]): + """Content-addressed work whose independent deliveries decode serialized data.""" + + class RendererSpawn(ABC): @staticmethod def start_workers(executor) -> None: diff --git a/src/toad/transcript_preparation.py b/src/toad/transcript_preparation.py index c00e81e95..9d2615322 100644 --- a/src/toad/transcript_preparation.py +++ b/src/toad/transcript_preparation.py @@ -20,7 +20,7 @@ ) from agent_comms.ui_model.message_category import MessageCategory -from toad.render_backend import ReusableRenderTask +from toad.render_backend import ContentAddressedRenderTask from toad.transcript_state import LatestViewportRequest from toad.widgets.committed_presentation import PreparedAppendCommit, PreparedPageCommit from toad.widgets.history_anchor import LatestPosition @@ -31,6 +31,8 @@ PreparationRuntime, PreparationScope, PreparedResult, + RenderPreparation, + ReusableStorage, ReusableWork, ScopedWork, ThreadWork, @@ -422,7 +424,7 @@ def fragments_for(self, page: PreparedTranscriptPage): @dataclass(frozen=True) -class TranscriptPageRenderTask(ReusableRenderTask[PreparedTranscriptPage]): +class TranscriptPageRenderTask(ReusableStorage[PreparedTranscriptPage], ContentAddressedRenderTask[PreparedTranscriptPage]): page: TranscriptPage @property @@ -473,7 +475,7 @@ async def execute( raise TranscriptNoProgress(request, self.through, page) if self.scope.closed: raise asyncio.CancelledError - prepared = await runtime.renderer.capture(TranscriptPageRenderTask(page)) + _, prepared = await runtime._prepare(RenderPreparation(TranscriptPageRenderTask(page))) return await runtime.run_thread(self.finish_prepared, key, prepared) diff --git a/tests/textual/test_saved_history_coverage.py b/tests/textual/test_saved_history_coverage.py index 7c5ee75c8..b2247dacb 100644 --- a/tests/textual/test_saved_history_coverage.py +++ b/tests/textual/test_saved_history_coverage.py @@ -76,3 +76,22 @@ async def test_a_live_message_does_not_count_as_saved_history_coverage(workspace conversation.transcript.received(page) await ui.until(lambda: "Saved history available" in str(details.title), why="saved history reported available") assert tuple(history.pages) == pages + + +async def test_read_receipts_reuse_original_prepared_suppliers(workspace): + from toad.transcript_preparation import TranscriptPageRenderTask + from toad.work_preparation import RenderPreparation + + async with workspace.app(size=(110, 35)) as ui: + page, _, _ = await saved_history(workspace, ui) + runtime = ui.app.preparation + first = await runtime.submit(RenderPreparation(TranscriptPageRenderTask(page))) + page.snapshot.records_cut.mark_read( + workspace.comms.bus.reads, "saved", page.snapshot.runtime.read_key, page.after.offset) + current = workspace.comms.transcripts.capture_page_read("saved").read() + assert page.snapshot != current.snapshot + assert page.snapshot.same_content(current.snapshot) + second = await runtime.submit(RenderPreparation(TranscriptPageRenderTask(current))) + assert first is not second + assert first.fragments is second.fragments + assert first.same_source(second) From 02a3ee166966504d0005e5432decef359722c632 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 08:01:07 -0500 Subject: [PATCH 27/35] Bind prepared suppliers to the current transcript read --- src/toad/screens/historical_sessions.py | 4 +--- src/toad/transcript_preparation.py | 22 ++++++++++++------- src/toad/transcript_source_preparation.py | 12 +++++----- src/toad/widgets/committed_presentation.py | 3 +-- tests/textual/test_message_filter_restyle.py | 3 +-- tests/textual/test_saved_history_coverage.py | 8 ++++--- .../test_transcript_presentation_fencing.py | 17 ++++++-------- 7 files changed, 35 insertions(+), 34 deletions(-) diff --git a/src/toad/screens/historical_sessions.py b/src/toad/screens/historical_sessions.py index d43970c8e..9db29d1f3 100644 --- a/src/toad/screens/historical_sessions.py +++ b/src/toad/screens/historical_sessions.py @@ -144,9 +144,7 @@ async def load(**kwargs): ) else: from toad.transcript_preparation import SavedHistory, TranscriptPageRenderTask - from toad.work_preparation import RenderPreparation - - prepared = await self.app.preparation.submit(RenderPreparation(TranscriptPageRenderTask(page))) + prepared = await TranscriptPageRenderTask(page).prepared(self.app.preparation) if generation != self._selection_generation or not self.is_attached: return history = LineHistory(prepared, SavedHistory(load, page.after, self.app.preparation), diff --git a/src/toad/transcript_preparation.py b/src/toad/transcript_preparation.py index 9d2615322..aa7606b51 100644 --- a/src/toad/transcript_preparation.py +++ b/src/toad/transcript_preparation.py @@ -184,12 +184,13 @@ def replace_range(self, view: TranscriptPageView, selected: slice, current: Call return True def same_source(self, other: PreparedTranscriptPage) -> bool: - """Keep a view only when its bounded read came from the same source snapshot.""" - # A filtered page can share the Core read but draw different members. - # The retained worker result shares this immutable tuple with its views; - # another projection cannot borrow that view's prepared rows. Each page - # type owns equality of its source, traversal and read evidence. - return self.fragments is other.fragments and self.page == other.page + """Keep original suppliers only when declared page content is unchanged.""" + # Filtered/restyled results must retain their distinct supplier tuples. + # Core's content projection excludes read annotations without discarding + # the fresh witness carried by each acquired page. + return (self.fragments is other.fragments + and FieldCodec.project(self.page, ContentExclude) + == FieldCodec.project(other.page, ContentExclude)) def for_view(self) -> PreparedTranscriptPage: """An independent view owns its drawing range and loaded source data. @@ -438,6 +439,11 @@ def result_size(self, result: PreparedTranscriptPage) -> int: def execute(self) -> PreparedTranscriptPage: return PreparedTranscriptPage.prepare(self.page) + async def prepared(self, runtime: PreparationRuntime) -> PreparedTranscriptPage: + """Reuse immutable suppliers, never another read's page metadata.""" + prepared = await runtime.submit(RenderPreparation(self)) + return replace(prepared, page=self.page) + class TranscriptNoProgress(ValueError): """A page read returned no new edge while its end still claimed more.""" @@ -475,8 +481,8 @@ async def execute( raise TranscriptNoProgress(request, self.through, page) if self.scope.closed: raise asyncio.CancelledError - _, prepared = await runtime._prepare(RenderPreparation(TranscriptPageRenderTask(page))) - return await runtime.run_thread(self.finish_prepared, key, prepared) + prepared = await TranscriptPageRenderTask(page).prepared(runtime) + return await runtime.run_thread(self.finish_result, key, prepared) @dataclass(frozen=True) diff --git a/src/toad/transcript_source_preparation.py b/src/toad/transcript_source_preparation.py index d3dce7976..5aa6469db 100644 --- a/src/toad/transcript_source_preparation.py +++ b/src/toad/transcript_source_preparation.py @@ -239,8 +239,6 @@ async def _publish_latest(self, current: Callable[[], bool], while rows > 0 and limit > 0: existing = next((view for view in self.pages if view.prepared.same_source(prepared)), None) - if existing is not None: - prepared = existing.prepared.for_view() rows -= await prepared.prepare_rows( self, current, target=self, selected=self.selected_categories, boundary=len(prepared.fragments), end=OLDER_END, @@ -280,10 +278,12 @@ async def _publish_latest(self, current: Callable[[], bool], views.appendleft(view) # Keep existing fragments, prepared lines and documents. for prepared, view in destination: - if view is not None and not view.prepared.replace_range( - view, slice(prepared.start, prepared.stop), current, - ): - raise _DrawingStopped + if view is not None: + if not view.prepared.replace_range( + view, slice(prepared.start, prepared.stop), current, + ): + raise _DrawingStopped + view.prepared.page = prepared.page previous_source = self.page_source self.page_source, self.pages = source, views source.through = latest.page.after diff --git a/src/toad/widgets/committed_presentation.py b/src/toad/widgets/committed_presentation.py index 255ad6540..adec4031a 100644 --- a/src/toad/widgets/committed_presentation.py +++ b/src/toad/widgets/committed_presentation.py @@ -279,9 +279,8 @@ async def read_snapshot[Snapshot]( @classmethod async def capture(cls, view: Conversation, history: TranscriptHistory, page: TranscriptPage, sequences: frozenset[int]) -> Self: from toad.transcript_preparation import TranscriptPageRenderTask - from toad.work_preparation import RenderPreparation - prepared = await view.app.preparation.submit(RenderPreparation(TranscriptPageRenderTask(page))) + prepared = await TranscriptPageRenderTask(page).prepared(view.app.preparation) return cls(history, sequences, prepared) @property diff --git a/tests/textual/test_message_filter_restyle.py b/tests/textual/test_message_filter_restyle.py index f2aab5ff0..b3496a054 100644 --- a/tests/textual/test_message_filter_restyle.py +++ b/tests/textual/test_message_filter_restyle.py @@ -16,7 +16,6 @@ from toad.transcript_preparation import TranscriptPageRenderTask from toad.transcript_state import PreparedReplacementViewportRequest from toad.widgets.history_anchor import LatestPosition -from toad.work_preparation import RenderPreparation async def test_category_change_does_not_restyle_the_whole_conversation(workspace): @@ -72,7 +71,7 @@ async def test_filtering_saved_history_paints_an_older_projection_and_returns_to workspace.comms.registry.require("source"), session_file=RegistrySessionLocation.from_path(str(replacement), workspace.comms.root))) page = await ui.app.preparation.run_thread( lambda: workspace.comms.transcripts.capture_page_read("source").read()) - prepared = await ui.app.preparation.submit(RenderPreparation(TranscriptPageRenderTask(page))) + prepared = await TranscriptPageRenderTask(page).prepared(ui.app.preparation) assert await history.show_newest(PreparedReplacementViewportRequest( LatestPosition.hold(view.window), prepared)) assert previous_source.closed diff --git a/tests/textual/test_saved_history_coverage.py b/tests/textual/test_saved_history_coverage.py index b2247dacb..9fe1cb97f 100644 --- a/tests/textual/test_saved_history_coverage.py +++ b/tests/textual/test_saved_history_coverage.py @@ -80,18 +80,20 @@ async def test_a_live_message_does_not_count_as_saved_history_coverage(workspace async def test_read_receipts_reuse_original_prepared_suppliers(workspace): from toad.transcript_preparation import TranscriptPageRenderTask - from toad.work_preparation import RenderPreparation async with workspace.app(size=(110, 35)) as ui: page, _, _ = await saved_history(workspace, ui) runtime = ui.app.preparation - first = await runtime.submit(RenderPreparation(TranscriptPageRenderTask(page))) + first = await TranscriptPageRenderTask(page).prepared(runtime) page.snapshot.records_cut.mark_read( workspace.comms.bus.reads, "saved", page.snapshot.runtime.read_key, page.after.offset) current = workspace.comms.transcripts.capture_page_read("saved").read() assert page.snapshot != current.snapshot assert page.snapshot.same_content(current.snapshot) - second = await runtime.submit(RenderPreparation(TranscriptPageRenderTask(current))) + second = await TranscriptPageRenderTask(current).prepared(runtime) assert first is not second + assert first.page is page + assert second.page is current + assert first.page.snapshot != second.page.snapshot assert first.fragments is second.fragments assert first.same_source(second) diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index bbcfaa1d3..81a8b2dcb 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -275,7 +275,6 @@ async def test_closed_same_source_commit_acquires_complete_coverage(workspace, f @pytest.mark.parametrize("following", (False, True)) async def test_rewritten_saved_file_replaces_paint_even_when_its_cursor_grew(workspace, following): from dataclasses import replace - from agent_comms.coordination_errors import TranscriptChanged thread = workspace.thread("rewritten-source") journal = write_journal(thread.project_path / "rewritten.jsonl", @@ -289,6 +288,7 @@ async def test_rewritten_saved_file_replaces_paint_even_when_its_cursor_grew(wor await view.transcript.suspend() history = view.transcript.binding.history original = history.page_source.through + original_snapshot = history.pages[0].prepared.page.snapshot if not following: view.window.release_anchor() # Reuse the path, but replace its native bytes with a longer record. @@ -296,13 +296,11 @@ async def test_rewritten_saved_file_replaces_paint_even_when_its_cursor_grew(wor write_journal(journal, assistant_entry( "REPLACEMENT_SOURCE_ROW " * 80, id="new")) - def read_replacement(): - with pytest.raises(TranscriptChanged): - workspace.comms.transcripts.capture_page_read( - "rewritten-source", position=original.later_page()).read() - return workspace.comms.transcripts.capture_page_read("rewritten-source").read() - - replacement = await asyncio.to_thread(read_replacement) + replacement = await asyncio.to_thread( + lambda: workspace.comms.transcripts.capture_page_read("rewritten-source").read()) + # A cursor contains path/offset, not the old bytes' witness. The + # snapshot owns prefix certification; stale offsets can land inside JSON. + assert not original_snapshot.contains_records_cut(replacement.snapshot) assert replacement.after.contains(original) view.transcript.dirty = view.transcript.commit_required = True await view.transcript.publish(CommitPublication) @@ -367,7 +365,6 @@ async def test_a_closed_history_accepts_a_prepared_replacement_without_a_storage from toad.transcript_preparation import ClosedHistory, TranscriptPageRenderTask from toad.transcript_state import PreparedReplacementViewportRequest from toad.widgets.history_anchor import LatestPosition - from toad.work_preparation import RenderPreparation old = write_journal(workspace.project / "old.jsonl", assistant_entry("OLD_RETAINED_ROWS", id="old")) thread = workspace.thread("source") @@ -384,7 +381,7 @@ async def test_a_closed_history_accepts_a_prepared_replacement_without_a_storage () if empty else (assistant_entry("PREPARED_REPLACEMENT", id="new"),))) workspace.comms.registry.declare(replace(thread, session_file=RegistrySessionLocation.from_path(str(new), workspace.comms.root))) page = workspace.comms.transcripts.capture_page_read("source").read() - prepared = await ui.app.preparation.submit(RenderPreparation(TranscriptPageRenderTask(page))) + prepared = await TranscriptPageRenderTask(page).prepared(ui.app.preparation) assert await history.show_newest(PreparedReplacementViewportRequest( LatestPosition.hold(history.window), prepared)) await ui.step() From 79f560ccdefe19dd56c22e8c7790e211fd1614ff Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 08:56:26 -0500 Subject: [PATCH 28/35] Correct the overlay-sensitive paint oracle and record capture15 limits --- docs/performance/scroll-series.md | 81 +++++++++++++++++++ .../test_transcript_presentation_fencing.py | 7 +- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/docs/performance/scroll-series.md b/docs/performance/scroll-series.md index 2feda9f87..b7d73de67 100644 --- a/docs/performance/scroll-series.md +++ b/docs/performance/scroll-series.md @@ -45,6 +45,87 @@ and adjacent controller logs/return codes.09 includes `frontend/loaded-ready.png The immutable archive remains borrowed by UI; these records do not replace the configured active-answer/held-PageUp/default-launcher journey still owed. +## 2026-11-02 candidate1408 capture15 — partial, not a latency result + +Target Toad `14f05429add046cf81d9535a787e7841fbfdfc82` with Core +`94631c18e04a168c48293073bc93fef40b1eb050` was prepared through the +existing WheelRuntime owner in `capture-1408-build-02`. Interpreter, native, +package, resolution and client probes completed. The dependency donor was the +immutable cb4 runtime, not the later installed stack. This is a source-qualified +candidate epoch; it does not exercise the default shell launcher or phone. + +The installed owner created a fresh private history copy/fork, stopped that +private source owner and retargeted only the copy. The target owner was started +through its existing lifecycle. On an owned isolated display/plain st with +private HOME/XDG/root, startup admission, saved history, Ready and composer focus +passed. The fresh input was attempted once. `before-observation.json` records +both `turn_busy=true` and actual response text containing +`STYLED_HEAD_14F_FRESH_STREAM`; the qualification also required that text in the +clipped published response's real st cells. This was not a user echo or thought. + +The controller dispatched held PageUp (8s), held PageDown (4s), wheel reversals +and End. **The End screenshot still shows old history and Back to bottom; it does +not establish an End-to-tail pass.** The run then failed `tab-away` (rc1). Its +predicate compared `active_mode`, but that field is the owning Textual screen +mode (`app.current_mode`), not tab identity; the image also shows only one tab. +A changing tab cannot be certified by changing workspace mode. The complete +scenario, tab-return and preservation assertions therefore remain unproved. + +Exceptional teardown bypassed the normal successful frontend exit path and no +`frontend/frame-meter.json` was published. All frame-work and input-to-paint +median/p95 values are **not measured**, not zero. The 24s CPU request stopped +when the selected process ended:741 samples,0 sampling errors, followed by +`The selected process birth ended during profiling`. Keep that partial +speedscope file as diagnostic evidence, not representative completed CPU proof. +The video and screenshots remain useful bounded visual evidence, not latency +numbers. st/Xvfb/recording and private-copy retirement records were retained; +this does not establish a successful application journey. + +Artifacts: `~/.cache/agent-scratch/pi-ui-performance/capture-1408-14f-15/`, +its adjacent log/rc, and `capture-1408-build-02/result.json`. Failed d34 capture14 +and capture15 retain their original input/session evidence and are not replay +candidates. No follow-up input or repeated capture is authorized by a new number. +The next harness correction must use actual tab identity/setup and preserve the +startup meter through its existing observation lifetime on exceptional close. +Separately,1408's late-first-styled-rows reader question is tested with the +anchor's actual painted screen position, not scroll_y alone. + +| Epoch | Frame median/p95 | Input-to-paint median/p95 | CPU | Result | +|---|---|---|---|---| +| Candidate1408/14f capture15 | Not measured: final artifact absent | Not measured: final artifact absent | Partial741-sample profile only | Active reply qualified; gestures dispatched; End-to-tail and tab-return not established; scenario failed | + +## 2026-11-02 source controls09/10 — overlay is not discarded rows + +Startup-only geometry observation09 reproduced the closed-prefix assertion at +immutable `prepared-metadata-08` source. Before the commit, painted row30 was +`▌CLOSED_PREFIX_MUST_SURVIVE`; afterward it was +`▌CLOSED_PREFIX_MUST_S ↓ Back to bottom · Esc`. The same acquired supplier still +held3 cached rows, with unchanged page and window regions. The existing +`#back-to-bottom-bar` is a3-row floating screen overlay; it covered the marker +suffix. This trace refutes the earlier diagnosis that the prefix's source rows +had disappeared. It does not prove unrestricted screen-position preservation. +The failed long-marker evidence remains in `closed-prefix-09/`. + +The control now uses a short distinct source marker clear of that overlay. +Its page identity, original fragment identity, reader equality, no-successor +paint and failure checks are unchanged. No overlay CSS or reader-placement +workaround was added to turn it green. + +Current-source controls10 ran Toad22d plus the retained uncommitted marker/doc +patch, Core946 and current-library paths recorded in `inputs.json`. Result: +**5 passed,1 failed in93.57s**, rc1; host lease released. Both closed-prefix +following modes, live-message coverage, fresh read metadata/original suppliers +and the restyle counterexample passed. The known-gap case still failed +`assert not history.drawn_end_more` immediately after publication, before its +later lookahead. It remains an open source-publication obligation; this is not a +complete green source or installed result. The separate Core rewritten-prefix +witness failure is not included in these six controls and remains open too. + +Artifacts: `~/.cache/agent-scratch/pi-ui-performance/closed-prefix-09/` and +`prepared-current-10/`, their adjacent logs/rc, and the exact uncommitted patch +retained by controls10. No provider input or physical frontend was started by +these source controls. + ## Historical series One line per change, from `tools/performance/scroll_scenario.sh` (real `toad` in `st` on display :121, a fork of `perf-base-20261009`, which is a fork of the saved `agent-comms-ux` session; an active response, held PageUp from the input box, wheel bursts with reversal, End, tab return). Frame work is UI-thread time per frame handed to the driver; input-to-paint runs from the driver posting an event to the first frame after its last handler returns. Input-to-paint pairs each input with the first frame after it is handled (fixed 2026-10-09; earlier summaries paired out of order). Raw results are under `~/.cache/agent-scratch/scroll-series//`. diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 81a8b2dcb..50a904506 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -238,11 +238,12 @@ async def test_closed_same_source_commit_acquires_complete_coverage(workspace, f thread = workspace.thread("closed-prefix") journal = write_journal(thread.project_path / "closed.jsonl", - assistant_entry("CLOSED_PREFIX_MUST_SURVIVE", id="before")) + assistant_entry("KEPT_ROW", id="before")) workspace.comms.registry.declare(replace( thread, session_file=RegistrySessionLocation.from_path(str(journal), workspace.comms.root))) async with workspace.app(resume="closed-prefix") as ui: - await ui.until_shows("CLOSED_PREFIX_MUST_SURVIVE") + # Keep the source marker clear of the window's floating jump control. + await ui.until_shows("KEPT_ROW") view = ui.conversation await ui.app.coordination_access.stop_observing() await view.transcript.suspend() @@ -267,7 +268,7 @@ async def test_closed_same_source_commit_acquires_complete_coverage(workspace, f zip(history.fragments_in_range, fragments, strict=True)) assert view.window.reader_position == position assert "CLOSED_PREFIX_SUCCESSOR" not in ui.painted(view.window) - assert "CLOSED_PREFIX_MUST_SURVIVE" in ui.painted(view.window) + assert "KEPT_ROW" in ui.painted(view.window) assert view.transcript.binding.history is history assert ui.failure() is None From 09d7ecc04b4baeabf324cb6eecdfa50b2abf4adf Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 09:34:46 -0500 Subject: [PATCH 29/35] Declare observer boundary names and preserve the selected tab identity --- src/toad/transcript_preparation.py | 2 +- src/toad/transcript_source_preparation.py | 3 +- tools/physical_ui.py | 34 ++++++++++++++++++++--- tools/physical_ui_remote.py | 15 ++++++++-- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/toad/transcript_preparation.py b/src/toad/transcript_preparation.py index aa7606b51..6202d0db9 100644 --- a/src/toad/transcript_preparation.py +++ b/src/toad/transcript_preparation.py @@ -430,7 +430,7 @@ class TranscriptPageRenderTask(ReusableStorage[PreparedTranscriptPage], ContentA @property def preparation_inputs(self) -> object: - """Read receipts and coordination activity cannot change prepared content.""" + """Read marks and coordination activity cannot change prepared content.""" return FieldCodec.project(self.page, ContentExclude) def result_size(self, result: PreparedTranscriptPage) -> int: diff --git a/src/toad/transcript_source_preparation.py b/src/toad/transcript_source_preparation.py index 5aa6469db..be0340efb 100644 --- a/src/toad/transcript_source_preparation.py +++ b/src/toad/transcript_source_preparation.py @@ -8,6 +8,7 @@ from functools import partial from typing import TYPE_CHECKING +from agent_comms.transcripts import TranscriptCursor from textual.await_complete import AwaitComplete from textual.worker import Worker, WorkerCancelled @@ -310,7 +311,7 @@ def drawn_end_more(self) -> bool: edge = self.pages[-1].prepared return NEWER_END.can_extend(edge) or edge.page.has_newer - async def publish_append(self, through, publication_current: Callable[[], bool]) -> bool: + async def publish_append(self, through: TranscriptCursor, publication_current: Callable[[], bool]) -> bool: """A commit may retire live blocks only after publishing its destination. extend also serves held readers: its True can mean access advanced diff --git a/tools/physical_ui.py b/tools/physical_ui.py index ab7c63151..11933aa83 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -10,6 +10,7 @@ from contextlib import AbstractContextManager, ExitStack, contextmanager import ctypes from dataclasses import dataclass +from enum import StrEnum from functools import cache import itertools import json @@ -51,6 +52,28 @@ ) +class DisplaySocket(StrEnum): + """X11's external socket spelling, not an application-owned kind.""" + + primary = "X0" + + +class DisplayProgram(StrEnum): + framebuffer = "Xvfb" + + +class FrontendVariable(StrEnum): + """Parent-only external settings excluded from an isolated frontend.""" + + python_path = "PYTHONPATH" + python_home = "PYTHONHOME" + no_color = "NO_COLOR" + validation = "TOAD_VALIDATION" + + def remove_from(self, environment: dict[str, str]) -> None: + environment.pop(self.value, None) + + class XvfbDisplay(AbstractContextManager): """Own a connection to a local framebuffer, resolving aliases at Xlib. @@ -83,7 +106,8 @@ def __init__(self, name: str): pid, uid, _gid = struct.unpack("3i", peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) address = Path(os.fsdecode(peer.getpeername()).lstrip("\0")) process = ProcessIdentity.capture(pid) - if address.name == "X0" or uid != os.getuid() or Path(f"/proc/{pid}/exe").resolve().name != "Xvfb": + if (address.name == DisplaySocket.primary.value or uid != os.getuid() + or Path(f"/proc/{pid}/exe").resolve().name != DisplayProgram.framebuffer.value): raise ValueError(f"Not an owned isolated Xvfb resource: {name}") self.name, self.process = name, process self.x11, self.connection = x11, connection @@ -142,7 +166,7 @@ def __init__(self, output: Path, *, size: Size = default_size): output.mkdir(parents=True, exist_ok=True) self.log = (output / "xvfb.log").open("wb") self.server = SpawnedProcess.launch( - ("Xvfb", "-displayfd", "1", "-screen", "0", f"{size.width}x{size.height}x24", "-nolisten", "tcp"), + (DisplayProgram.framebuffer.value, "-displayfd", "1", "-screen", "0", f"{size.width}x{size.height}x24", "-nolisten", "tcp"), stdio=DetachedChildStdio(stdout=subprocess.PIPE, stderr=self.log), ) try: @@ -267,8 +291,10 @@ def __init__(self, command: tuple[str, ...], display: XvfbDisplay, output: Path, ConfiguredBinary().configured(environment), target.launch_runtime, ) for key in tuple(environment): - if key.startswith("XDG_") or key in ("PYTHONPATH", "PYTHONHOME", "NO_COLOR", "TOAD_VALIDATION"): + if key.startswith("XDG_"): del environment[key] + for variable in FrontendVariable: + variable.remove_from(environment) environment.update( HOME=str(target.home), DISPLAY=display.name, XDG_CONFIG_HOME=str(target.config), XDG_STATE_HOME=str(target.state), @@ -402,7 +428,7 @@ def stop_copy(cls, target: RunResources, directory: Path) -> None: def _packet(self, operation: ObserveOperation) -> Path: directory = self.output / "operations" / f"{next(self.operations):06d}" directory.mkdir(mode=0o700, parents=True) - packet = directory / "request.json" + packet = ObserveOperation.packet_path(directory) PhysicalOperation.publish(packet, operation) return packet diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index fc6effccb..2d2f50e2c 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -35,6 +35,7 @@ from toad.acp.agent import Agent from toad.agent_binding import NoAgentConnection from toad.core.projection import MroProjection +from toad.tab_identity import TabIdentity from ui_theme.wal import WalFile from agent_comms.store_files import ExclusiveAtomicFile @@ -170,7 +171,7 @@ class RunObservation: responses: tuple[CapturedResponse, ...] = () composer_focused: bool = False composer_regions: tuple[CapturedRegion, ...] = () - active_mode: str = "" + selected_tabs: tuple[TabIdentity, ...] = () @dataclass(frozen=True) @@ -366,7 +367,7 @@ def serve(self, requests: Connection, output: Path) -> None: packet = FieldCodec.decode(Path, FieldCodec.parse(raw.decode())) if (not packet.is_absolute() or packet != packet.resolve(strict=True) or not packet.is_relative_to(output / "operations") - or packet.name != "request.json"): + or packet != ObserveOperation.packet_path(packet.parent)): raise ValueError("Observation packet is outside this startup's operation directory") from textual._context import NoActiveAppError from toad.app import ToadApp @@ -449,6 +450,11 @@ class ObserveOperation(PhysicalOperation): identity: ProcessIdentity tools: Path + @classmethod + def packet_path(cls, directory: Path) -> Path: + """The read-only observation protocol owns its one packet filename.""" + return directory / "request.json" + @contextmanager def tool_imports(self): previous = sys.path[:] @@ -548,7 +554,10 @@ def execute(self, directory: Path) -> None: for editor in screen.query(PromptTextArea) for placement in (published.get(editor),) if placement is not None ), - active_mode=app.current_mode, + selected_tabs=tuple( + entry.identity for entry in sessions.members + if app.workspace_sessions.source.current(entry.source(sessions)) + ), ) self.target.require_resources(observed) self.publish(directory / "result.json", self.observed_result(observed, directory)) From 207e69894031c6d05e41203dd3da72eb012058a0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 10:21:50 -0500 Subject: [PATCH 30/35] Keep physical frontend contracts in the Textual test layer --- tests/{model => textual}/test_physical_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/{model => textual}/test_physical_run.py (99%) diff --git a/tests/model/test_physical_run.py b/tests/textual/test_physical_run.py similarity index 99% rename from tests/model/test_physical_run.py rename to tests/textual/test_physical_run.py index 98bb20e32..bfb659b1b 100644 --- a/tests/model/test_physical_run.py +++ b/tests/textual/test_physical_run.py @@ -1,4 +1,4 @@ -"""Admission and current-resource contracts; no frontend, provider or private bus.""" +"""Physical frontend observation and admission contracts, using native geometry.""" from dataclasses import replace from pathlib import Path From f1363471cf8a6b9d417d2f49d858baf515a58e1b Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 10:26:21 -0500 Subject: [PATCH 31/35] Check read blocking and native paint instead of publication completion --- docs/performance/scroll-series.md | 16 ++++++++++++++++ .../test_transcript_presentation_fencing.py | 10 +++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/performance/scroll-series.md b/docs/performance/scroll-series.md index b7d73de67..66feb4578 100644 --- a/docs/performance/scroll-series.md +++ b/docs/performance/scroll-series.md @@ -126,6 +126,22 @@ Artifacts: `~/.cache/agent-scratch/pi-ui-performance/closed-prefix-09/` and retained by controls10. No provider input or physical frontend was started by these source controls. +### Source-trace correction to the known-gap oracle + +`CommitPublication.publish_source_page` treats a guarded no-op as completion; +its True return does not certify a drawn destination. `PreparedPageCommit.draw` +may decline while the pager owns source work. Retirement occurs only after its +successful draw and the final current-source checks. Source work finishes through +`TranscriptSourcePreparation.finish_source_work` → `TranscriptSourceWorkFinished` +→ `TranscriptPresentation.source_work_finished` → `retry`. + +The gap test now checks the reader contract rather than assigning a stronger +meaning to that return: the known gap blocks visible read acknowledgement, +the destination becomes visible through native paint, and source settlement +removes the gap and retains the expected cursor. This corrects the immediate +assertion above; the earlier failing run remains retained. The corrected case +has not yet been executed, so this trace is not a new passing-run claim. + ## Historical series One line per change, from `tools/performance/scroll_scenario.sh` (real `toad` in `st` on display :121, a fork of `perf-base-20261009`, which is a fork of the saved `agent-comms-ux` session; an active response, held PageUp from the input box, wheel bursts with reversal, End, tab return). Frame work is UI-thread time per frame handed to the driver; input-to-paint runs from the driver posting an event to the first frame after its last handler returns. Input-to-paint pairs each input with the first frame after it is handled (fixed 2026-10-09; earlier summaries paired out of order). Raw results are under `~/.cache/agent-scratch/scroll-series//`. diff --git a/tests/textual/test_transcript_presentation_fencing.py b/tests/textual/test_transcript_presentation_fencing.py index 50a904506..d2c7b88b7 100644 --- a/tests/textual/test_transcript_presentation_fencing.py +++ b/tests/textual/test_transcript_presentation_fencing.py @@ -192,7 +192,7 @@ async def test_followed_saved_append_keeps_painted_page_and_fragments(workspace) assert ui.failure() is None -async def test_followed_commit_publishes_a_known_undrawn_gap_before_settling(workspace): +async def test_known_source_gap_blocks_read_acknowledgement_until_painted(workspace): import json from dataclasses import replace @@ -221,13 +221,17 @@ async def test_followed_commit_publishes_a_known_undrawn_gap_before_settling(wor page = workspace.comms.transcripts.capture_page_read("gap-source").read() history.retain_committed(page.after) assert view.window.follows_tail and history.drawn_end_more + assert history.blocks_visible_read finally: release.set() await pending - # Check the publication contract before a later lookahead can hide it. + # Completion includes guarded no-ops. The source-work completion event + # retries pending commits; native paint is the observation that settles + # this check, not the return of one publication attempt. + await ui.until_shows("AFTER_SOURCE_GAP") + await ui.until(lambda: not history.blocks_visible_read, why="the painted source to settle") assert not history.drawn_end_more assert history.pages[-1].prepared.page.after.same_position(page.after) - await ui.until_shows("AFTER_SOURCE_GAP") assert ui.failure() is None From d8ed02e002bcf5b87a15f651bf0c12ddbbec61e7 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 11:11:44 -0500 Subject: [PATCH 32/35] Own observer imports at startup and delete the retired injection collector --- docs/performance/bounded-history-capture.md | 146 ++------- docs/performance/physical-run.md | 7 +- docs/performance/scroll-series.md | 4 +- tests/textual/test_physical_run.py | 23 +- tools/performance/capture_history.py | 205 ------------- tools/performance/history_capture_remote.py | 315 -------------------- tools/physical_ui.py | 23 +- tools/physical_ui_remote.py | 49 ++- 8 files changed, 114 insertions(+), 658 deletions(-) delete mode 100644 tools/performance/capture_history.py delete mode 100644 tools/performance/history_capture_remote.py diff --git a/docs/performance/bounded-history-capture.md b/docs/performance/bounded-history-capture.md index 339eca50b..6b5d50dad 100644 --- a/docs/performance/bounded-history-capture.md +++ b/docs/performance/bounded-history-capture.md @@ -1,113 +1,33 @@ -# Historical history/frame observation — do not run - -The frozen inputs documented here remain unchanged for retained diagnosis. -Their running-process injection is forbidden by the current host permission; -the old approval is not launch authority. See [the startup replacement](physical-run.md#startup-observation-current-source-not-yet-run). - -This is an observation tool, not a history performance repair. Authors do not run it. Root reviews the exact head, selects the private-process controls below, and only then attaches to its explicitly selected copied-bus frontend. No synthetic app, UI input, screenshots, provider calls, saved-history decoding or service operations are involved. - -**Xvfb isolates the display, not data.** Root's earlier frontend used the published bus and user XDG paths; merely painting marked human reads. Every future root frontend must use root's existing CheckRoot/actual copied bus through `--comms-root` AND empty/private `XDG_STATE_HOME`, `XDG_CONFIG_HOME` and `XDG_DATA_HOME`. Copy only approved real history through its owner, never operate on the original. Do not blindly copy the live tab database: saved tab roots can still point back at original data. Neither the observer nor an author may launch, attach or send input against the live root. Preserve original human read markers and XDG files; do not manually restore or rewrite them. An earlier successful collector run is not proof of data isolation. - -## Owners and first split - -`frame_meter.BoundedCapture` owns admission, encoded record bytes, weak labels, timers, hook restoration and atomic publication. Both public entrypoints accept `max_records`, `max_bytes`, `metadata_only` and return its idempotent bound `finish` method. `MethodTarget` owns descriptor preservation and inherited-declaration restoration. Metadata adapters register their target at subclass declaration; there is no separate target roster in the launcher. - -`history_capture_remote.RemoteCaptureRun` owns completion. Its run directory fixes `complete.json` at construction. Duration, failed setup and explicit controller cleanup all call its one `finish`: restore handles once, preserve the first status, publish that same canonical completion, then retire the in-memory owner. A failed publication leaves the completed owner reachable without restoring hooks twice. The existing bounded `publish` owns its fixed `.partial` only after successful `os.open(..., O_EXCL)`. Write/close/rename failure releases only that attempt's partial and propagates the original exception; a cleanup failure remains its chained cause. A foreign preexisting partial is never removed, and an existing final file changes only on successful atomic rename. After repairing the actual filesystem prerequisite, a distinct cleanup can publish the same first status and retire its owner without restoring twice. `read_completion` owns the bounded file read and matching-token check; operation-specific replies are only views of this completion, never another completion decision. - -`history_capture_remote.CaptureSpec` is the one frozen, data-only selection for duration and the frame/call record and byte limits. Its dataclass fields derive the controller's CLI arguments and defaults. `CaptureRequest` carries that same value through FieldCodec into both immutable copied requests; manifest, entered receipt, collectors and group timer consume it rather than restating the limits. `ProcessIdentity` carries PID/boot/start ticks. The nominal `CaptureOperation` members own remote start and cleanup, and derive request-script names; there is no second operation roster or raw request-key reader. - -The source declarations used are: - -- `PreparedPageSource.get`: source/scope, request end/cursor; the result is the actual `PreparedTranscriptPage` passed to row preparation. -- `PreparedTranscriptPage.prepare_rows`: the same weak page label, cursors, fragment count, actual selected-category identity, direction, target width, requested rows and returned rows. -- `LineHistory._load_page`: inclusive parent span, history/source/scope/filter identity and width. `needs_more` is its boolean return, **not** proof of publication. -- `LineHistory._extend_and_trim`: actual staged page label and local/source growth. `TranscriptSourcePreparation.rows_published` joins through that history and `_load_page` parent. A row-admission event is not completed layout. -- `ToadApp.select_session` / `_switch_mode_ready`, `SwitchToast.show`: natural tab request/switch and toast duration, never title/body serialization. -- `FramePresentation.written` and `_FlushSignal.deliver`: UI acknowledgement versus writer-thread flush signal. Neither is pixels. -- Frame/input/layout hooks remain in `install`. A frame sample requires an actual successful `LinuxDriver.write` directly from the declared `App._display` code. Merely entering `_display`, headless publication, a batch-gated call, or begin/end-update control writes cannot create a sample. - -Labels are local to each output, use monotonically assigned integers and weak references, and emit identity/retirement records. No product object is retained or hashed. Unsupported weak lifetimes are `null`, not bare reusable `id()` values. Cross-file correlation uses the common monotonic clock, not coincident label integers. `ValueMetadata`, declared once at the lazy observation boundary, uses the existing `MroProjection`/`handles` mechanism. Native `TranscriptPage`/`TranscriptCursor` retain native byte offset, assigned wire sequence and selected witness scalars. `WirePage`/`WireCut` use their actual `source`/`cursor` declarations; `PageCursor` supplies lower/upper/history-source capabilities, including live, historical and unread cuts. `PreparedWirePage` inherits the shared prepared-page handler. `MessagePage` contributes selected evidence type, message count and derived endpoint cursors, never message bodies. Missing values are explicit absent records; other unsupported classes emit a bounded type/unsupported record instead of probing native fields or stopping the capture. Snapshot identity is a weak label, not a serialized `TranscriptReadIdentity`. - -Parent spans are inclusive. Awaited thread CPU includes other work on that thread during suspension; it is not exclusive task CPU. Never add nested/concurrent durations. A first later display is a temporal candidate, not evidence that the same page geometry was painted. This first split can identify a dominant inclusive term; it cannot prove pixels, exact child cost, lock wait, ready-hit/eviction or a usable-page deadline. Missing page match, no older edge, zero frames or exhausted caps means inconclusive attribution, not success. - -## Bounds and lifecycle, from the source (not executed proof) - -1. Each record is JSON-encoded once using compact separators and UTF-8 before admission. Retained buffers contain only those bytes. No mutable nested product records are admitted. Metadata records have a fixed scalar shape, bounded names/paths, and no event/message/fragment bodies. -2. `used` starts with the exact static envelope and every stream's key/brackets/separators. Each admitted record charges its encoded length plus its actual comma. Admission requires `used + incoming + FINAL_RESERVE <= max_bytes` and count below `max_records`. A cap stops admission immediately, even when restoration must await the UI loop. -3. `FINAL_RESERVE` is 8192 bytes for status/key/separators. Status fields are bounded strings, scalar clocks/counts and booleans; output byte count includes its own decimal spelling via fixed-point accounting. Hooks are already restored before writing. Output streams directly to one exclusive `.partial` and is renamed; no second bulk output is made. -4. Exact count/cap/error/censored-open-span/censored-input status is retained. Work occurring after close is unobserved, not falsely counted as zero drops. No record is appended by a late async completion. Async wrappers still return/cancel/raise through the original function. -5. All descriptors, target existence and running loop are checked before patching. Each installed hook remembers the original descriptor and whether it was inherited. Reverse restoration only replaces this capture's exact wrapper; a later wrapper is left alone and reported as superseded. Timer, explicit finish, admission error, cap and partial setup use the same owner. -6. Metadata collection does not start a stall watcher, install a GC callback, call `gc.get_objects`, traverse a heap, inspect coroutine locals, walk arbitrary dataclasses, or call `_subject`, `_field`, `_result` or product `repr`. Old standalone heap diagnostic functions remain explicit opt-ins; this launcher never invokes them. -7. Legacy `trace_calls(metadata_only=False)` retains its six-column JSON array. Metadata mode uses a `calls/events/pauses/capture` envelope. Existing frame/input/lag array shapes and `summarize` remain; immutable input-handler completions are coalesced by arrival using the last observed handler end. Intrusive stack/heap diagnostics are no longer part of `install`; flat bounded events replace those unbounded maps. -8. `measuring_pause` borrows each active capped lifetime. Start markers, closed interval pairs and open interval IDs share its record/byte limits; there is no uncapped global interval list. Finish retains the earliest open/censored start and the admission-end clock in the reserved scalar footer: all unfinished exclusions overlap that end, so one union interval suffices. Nested intervals are bounded by admitted starts; late ends cannot alter a sealed capture. `summarize` preserves the complete capture status, excludes closed/censored pauses (including input-to-handoff crossings), and reports absent switch samples or historical missing exclusions as `unobserved`, not zero. -9. `BoundedCapture.complete_span` holds the existing `RLock` across accepting check, terminal admission, open-span retirement and censorship. Metadata errors and cap exhaustion remain inside that same transaction. Entry also checks admission under this lock; a wrapper losing the race with finish does not allocate an observed span. Finish cannot see a terminal record while still counting that same span as open. - -## Root launch and recovery - -Use the reviewed `tools/performance/capture_history.py` with the explicit copied-bus/private-XDG frontend PID, boot ID, `/proc/PID/stat` start ticks, canonical installed runtime directory and exact approved meter SHA256. Root must establish copied-bus and all three XDG paths before any launch or attach; the capture arguments below do not establish that isolation. The launcher checks the meter AST interface/hash and process birth/mapped runtime before any attach; unsupported contract fails before `sys.remote_exec`. - -``` -python tools/performance/capture_history.py start --approved \ - --pid CURRENT_PID --boot CURRENT_BOOT --start-ticks CURRENT_TICKS \ - --runtime /absolute/canonical/runtime \ - --meter /reviewed/checkout/tools/performance/frame_meter.py --sha256 APPROVED_SHA256 \ - --output /home/ts/.cache/agent-scratch/ROOT_OWNER/p903-UNIQUE -``` - -The controller writes immutable typed requests and copies `history_capture_remote.py` as `start.py` and **`cleanup.py`** in that run directory. Only copies with an adjacent request execute remote work; ordinary import performs no attachment. Both file boundaries decode `CaptureRequest` with the existing FieldCodec; there are no compatibility readers for older requests. Root must preserve the original recovery scripts with their original requests after uncertainty, never replace them with a new revision or retry their starts. - -`CaptureSpec` owns the shared allowances: attach helper 5s, completion grace 3s, cleanup receipt 2s, poll interval 250ms. Observation budget is `attach_timeout + seconds + completion_grace`; cleanup budget is `attach_timeout + cleanup_receipt_timeout`; controller budget is their sum. Default selection therefore has a 33s observation wait and a 40s controller wait budget, rather than unrelated 28/35 literals. These bound waits, not arbitrary filesystem/scheduler latency. Root runs the command in the background; an external timeout must allow the cleanup budget, and interrupting a helper never proves remote code did not run. The helper's bounded stderr/exception cause is retained in the existing controller status, including EPERM, nonzero exit and timeout. Setup failures retain their cause in the complete result even if that file appears before the separate failure receipt. No arbitrary product representation or traceback enters the receipt. - -The target rechecks birth/runtime, all loaded Toad/Textual/Core module locations, the meter hash and actual descriptors before patching. At the foreign module boundary, `require_modules` requires a real `ModuleSpec`: its concrete `origin` when `has_location`, otherwise every namespace `submodule_search_locations` entry. The applicable set must be nonempty and **every** resolved location must lie inside the selected runtime. A namespace such as `agent_comms.skills` is not rejected just for lacking `__file__`; it gets no spelling exemption either. Missing specs/locations, mixed or outside locations and missing `toad.app` fail closed. Controller and target use the same size-bounded source reader (256KiB plus one overflow byte). The target compiles and executes exactly that verified buffer with its registered module namespace and `__file__`; neither loader reread nor cached bytecode can replace the approved bytes. Metadata imports and actual descriptors are checked before the first patch and followed by another provenance check. Locations prove provenance, **not a hash of already-loaded code**. No particular conversation is required to obtain global frames; source mismatch and absence of an older page remain visible/inconclusive. - -Default parity: `CaptureSpec` selects 25s, 20,000 frame records, 20,000 call records, 3MiB frames and 12MiB calls. Two code copies, two bounded requests and all bounded receipts reserve less than 1MiB, so default partial/final output and metadata fit 16MiB. CLI field names derive directly: `--seconds`, `--frames-max-records`, `--calls-max-records`, `--frames-max-bytes`, `--calls-max-bytes`. Foreign CLI/JSON values must have their declared scalar types and be finite and positive before setup. BoundedCapture still enforces its existing physical ceilings and byte accounting; neither collector algorithm changes. Invalid physical limits fail during its setup and use the same restoration path if another collector was already armed. No ceiling values are copied into CaptureSpec. - -For root to choose **after** a fresh isolated qualification measures actual frame rate, bytes and overhead, an example long selection is 900s, 500,000 frame records/64MiB and 200,000 call records/64MiB. This gives a 908s observation wait and 915s controller wait budget, with less than 129MiB reserved output. The current meter's 4ms lag probe also spends the frame record budget; a large seconds flag alone never guarantees exposure. Use a fresh selected PID/birth and directory, with the same identity/runtime/meter/hash arguments as above: - -``` -timeout -s INT 915 python tools/performance/capture_history.py start --approved \ - --pid CURRENT_PID --boot CURRENT_BOOT --start-ticks CURRENT_TICKS \ - --runtime /absolute/canonical/runtime \ - --meter /reviewed/checkout/tools/performance/frame_meter.py --sha256 APPROVED_SHA256 \ - --seconds 900 --frames-max-records 500000 --calls-max-records 200000 \ - --frames-max-bytes 67108864 --calls-max-bytes 67108864 \ - --output /home/ts/.cache/agent-scratch/ROOT_OWNER/p948-UNIQUE -``` - -A second terminal can derive the selected wait budget without attaching: import `CaptureSpec`, construct it with the same CLI selection and read `controller_budget`. The example 915 is this formula's result, not a second code default. An external timeout can interrupt final publication; inspect retained status and use the distinct cleanup operation below if restoration is unconfirmed. Neither 900s nor a successful footer proves 10,000 loaded frames. Actual caps, censorship, interval, count and concurrent N/M determine the evidence; cap exhaustion is inconclusive. Descriptive nearest-rank p99.9 is sorted sample rank `ceil(.999*n)` (one-based), and equals the maximum when n<1000. At 10,000 frames only about ten expected exceedances support that tail; do not assert independent-sample confidence for correlated frames. - -Root may start existing read-only external py-spy concurrently **after** `entered.json` and correlate its start/end clocks with the same frame interval. `capture_live.py` runs profiling before its meter if both options are supplied, so that combination is not contemporaneous. Keep nested inclusive profile/call/layout durations separate. The bounded meter has no GC stream: old GC captures are leads, not a current cause. No GC/heap callback, scheduling experiment or new meter is introduced here. - -For a separate explicitly authorized recovery after inspection: - -``` -python tools/performance/capture_history.py cleanup --approved --output SAME_RUN_DIRECTORY -``` - -The ordinary controller cleanup first reads the matching-token canonical completion, before any process-birth check or attach. If present, it returns that original result without another remote operation, even if the process has since exited. Otherwise it checks birth, makes ONE distinct cleanup attachment and waits only for canonical completion, never a pre-existing `cleanup-result.json`. Remote cleanup likewise returns the active owner's `finish` result, or derives it from matching-token `complete.json`; absent completion stays explicitly unconfirmed. The operation-specific reply uses that returned status and cannot overwrite canonical completion with a later failure. - -The actual race exposed by root's invalidated profile calibration was: first cleanup restored 20 hooks but wrote only `cleanup-result.json`; its owner retired without `complete.json`; a second controller cleanup returned the old positive reply before its own remote operation ran; that operation then wrote `token-not-present`. Binding the completion path to RemoteCaptureRun deletes the caller-selected publication path and the controller's stale reply reader. The calibration also had a 49Hz profiler overrun of 48.5s and no speedscope artifact: it is not performance evidence or a meter-algorithm defect. - -Do not retry a start or kill the frontend. A blocked/uninterruptible target may not execute either the duration timer or remote cleanup. `CLEANUP_UNCONFIRMED` is a real possible result; keeping a recovery script is not proof hooks were restored. A missing token does not prove an uncertain attach had no effect. Read each capture's `cleanup`, failed/superseded hook counts and cap/error fields in canonical completion, not just a group `FINISHED` label. Preserve every original failed-run directory, request and recovery script; new source is not permission to replace or replay an old recovery protocol. - -Root still owes this correction's actual copied-bus/private-XDG qualification: normal duration finish; explicit early cleanup then cleanup again; failed setup then cleanup; scalar/output-bound controls; every hook restored; original human read ledger and XDG files unchanged. Root also owns the publication-failure control: occupy private `complete.json` with an empty directory, reach the existing remote cleanup operation so hooks restore once and rename fails visibly, remove only that private obstruction, then perform distinct cleanup and confirm the original status publishes and the owner retires without restoration replay. A foreign preexisting partial must remain untouched. The ordinary controller's canonical reader refuses that directory before attachment, so it cannot itself drive the obstructed finish; do not loosen the reader for this control. No author test, attach or performance completion is claimed. - -## Required private-process controls — UNRUN, root-owned - -Project rules prohibit authoring/running automated tests here. These are the precise controls root must implement/select in its existing combined execution; they are not passing test claims: - -- Real private event loop, actual `BoundedCapture`, fresh persistent output: Unicode scalar records at the exact comma/envelope boundary; assert final stat size equals `capture.output_bytes` and never exceeds byte cap. Repeat with count cap, oversize first record, no records and recording-encoding error. Include the final/error record and intermediate `.partial` size. -- Resolve valid sync, coroutine, inherited, classmethod and staticmethod declarations plus one invalid/missing/property target. Invalid descriptor/loop validation must occur before any assignment. Arrange a real partial-assignment failure in private control classes; assert reverse restoration and a bounded setup-error receipt. -- Start/finish/timer-finish twice; finish an async call before/after expiry and after cancellation. Assert descriptor binding, original return/exception/cancellation behavior, no late record resurrection, no retained product objects, timer cancellation, open-span censorship and identity retirement/reused-id distinction. -- Put a later wrapper over this capture and finish it; later wrapper remains installed and supersession is reported. A pre-existing capture must not be overwritten. Restore inherited declarations by deleting the owned override, not copying the inherited method onto the child. -- Run the exact remote entry/copy protocol on one private process with a real event loop, no Textual app: unsupported contract/provenance must fail before patching. For successful setup root can use its private real frontend path. Exercise setup failure after the first returned handle, INT/TERM controller cleanup and late completion. Never replace production owner algorithms with mocks. -- Check legacy trace array and frame summary callers, zero-display runs, multiple handlers for one input, and that no headless/batched/no-payload display creates a sample. Confirm live metadata paths cannot reach representation/heap/stall helpers. - -### Four source-review counterexamples — root's additional UNRUN selection - -- Drive the actual observation decoder with native prepared pages/cursors and wire prepared pages, `WirePage.unread`, live/historical/before/after cuts and unknown objects. Confirm the decoder uses only declared fields, emits explicit absent/unsupported evidence, and unrelated call/frame admission continues. Verify weak-label limitations remain visible rather than replaced by bare IDs. -- On the real private loop, nest `measuring_pause`, close normally, finish inside an open pause, and exhaust each cap before a pause closes. Verify closed pairs are bounded, the censored union clips at admission end, late ends do nothing, all counts/cleanup/errors survive summary, and missing switch data says `unobserved`. Cover the first frame and input-to-handoff intervals crossing exclusions, plus historical summaries with and without actual pause/switch arrays. -- Retain the verified source buffer, mutate the backing path and place conflicting bytecode beside it before module execution. Verify only those retained bytes execute, with correct `__file__`, `__name__` and `sys.modules` registration. Oversize and hash mismatch fail before any patch. Exercise this through the real copied remote protocol; do not replace a product algorithm. -- Let a real writer-thread span complete while the owner loop attempts finish. Both return/error/cancel and cap-exhaustion cases must admit/retire atomically: no admitted terminal record also counted censored, no negative open count, and no post-finish entry/completion resurrection. Include an actual async suspension and later-wrapper restoration. - -Root reported five private UTF-8/cap/oversize/empty/error-idempotence controls passing on parent `96a4e0d3cb561fe2efcca6b3905d08adeb20ae5d`. That is historical evidence, not qualification of this correction. No private process, product/meter import, test, capture or GUI operation was run by the author. Root still owes the combined controls and actual installed-window measurement; no speedup or dominant delay is established by this implementation. +# Historical history/frame observation — retired + +The running-process injection collector is deleted from current tools. Its +frozen source and original requests remain evidence for retained diagnoses; +the current host permissions forbid executing that injection. An earlier +approval or successful collector run is not launch authority. + +The current owner is [startup observation](physical-run.md). `StartOperation` +runs the distribution entrypoint in a new private frontend. `ObserveOperation` +owns module-location validation; its packets cannot select imported tools. +`MeasuredStartOperation` composes measurement with that same startup lifetime, +using the meter dependency frozen before launch. + +## Retained evidence + +The historical controller was `capture_history.py`; its remote collector was +`history_capture_remote.py`. Their complete source is retained at Toad +`644dd376242a64f62b5d1564ca14ca202150108d`, alongside the earlier version of +this document. On the development host, borrowers also retain the immutable +`platform-source/observer-head-644dd3762/tools/` archive under the performance +colleague's Core worktree. The older53e04c515 archive remains borrowed separately. +Neither archive is a current execution path. + +Original run directories, failed or uncertain input records, copied requests, +profiles, sessions and observations remain unchanged. Their timestamps and +first outcomes belong to their original runs. A new tool revision does not +rewrite those outcomes or authorize replay of their inputs. + +Xvfb isolates a display, not application data. Current physical runs use the +existing CheckRoot selection and a private HOME with private config, state and +data directories. A saved-tab database can retain foreign roots; it does not +travel by blind copying. Original human read markers, sessions, drafts and +undo state remain owned by their original application. diff --git a/docs/performance/physical-run.md b/docs/performance/physical-run.md index 52184b2db..6571f1012 100644 --- a/docs/performance/physical-run.md +++ b/docs/performance/physical-run.md @@ -137,7 +137,12 @@ module stays alive while the distribution entrypoint runs so its declared wire types do not acquire a different `__main__` namespace. `AcpObservationDecoder` is declared once per module, not rebuilt and registered on every observation. -The startup/shutdown programs retain the bounded immutable source snapshot. +The startup/shutdown programs retain the bounded immutable source snapshot, +including the measurement dependency when that capability is selected. Frozen +dependencies retain their package layout with owned package initializers, so +Python cannot merge them with another tools namespace. `ObserveOperation` owns +the module-location validator; observation packets contain resources and process +identity, not a tools directory or another authority to select imported code. Ordinary observations write only typed request/result data, not another copy of the program. Closing the controller pipe and the existing child lifetime retires this reader. Exceptional exit sends no further observation or physical input. diff --git a/docs/performance/scroll-series.md b/docs/performance/scroll-series.md index 66feb4578..c57b99e9a 100644 --- a/docs/performance/scroll-series.md +++ b/docs/performance/scroll-series.md @@ -140,7 +140,9 @@ meaning to that return: the known gap blocks visible read acknowledgement, the destination becomes visible through native paint, and source settlement removes the gap and retains the expected cursor. This corrects the immediate assertion above; the earlier failing run remains retained. The corrected case -has not yet been executed, so this trace is not a new passing-run claim. +passed once: 1 passed in16.21s, rc0, with the host lease acquired and released. +`pi-ui-performance/gap-contract-27/pytest.log` retains that exact execution and +its source snapshot; it is not a new physical or installed-app result. ## Historical series diff --git a/tests/textual/test_physical_run.py b/tests/textual/test_physical_run.py index bfb659b1b..4f458ef9d 100644 --- a/tests/textual/test_physical_run.py +++ b/tests/textual/test_physical_run.py @@ -1,11 +1,13 @@ """Physical frontend observation and admission contracts, using native geometry.""" from dataclasses import replace +from importlib.machinery import PathFinder from pathlib import Path import pytest from textual.geometry import Region -from tools.physical_ui import ScreenCapture +from tools.physical_ui import MeasuredTerminalApp, ObservedTerminalApp, ScreenCapture +from tools.physical_ui_candidate import MeasuredCandidateTerminalApp from agent_comms.child_process import ProcessIdentity from agent_comms.installer.probes.interpreter import InterpreterBuildProbe @@ -69,9 +71,8 @@ def test_startup_connection_decodes_only_observation_operations( physical_target: RunTarget, physical_resources: RunResources, ) -> None: identity = ProcessIdentity.current() - tools = Path(__file__).resolve().parents[2] / "tools" - for operation in (ObserveOperation(physical_resources, identity, tools), - AdmitOperation(physical_target, identity, tools)): + for operation in (ObserveOperation(physical_resources, identity), + AdmitOperation(physical_target, identity)): assert FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) == operation start = StartOperation(physical_resources, identity, (str(physical_target.runtime / "bin/toad"),), 9) strict = StartAdmittedOperation( @@ -86,6 +87,20 @@ def test_startup_connection_decodes_only_observation_operations( FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) +@pytest.mark.parametrize("app_type", (MeasuredTerminalApp, MeasuredCandidateTerminalApp)) +def test_measurement_dependency_resolves_inside_frozen_operation( + app_type: type[ObservedTerminalApp], physical_resources: RunResources, tmp_path: Path, +) -> None: + script = app_type.operation_file(StopRootOperation(physical_resources), tmp_path / "operation") + tools = PathFinder.find_spec("tools", [str(script.parent)]) + assert tools is not None and tools.has_location + performance = PathFinder.find_spec("tools.performance", tools.submodule_search_locations) + assert performance is not None and performance.has_location + meter = PathFinder.find_spec("tools.performance.frame_meter", performance.submodule_search_locations) + assert meter is not None and meter.origin is not None + assert Path(meter.origin).is_relative_to(script.parent) + + def test_current_observation_cannot_substitute_for_build_admission( physical_target: RunTarget, physical_observation: RunObservation, physical_admission: AdmittedResult, ) -> None: diff --git a/tools/performance/capture_history.py b/tools/performance/capture_history.py deleted file mode 100644 index 5c9fe42f2..000000000 --- a/tools/performance/capture_history.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Root-only, normal-user bounded history/frame observer. AUTHOR MUST NOT RUN. - -Example (fill current identity, approved runtime and reviewed meter hash): - python capture_history.py start --pid PID --boot BOOT --start-ticks TICKS \ - --runtime /absolute/runtime --meter /reviewed/frame_meter.py --sha256 HASH \ - --output /home/ts/.cache/agent-scratch/OWNER/p903-UNIQUE --approved - -Recovery (one distinct stop operation; never repeat an uncertain start): - python capture_history.py cleanup --output SAME_DIRECTORY --approved - -No screenshots, display ownership, GUI input, providers, services or heap walks. -A blocked target cannot promise remote restoration; retain cleanup.py and report it. -""" - -import argparse -import ast -from dataclasses import fields, replace -import json -import os -from pathlib import Path -import signal -import subprocess -import sys -import tempfile -import time -import uuid - -from agent_comms.nominal.field_codec import FieldCodec -from history_capture_remote import ( - CaptureOperation, CaptureRequest, CaptureSpec, ProcessIdentity, RemoteCaptureRun, StartCapture, - error_metadata, process_identity as identity, publish, read_meter_source, -) - - -# Ordinary helper import performs no attachment. Controller and target share -# the typed request, bounded source reader and publisher; only a copied request executes. - - -def validate_meter(path, digest): - raw = read_meter_source(path, digest) - tree = ast.parse(raw) - declarations = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)} - for name in ('install', 'trace_calls'): - if name not in declarations or not {'max_records', 'max_bytes', 'metadata_only'} <= { - argument.arg for argument in declarations[name].args.kwonlyargs}: - raise ValueError('UNSUPPORTED_BOUNDED_TRACE') - owners = [node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == 'BoundedCapture'] - if len(owners) != 1 or not any(isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == 'CONTRACT' for target in node.targets) - and isinstance(node.value, ast.Constant) and node.value.value == 1 for node in owners[0].body): - raise ValueError('UNSUPPORTED_BOUNDED_TRACE') - return raw - - -def attach(pid, script, capture): - # A fresh helper bounds a stuck sys.remote_exec itself. Killing this helper - # does NOT kill the frontend and does NOT prove its remote code did not run. - # The helper emits bounded exception metadata. A file avoids an unbounded - # parent-side PIPE buffer; only the bounded diagnostic enters our receipt. - with tempfile.TemporaryFile(dir=script.parent) as stderr: - try: - result = subprocess.run([sys.executable, str(Path(__file__).resolve()), 'attach', - '--pid', str(pid), '--script', str(script)], - stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, - stderr=stderr, timeout=capture.attach_timeout, check=False) - except (subprocess.TimeoutExpired, OSError) as error: - stderr.seek(0) - detail = stderr.read(capture.error_bytes).decode('utf-8', errors='replace') - raise RuntimeError(f'ATTACH_OUTCOME_UNCERTAIN: {detail} {type(error).__name__}: {str(error)[:capture.error_bytes]}') from error - if result.returncode: - stderr.seek(0) - detail = stderr.read(capture.error_bytes).decode('utf-8', errors='replace') - raise RuntimeError(f'ATTACH_OUTCOME_UNCERTAIN: helper exit {result.returncode}: {detail}') - - -def cleanup(directory): - request = FieldCodec.decode(CaptureRequest, json.loads((directory / 'cleanup.json').read_text())) - if (result := RemoteCaptureRun.read_completion(directory, request.token)) is not None: - return result - if identity(request.identity.pid) != request.identity: - raise RuntimeError('Process birth changed: do not attach') - capture = request.capture - attach(request.identity.pid, directory / 'cleanup.py', capture) - deadline = time.monotonic() + capture.cleanup_receipt_timeout - while time.monotonic() < deadline: - if (result := RemoteCaptureRun.read_completion(directory, request.token)) is not None: - return result - time.sleep(capture.poll_interval) - return dict(status='CLEANUP_UNCONFIRMED', reason='target did not publish matching canonical completion') - - -def start(args): - capture = FieldCodec.decode(CaptureSpec, {item.name: vars(args)[item.name] for item in fields(CaptureSpec)}) - directory = args.output.resolve() - if not directory.is_relative_to(Path('/home/ts/.cache/agent-scratch')) or directory.exists(): - raise ValueError('Use a fresh, persistent owned scratch run directory') - expected = ProcessIdentity(pid=args.pid, boot=args.boot, start_ticks=args.start_ticks) - if identity(args.pid) != expected: - raise RuntimeError('Process birth mismatch') - runtime = args.runtime.resolve(strict=True) - if runtime != args.runtime or not runtime.is_dir(): - raise ValueError('Runtime must be an explicit canonical directory') - maps = Path(f'/proc/{args.pid}/maps').read_text() - if str(runtime) + '/' not in maps: - raise ValueError('Expected runtime not mapped by requested process') - meter = args.meter.resolve(strict=True) - validate_meter(meter, args.sha256) # interface/hash failure BEFORE any attach - if not hasattr(sys, 'remote_exec'): - raise RuntimeError('UNSUPPORTED_BOUNDED_TRACE') - remote = Path(__file__).with_name('history_capture_remote.py').read_bytes() - # Charge both source copies, their actual path prefixes and bounded receipts. - metadata_size = 2 * len(remote) + 2 * 8192 + 6 * 65536 - directory.mkdir(mode=0o700) - request = CaptureRequest(identity=expected, runtime=runtime, meter=meter, - meter_sha256=args.sha256, contract=1, token=uuid.uuid4().hex, - capture=capture, operation=StartCapture()) - for operation in CaptureOperation.members_with(CaptureOperation): - script = directory / (operation.declared_name + '.py') - # sys.remote_exec supplies no __file__; each copy owns its adjacent request. - prefix = f'__file__ = {str(script)!r}\n'.encode('utf-8') - metadata_size += len(prefix) - if metadata_size > capture.metadata_max_bytes: - raise ValueError('Launcher metadata exceeds reservation') - request_path = script.with_suffix('.json') - value = FieldCodec.encode(replace(request, operation=operation())) - if len(json.dumps(value).encode('utf-8')) > 8192: - raise ValueError('Request exceeds reserved remote envelope') - publish(request_path, value) - script.write_bytes(prefix + remote) - script.chmod(0o600) - publish(directory / 'manifest.json', dict(identity=FieldCodec.encode(expected), runtime=str(runtime), - meter_sha256=args.sha256, capture=FieldCodec.encode(capture), - metadata_max_bytes=capture.metadata_max_bytes, - controller_budget=capture.controller_budget, - status='PREPARED_NOT_ATTACHED')) - began = time.monotonic() - try: - # Recheck birth immediately before the only start attempt. - if identity(args.pid) != expected: - raise RuntimeError('Process birth changed') - attach(args.pid, directory / 'start.py', capture) - deadline = began + capture.observation_budget - while time.monotonic() < deadline: - if (result := RemoteCaptureRun.read_completion(directory, request.token)) is not None: - publish(directory / 'controller-result.json', result) - return - failure = directory / 'start-result.json' - if failure.exists(): - raise RuntimeError('Remote setup failed: ' + failure.read_text()[:capture.error_bytes]) - time.sleep(capture.poll_interval) - raise TimeoutError('Capture did not finish by controller deadline') - except BaseException as error: - try: - result = cleanup(directory) # a stop, NOT replay of the start input - except BaseException as stop_error: - result = dict(status='CLEANUP_UNCONFIRMED', **error_metadata(stop_error)) - publish(directory / 'controller-result.json', dict(status='INCONCLUSIVE', - **error_metadata(error), cleanup=result, recovery_script=str(directory / 'cleanup.py'))) - raise - - -def interrupted(_signal, _frame): - raise InterruptedError('Controller interrupted') - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('operation', choices=(*(member.declared_name for member in - CaptureOperation.members_with(CaptureOperation)), 'attach')) - parser.add_argument('--approved', action='store_true', help='root reviewed this exact source') - parser.add_argument('--pid', type=int) - parser.add_argument('--boot') - parser.add_argument('--start-ticks', type=int) - parser.add_argument('--runtime', type=Path) - parser.add_argument('--meter', type=Path) - parser.add_argument('--sha256') - parser.add_argument('--output', type=Path) - parser.add_argument('--script', type=Path) - for item in fields(CaptureSpec): - parser.add_argument('--' + item.name.replace('_', '-'), type=item.type, default=item.default) - args = parser.parse_args() - if os.geteuid() == 0: - parser.error('Normal-user observation only; never sudo this launcher') - if args.operation == 'attach': - try: - sys.remote_exec(args.pid, str(args.script)) - except BaseException as error: - sys.stderr.write(json.dumps(error_metadata(error), ensure_ascii=False)) - raise SystemExit(1) from None - return - if not args.approved: - parser.error('Root must review source and explicitly approve execution') - signal.signal(signal.SIGTERM, interrupted) - signal.signal(signal.SIGINT, interrupted) - if args.operation == 'cleanup': - print(json.dumps(cleanup(args.output.resolve()), separators=(',', ':'))) - else: - if any(value is None for value in (args.pid, args.boot, args.start_ticks, args.runtime, - args.meter, args.sha256, args.output)): - parser.error('Start needs every explicit identity/runtime/meter/hash/output field') - start(args) - - -if __name__ == '__main__': - main() diff --git a/tools/performance/history_capture_remote.py b/tools/performance/history_capture_remote.py deleted file mode 100644 index 44275e8e3..000000000 --- a/tools/performance/history_capture_remote.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Root's remote start/cleanup script; copied beside an immutable typed request. - -Executed by normal-user sys.remote_exec, never by an author or from a renderer. -No UI input, provider traffic, snapshot, product repr or heap traversal. -""" - -from abc import abstractmethod -import asyncio -from dataclasses import dataclass, fields -import hashlib -import importlib.machinery -import importlib.util -import json -import math -import os -from pathlib import Path -import sys -import time -from typing import ClassVar - -from agent_comms.nominal.declared_family import DeclaredFamily -from agent_comms.nominal.field_codec import FieldCodec - - -STATUS_LIMIT = 65536 -REGISTRY = '_toad_history_captures' -METER_SOURCE_LIMIT = 256 << 10 - - -@dataclass(frozen=True) -class CaptureSpec: - """One immutable selection for both collectors and their enclosing lifetime. - - Scalar shape/positivity is checked at construction. BoundedCapture remains - the owner of physical admission ceilings and exact encoded byte accounting. - """ - - seconds: float = 25.0 - frames_max_records: int = 20000 - calls_max_records: int = 20000 - frames_max_bytes: int = 3 << 20 - calls_max_bytes: int = 12 << 20 - - attach_timeout: ClassVar[float] = 5.0 - completion_grace: ClassVar[float] = 3.0 - cleanup_receipt_timeout: ClassVar[float] = 2.0 - poll_interval: ClassVar[float] = .25 - error_bytes: ClassVar[int] = 2048 - metadata_max_bytes: ClassVar[int] = 1 << 20 - - def __post_init__(self): - # This is the foreign CLI/JSON scalar boundary, not product dispatch. - for item in fields(self): - value = getattr(self, item.name) - if type(value) is not item.type or not math.isfinite(value) or value <= 0: - raise ValueError(f'Invalid positive finite capture scalar: {item.name}') - - @property - def observation_budget(self): - return self.attach_timeout + self.seconds + self.completion_grace - - @property - def cleanup_budget(self): - return self.attach_timeout + self.cleanup_receipt_timeout - - @property - def controller_budget(self): - return self.observation_budget + self.cleanup_budget - - -@dataclass(frozen=True) -class ProcessIdentity: - pid: int - boot: str - start_ticks: int - - -class CaptureOperation(DeclaredFamily, affix='Capture'): - """Each copied request declares which operation its script may execute.""" - - @abstractmethod - def execute(self, request: 'CaptureRequest', directory: Path): ... - - -@dataclass(frozen=True) -class CaptureRequest: - identity: ProcessIdentity - runtime: Path - meter: Path - meter_sha256: str - contract: int - token: str - capture: CaptureSpec - operation: CaptureOperation - - -def error_metadata(error): - """Bound exception diagnostics; never serialize a product object or traceback.""" - detail = str(error).encode('utf-8', errors='replace')[:CaptureSpec.error_bytes] - return dict(error=type(error).__name__[:96], detail=detail.decode('utf-8', errors='ignore')) - - -def read_meter_source(path, digest): - """The approved source buffer, with one bounded read and one identity check.""" - with path.open('rb') as source: - raw = source.read(METER_SOURCE_LIMIT + 1) - if len(raw) > METER_SOURCE_LIMIT: - raise ValueError('Oversize meter source') - if hashlib.sha256(raw).hexdigest() != digest: - raise ValueError('Approved meter hash mismatch') - return raw - - -def process_identity(pid=None): - pid = os.getpid() if pid is None else pid - stat = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() - return ProcessIdentity(pid=pid, boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip(), - start_ticks=int(stat[19])) - - -def publish(path, value): - data = json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode('utf-8') - if len(data) > STATUS_LIMIT: - raise ValueError('Manifest exceeds reserved metadata bound') - partial = path.with_name(path.name + '.partial') - fd = os.open(partial, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - # Only successful exclusive creation grants this attempt cleanup ownership. - try: - with os.fdopen(fd, 'wb') as file: - file.write(data) - os.rename(partial, path) - except BaseException as error: - try: - partial.unlink(missing_ok=True) - except BaseException as cleanup_error: - raise error from cleanup_error - raise - - -def require_identity(request): - if process_identity() != request.identity: - raise RuntimeError('Capture process birth mismatch') - runtime = request.runtime.resolve(strict=True) - if runtime != request.runtime: - raise ValueError('Runtime must be an explicit canonical directory') - return runtime - - -def require_modules(runtime): - """Read the foreign ModuleSpec's origin or every namespace search location. - - Locations establish provenance, NOT hashes of already-loaded code. No - package is imported or excused by its spelling at this boundary. - """ - checked = 0 - for name, module in tuple(sys.modules.items()): - if name.split('.', 1)[0] not in ('toad', 'textual', 'agent_comms'): - continue - spec = getattr(module, '__spec__', None) # foreign Python module boundary - if not isinstance(spec, importlib.machinery.ModuleSpec): - raise RuntimeError(f'Loaded package has no ModuleSpec: {name[:160]}') - locations = ((spec.origin,) if spec.has_location else - tuple(spec.submodule_search_locations or ())) - if not locations: - raise RuntimeError(f'Loaded package has no owned locations: {name[:160]}') - for location in locations: - resolved = Path(location).resolve(strict=True) - if not resolved.is_relative_to(runtime): - raise RuntimeError(f'Loaded package outside approved runtime: {name[:160]} {str(resolved)[:384]}') - checked += 1 - if not checked or 'toad.app' not in sys.modules: - raise RuntimeError('No loaded Toad runtime to observe') - return checked - - -class RemoteCaptureRun: - """The one token's returned handles and group timer, never product objects.""" - - def __init__(self, token, module, registry, directory): - self.token, self.module, self.registry = token, module, registry - self.completion_path = self.completion_file(directory) - self.handles = [] - self.timer = None - self.status = None - - @staticmethod - def completion_file(directory): - return directory / 'complete.json' - - @classmethod - def read_completion(cls, directory, token): - """Only this run's bounded canonical completion can prove it finished.""" - path = cls.completion_file(directory) - if not path.exists(): - return None - with path.open('rb') as source: - raw = source.read(STATUS_LIMIT + 1) - if len(raw) > STATUS_LIMIT: - raise ValueError('Oversize canonical completion') - result = json.loads(raw) - return result if result.get('token') == token else None - - def finish(self, reason, **diagnostic): - if self.status is None: - if self.timer is not None: - self.timer.cancel() - self.timer = None - results = [] - for finish in reversed(self.handles): - try: - results.append(finish(reason)) - except BaseException as error: - results.append(dict(cleanup='failed', **error_metadata(error))) - self.handles.clear() - successful = reason != 'setup-error' and all( - result.get('cleanup') == 'restored' and not result.get('error') - and not result.get('cap_exhausted') for result in results) - self.status = dict(status='FINISHED' if successful else 'INCONCLUSIVE', - reason=reason, token=self.token, captures=results, - attribution='INCONCLUSIVE_UNTIL_PAGE_CORRELATION', - pixels_observed=False, **diagnostic) - # Keep the completed owner reachable if publication fails. A later - # distinct cleanup may publish its original status without restoring twice. - publish(self.completion_path, self.status) - self.registry.pop(self.token, None) - sys.modules.pop(self.module, None) - return self.status - - -@dataclass(frozen=True) -class CleanupCapture(CaptureOperation): - def execute(self, request, directory): - require_identity(request) - registry = getattr(sys, REGISTRY, {}) - entry = registry.get(request.token) - if entry is not None: - return entry.finish('controller-cleanup') - if (result := RemoteCaptureRun.read_completion(directory, request.token)) is not None: - return result - # Missing token is NOT evidence that an uncertain attach did nothing. - return dict(status='CLEANUP_UNCONFIRMED', reason='token-not-present') - - -@dataclass(frozen=True) -class StartCapture(CaptureOperation): - def execute(self, request, directory): - runtime = require_identity(request) - loop = asyncio.get_running_loop() # never set up in a synchronous child - checked = require_modules(runtime) - meter_path = request.meter.resolve(strict=True) - source = read_meter_source(meter_path, request.meter_sha256) - module_name = '_toad_history_meter_' + request.token - registry = getattr(sys, REGISTRY, None) - if registry is None: - registry = {} - setattr(sys, REGISTRY, registry) - if registry: # no parallel capture and no overwrite of uncertain retained handles - raise RuntimeError('An existing history capture still owns handles') - spec = importlib.util.spec_from_file_location(module_name, meter_path) - meter = importlib.util.module_from_spec(spec) - sys.modules[module_name] = meter - entry = RemoteCaptureRun(request.token, module_name, registry, directory) - registry[request.token] = entry - handles = entry.handles - capture = request.capture - try: - # Execute precisely the hash-verified bytes. Never reopen this mutable - # path through the loader or accept its cached bytecode instead. - exec(compile(source, str(meter_path), 'exec', dont_inherit=True), vars(meter)) - if meter.BoundedCapture.CONTRACT != request.contract: - raise RuntimeError('UNSUPPORTED_BOUNDED_TRACE') - # Validate ALL actual methods/metadata adapters before either collector patches. - targets = tuple(meter.CallMetadata.targets) - meter._metadata_boundary() # resolve all nominal metadata declarations before patching - for target in targets: - meter.MethodTarget(target) - checked = require_modules(runtime) # include any modules target resolution loaded - handles.append(meter.install(expected_pid=request.identity.pid, seconds=capture.seconds, - output=str(directory / 'frames.json'), max_records=capture.frames_max_records, - max_bytes=capture.frames_max_bytes, metadata_only=True)) - handles.append(meter.trace_calls(expected_pid=request.identity.pid, seconds=capture.seconds, - output=str(directory / 'calls.json'), targets=targets, - max_records=capture.calls_max_records, - max_bytes=capture.calls_max_bytes, metadata_only=True)) - - entry.timer = loop.call_later(capture.seconds, entry.finish, 'duration') - publish(directory / 'entered.json', dict(status='ENTERED', identity=FieldCodec.encode(process_identity()), - modules_checked=checked, runtime=str(runtime), - meter_sha256=request.meter_sha256, - provenance='loaded module locations, not loaded-code hashes', - started_ns=time.monotonic_ns(), capture=FieldCodec.encode(capture), - source_match='not-required; infer only from observed page spans')) - except BaseException as error: - return entry.finish('setup-error', **error_metadata(error)) - - -def execute(): - script = Path(__file__).resolve() - status = script.parent / (script.stem + '-result.json') - try: - request_path = script.with_suffix('.json') - if request_path.stat().st_size > 8192: - raise ValueError('Oversize capture request') - request = FieldCodec.decode(CaptureRequest, json.loads(request_path.read_text())) - result = request.operation.execute(request, script.parent) - except BaseException as error: - result = dict(status='FAILED_CLEANUP_UNCONFIRMED', **error_metadata(error)) - if result is not None: - publish(status, result) - - -# Only the copied execution scripts have their explicit request beside them. -# Ordinary import by the controller has no request and performs no remote work. -if Path(__file__).with_suffix('.json').is_file(): - execute() diff --git a/tools/physical_ui.py b/tools/physical_ui.py index 11933aa83..27c0c70f9 100644 --- a/tools/physical_ui.py +++ b/tools/physical_ui.py @@ -253,6 +253,23 @@ class OperationProgram: path: Path source: bytes + @property + def relative_path(self) -> Path: + """Retain the source package layout in this startup's frozen inputs.""" + return self.path.relative_to(Path(__file__).resolve().parents[1]) + + def freeze(self, directory: Path) -> None: + """Freeze a dependency as an owned package, never a merged namespace.""" + relative = self.relative_path + dependency = directory / relative + dependency.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + for package in relative.parents: + if package.parts: + initializer = directory / package / "__init__.py" + if not initializer.exists(): + ExclusiveAtomicFile(initializer).write_bytes(b"") + ExclusiveAtomicFile(dependency).write_bytes(self.source) + @classmethod def capture(cls, path: Path): with path.open("rb") as source: @@ -398,9 +415,7 @@ def operation_file(cls, operation: PhysicalOperation, directory: Path) -> Path: # controller lifetime. Later source edits cannot change a run's wire. programs = cls.operation_programs() for program in programs[:-1]: - package = directory / program.path.parent.name - package.mkdir(mode=0o700, exist_ok=True) - ExclusiveAtomicFile(package / program.path.name).write_bytes(program.source) + program.freeze(directory) raw = programs[-1].source script = directory / "operation.py" # Only startup/shutdown executes a program. Normal Python supplies its @@ -500,7 +515,7 @@ def _observe(self, operation: type[ObserveOperation]): deadline = time.monotonic() + PhysicalOperation.timeout while True: self.terminal.platform.require(self.frontend) - packet = self._packet(operation(self.target, self.frontend, Path(__file__).resolve().parent)) + packet = self._packet(operation(self.target, self.frontend)) # Fresh read-only observations may see an ACP which has not yet # published its root. No physical input is sent until it does; # foreign resources and failed input are never retried here. diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index 2d2f50e2c..cdf85e3b2 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -10,6 +10,7 @@ from functools import partial from dataclasses import dataclass from importlib.metadata import distribution +import importlib.machinery import json from multiprocessing.connection import Connection import os @@ -20,6 +21,11 @@ from threading import Thread from typing import ClassVar, Self +# Only a newly launched program adds its frozen dependency directory. An +# observation packet cannot change import paths inside an existing frontend. +if __name__ == "__main__": + sys.path.insert(0, str(Path(__file__).parent)) + from agent_comms.acp_extension import CoordinationChangedUpdate from agent_comms.child_process import ChildOutcome, ProcessIdentity from agent_comms.launch_runtime import LaunchRuntime @@ -401,7 +407,7 @@ def startup_observed(self, directory: Path): yield def measure(self, directory: Path, cleanup: ExitStack, lifetime) -> None: - from performance.frame_meter import install + from tools.performance.frame_meter import install # install resolves and validates every target before changing any hook; # its returned closer restores them and publishes bounded output. @@ -448,21 +454,38 @@ def coordinated(self, value): class ObserveOperation(PhysicalOperation): target: RunResources identity: ProcessIdentity - tools: Path @classmethod def packet_path(cls, directory: Path) -> Path: """The read-only observation protocol owns its one packet filename.""" return directory / "request.json" - @contextmanager - def tool_imports(self): - previous = sys.path[:] - try: - sys.path.insert(0, str(self.tools)) - yield - finally: - sys.path[:] = previous + def require_modules(self) -> int: + """Observe foreign ModuleSpec origins through this startup's validator. + + Locations establish provenance, not hashes of already-loaded code. + Namespace packages must supply every owned search location. + """ + runtime = self.target.runtime + checked = 0 + for name, module in tuple(sys.modules.items()): + if name.split('.', 1)[0] not in ('toad', 'textual', 'agent_comms'): + continue + spec = getattr(module, '__spec__', None) # foreign Python module boundary + if not isinstance(spec, importlib.machinery.ModuleSpec): + raise RuntimeError(f'Loaded package has no ModuleSpec: {name[:160]}') + locations = ((spec.origin,) if spec.has_location else + tuple(spec.submodule_search_locations or ())) + if not locations: + raise RuntimeError(f'Loaded package has no owned locations: {name[:160]}') + for location in locations: + resolved = Path(location).resolve(strict=True) + if not resolved.is_relative_to(runtime): + raise RuntimeError(f'Loaded package outside approved runtime: {name[:160]} {str(resolved)[:384]}') + checked += 1 + if not checked or 'toad.app' not in sys.modules: + raise RuntimeError('No loaded Toad runtime to observe') + return checked def observed_result(self, observed: RunObservation, directory: Path) -> PhysicalResult: return ObservedResult(observed) @@ -473,10 +496,6 @@ def execute(self, directory: Path) -> None: raise ValueError("Observed frontend incarnation changed") if Path.home().resolve() != self.target.home: raise ValueError("Observed HOME is not private") - # The existing reviewed validator remains the single module-location policy. - # The temporary tool import path is retired before inspecting application state. - with self.tool_imports(): - from performance.history_capture_remote import require_modules import agent_comms import textual import toad @@ -494,7 +513,7 @@ def execute(self, directory: Path) -> None: screen = app.screen except (NoActiveAppError, ScreenStackError) as error: raise PendingRun("No sole running Toad application with a screen yet") from error - checked = require_modules(self.target.runtime) + checked = self.require_modules() access = app.coordination_access root = access.selection.root observation = access.observation From e07f4791ec6dda7da142ec46206964cc2fb73f96 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 11:29:34 -0500 Subject: [PATCH 33/35] Derive observation module identities from their imported packages --- tests/textual/test_physical_run.py | 13 +++++++++++++ tools/physical_ui_remote.py | 14 ++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/textual/test_physical_run.py b/tests/textual/test_physical_run.py index 4f458ef9d..015e5b8ab 100644 --- a/tests/textual/test_physical_run.py +++ b/tests/textual/test_physical_run.py @@ -1,6 +1,7 @@ """Physical frontend observation and admission contracts, using native geometry.""" from dataclasses import replace from importlib.machinery import PathFinder +import json from pathlib import Path import pytest @@ -87,6 +88,18 @@ def test_startup_connection_decodes_only_observation_operations( FieldCodec.decode(ObserveOperation, FieldCodec.encode(operation)) +def test_module_observation_uses_actual_package_locations(physical_resources: RunResources) -> None: + # Exercise real stdlib ModuleSpecs without claiming a running Toad admission. + package_root = Path(json.__file__).resolve().parent + inside = replace(physical_resources, runtime=package_root) + identity = ProcessIdentity.current() + assert ObserveOperation(inside, identity).require_modules((json,)) > 0 + with pytest.raises(RuntimeError, match="outside approved runtime"): + ObserveOperation(physical_resources, identity).require_modules((json,)) + with pytest.raises(RuntimeError, match="No loaded runtime packages"): + ObserveOperation(inside, identity).require_modules(()) + + @pytest.mark.parametrize("app_type", (MeasuredTerminalApp, MeasuredCandidateTerminalApp)) def test_measurement_dependency_resolves_inside_frozen_operation( app_type: type[ObservedTerminalApp], physical_resources: RunResources, tmp_path: Path, diff --git a/tools/physical_ui_remote.py b/tools/physical_ui_remote.py index cdf85e3b2..c43afe629 100644 --- a/tools/physical_ui_remote.py +++ b/tools/physical_ui_remote.py @@ -19,6 +19,7 @@ import sys import time from threading import Thread +from types import ModuleType from typing import ClassVar, Self # Only a newly launched program adds its frozen dependency directory. An @@ -460,18 +461,19 @@ def packet_path(cls, directory: Path) -> Path: """The read-only observation protocol owns its one packet filename.""" return directory / "request.json" - def require_modules(self) -> int: + def require_modules(self, packages: tuple[ModuleType, ...]) -> int: """Observe foreign ModuleSpec origins through this startup's validator. Locations establish provenance, not hashes of already-loaded code. Namespace packages must supply every owned search location. """ runtime = self.target.runtime + package_names = frozenset(package.__name__ for package in packages) checked = 0 for name, module in tuple(sys.modules.items()): - if name.split('.', 1)[0] not in ('toad', 'textual', 'agent_comms'): + if name.partition('.')[0] not in package_names: continue - spec = getattr(module, '__spec__', None) # foreign Python module boundary + spec = module.__spec__ # foreign Python module boundary if not isinstance(spec, importlib.machinery.ModuleSpec): raise RuntimeError(f'Loaded package has no ModuleSpec: {name[:160]}') locations = ((spec.origin,) if spec.has_location else @@ -483,8 +485,8 @@ def require_modules(self) -> int: if not resolved.is_relative_to(runtime): raise RuntimeError(f'Loaded package outside approved runtime: {name[:160]} {str(resolved)[:384]}') checked += 1 - if not checked or 'toad.app' not in sys.modules: - raise RuntimeError('No loaded Toad runtime to observe') + if not checked: + raise RuntimeError('No loaded runtime packages to observe') return checked def observed_result(self, observed: RunObservation, directory: Path) -> PhysicalResult: @@ -513,7 +515,7 @@ def execute(self, directory: Path) -> None: screen = app.screen except (NoActiveAppError, ScreenStackError) as error: raise PendingRun("No sole running Toad application with a screen yet") from error - checked = self.require_modules() + checked = self.require_modules((toad, textual, agent_comms)) access = app.coordination_access root = access.selection.root observation = access.observation From 7039ec7eab19375b93ed9c9efb97838cd0331ac9 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 11:40:37 -0500 Subject: [PATCH 34/35] Publish the status check required by main protection --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad1a06c3d..c58cd5efb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ concurrency: cancel-in-progress: true jobs: nra-gate: - name: NRA gate + name: Debt ratchet timeout-minutes: 5 runs-on: ubuntu-latest permissions: From db8a6aad70f0dec210a2315a6f53162ebe6d62f5 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Mon, 2 Nov 2026 13:03:56 -0500 Subject: [PATCH 35/35] Keep NRA gate aligned with the directed protection setting --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c58cd5efb..ad1a06c3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ concurrency: cancel-in-progress: true jobs: nra-gate: - name: Debt ratchet + name: NRA gate timeout-minutes: 5 runs-on: ubuntu-latest permissions: