From f8ea408d3d3c74ee7c59dbb9ced7ccb389dede4a Mon Sep 17 00:00:00 2001 From: Agent Zero Date: Sat, 1 Aug 2026 23:20:20 +0100 Subject: [PATCH 1/3] fix(ci): sample-e2e on the public upstream image + fast red-proof gate sample-plugin-e2e pinned A0_IMAGE to the PRIVATE fork image and passed GHCR_PULL_TOKEN. Dependabot runs and fork PRs never receive secrets, so the pull 401d and the workflow was red for everyone outside the org -- which would be every external contributor once this repo is public. Switch to the DEC-019 default docker.io/agent0ai/agent-zero:latest (public, no creds). Consumers needing the ML-bundled fork still override A0_IMAGE per-plugin via .devkit.yml. Red-proof (Gate 3): the binding clock is Playwright test-level timeout, not action/expect timeouts -- a step passing an explicit per-call timeout cannot be overridden from config but IS capped by the test timeout. Measured 12 x ~113s = 22m40s. Cap the test timeout for the red-proof pass only; disable video/screenshot/trace capture there (expected failures, artifacts nobody reads). Semantics unchanged: every scenario still runs, still exactly 0 passes. Two guards make weakening impossible to miss -- Guard A asserts red-proof and real run cover the same scenario count; Guard B fails if the slowest genuinely-passing scenario exceeds 80%% of the cap (a cap that could clip an honest pass could mask a fake-green). Also restores the exec bit on run-bdd.sh (text_editor clears it). --- .github/workflows/sample-plugin-e2e.yml | 14 ++--- e2e/bdd/playwright.config.ts | 14 +++-- e2e/harness/run-bdd.sh | 69 +++++++++++++++++++++++-- e2e/node_modules | 1 + 4 files changed, 85 insertions(+), 13 deletions(-) create mode 120000 e2e/node_modules diff --git a/.github/workflows/sample-plugin-e2e.yml b/.github/workflows/sample-plugin-e2e.yml index 383b27c..e7b37d9 100644 --- a/.github/workflows/sample-plugin-e2e.yml +++ b/.github/workflows/sample-plugin-e2e.yml @@ -35,10 +35,12 @@ jobs: run: podman build -t plugin-devkit:ci -f devcontainer/Containerfile devcontainer - name: Package sample + run lifecycle (A0 nested, unprivileged) - # Validates DEC-055 fork-first: boots the PRIVATE non-root fork image - # (ghcr.io/nuevanext/agent-zero:latest-nonroot) via the harness ghcr login. - env: - GHCR_TOKEN: ${{ secrets.GHCR_PULL_TOKEN }} + # Boots the PUBLIC upstream image (docker.io/agent0ai/agent-zero:latest) — + # the A0_IMAGE default from DEC-019. This job must pass with NO secrets: + # dependabot runs and fork PRs from outside contributors never receive + # them, so pinning the private fork image here made this workflow red for + # everyone outside the org. Consumers that need the ML-bundled fork still + # override A0_IMAGE per-plugin via .devkit.yml. run: | set -euo pipefail mkdir -p dist "$GITHUB_WORKSPACE/artifacts" @@ -56,9 +58,7 @@ jobs: -e PLUGIN_NAME=sample_plugin \ -e HOOK_DIR=/workspace/examples/sample-plugin/tests/e2e/hooks \ -e BEHAVIOUR_FILE=/workspace/examples/sample-plugin/tests/e2e/behaviour.mjs \ - -e A0_IMAGE="ghcr.io/nuevanext/agent-zero:latest-nonroot" \ - -e GHCR_USER=x \ - -e GHCR_TOKEN="$GHCR_TOKEN" \ + -e A0_IMAGE="docker.io/agent0ai/agent-zero:latest" \ -e ARTIFACT_DIR=/artifacts \ plugin-devkit:ci \ -lc 'bash /workspace/e2e/harness/run-lifecycle.sh' diff --git a/e2e/bdd/playwright.config.ts b/e2e/bdd/playwright.config.ts index fe830f9..3320949 100644 --- a/e2e/bdd/playwright.config.ts +++ b/e2e/bdd/playwright.config.ts @@ -9,6 +9,14 @@ const testDir = defineBddConfig({ features: ["features/**/*.feature", ...(P ? [`${P}/features/**/*.feature`] : [])], steps: ["bdd-fixtures.ts", "steps/**/*.ts", ...(P ? [`${P}/steps/**/*.ts`] : [])], }); +// Red-proof mode (Gate 3, DEC-066 / DEC-070): the suite runs with the plugin NOT +// installed, so EVERY scenario is expected to fail. Capturing video + a +// retain-on-failure trace for every one of those expected failures is pure I/O +// cost for artifacts nobody reads (the gate's signal is the pass COUNT, not the +// artifacts). Disable capture in red-proof mode only — the real run below is +// unaffected, so debuggability of genuine failures is unchanged. +const RED_PROOF = process.env.BDD_SKIP_INSTALL === "1"; + export default defineConfig({ testDir, reporter: [["list"]], timeout: 120000, workers: 1, fullyParallel: false, // trace = the rich single-file artifact (network + DOM snapshots + console + video + timeline), @@ -16,8 +24,8 @@ export default defineConfig({ // (retain-on-failure); set BDD_TRACE=on (the workflow's capture-all-traces dispatch input) for every one. use: { baseURL: process.env.A0_BASE || "http://localhost:8099", - video: "on", - screenshot: "on", - trace: (process.env.BDD_TRACE as any) || "retain-on-failure", + video: RED_PROOF ? "off" : "on", + screenshot: RED_PROOF ? "off" : "on", + trace: RED_PROOF ? "off" : ((process.env.BDD_TRACE as any) || "retain-on-failure"), }, }); diff --git a/e2e/harness/run-bdd.sh b/e2e/harness/run-bdd.sh index 69390a2..0117ac4 100755 --- a/e2e/harness/run-bdd.sh +++ b/e2e/harness/run-bdd.sh @@ -53,13 +53,27 @@ export A0_USERNAME="${A0_USERNAME:-admin}" A0_PASSWORD="${A0_PASSWORD:-admin}" # plugin, the behaviour scenarios must NOT pass — the seam endpoint 404s without the # plugin, so honest scenarios go RED. Any pass here is fake-green. Cheap: one extra # playwright pass, no second A0 boot. +# SPEED (DEC-070): the red-proof's cost is ENTIRELY in the fail path. With no +# plugin the seam 404s, so every scenario runs until something times out — and +# the binding clock is Playwright's TEST-level timeout (120s), not the action / +# expect timeouts: a step's explicit per-call `{ timeout: N }` (which plugin +# steps routinely pass) is NOT overridable from config, but IS capped by the +# test timeout. Measured: 12 scenarios x ~113s = 22m40s ~= 12 x the 120s cap. +# So cap the test timeout for the red-proof pass only. Semantics are untouched: +# every scenario still runs, and the assertion is still exactly 0 passes. +RED_PROOF_TIMEOUT_MS="${RED_PROOF_TIMEOUT_MS:-30000}" RP_LOG="$(mktemp)" -( cd "$BDD" && BDD_SKIP_INSTALL=1 npx playwright test --config=playwright.config.ts ) > "$RP_LOG" 2>&1 || true +rp_t0=$SECONDS +( cd "$BDD" && BDD_SKIP_INSTALL=1 npx playwright test --config=playwright.config.ts \ + --timeout="$RED_PROOF_TIMEOUT_MS" ) > "$RP_LOG" 2>&1 || true +rp_secs=$(( SECONDS - rp_t0 )) # NB: 0-passed is the SUCCESS case and prints no "N passed" line, so BOTH greps # exit non-zero on empty input; with pipefail that propagates, so `|| true` must # guard the WHOLE pipeline (not just the first grep) or set -e kills the script. RP_PASSED=$(grep -oE '[0-9]+ passed' "$RP_LOG" | grep -oE '[0-9]+' | head -1 || true); RP_PASSED=${RP_PASSED:-0} -echo "[run-bdd] red-proof: $RP_PASSED scenario(s) passed with NO plugin installed (want 0)" +RP_TOTAL=$(grep -oE 'Running [0-9]+ test' "$RP_LOG" | grep -oE '[0-9]+' | head -1 || true); RP_TOTAL=${RP_TOTAL:-0} +echo "[run-bdd] red-proof: $RP_PASSED scenario(s) passed with NO plugin installed (want 0) \ +— $RP_TOTAL scenario(s) run in ${rp_secs}s (test-timeout cap ${RED_PROOF_TIMEOUT_MS}ms)" if [ "$RP_PASSED" -gt 0 ]; then echo "::error::seam-off red-proof FAILED — $RP_PASSED behaviour scenario(s) passed with no plugin installed (fake-green). See tests/_testkit/docs/BDD-GATES.md (Gate 3)." grep -aE '✓|passed|failed' "$RP_LOG" | tail -20 @@ -68,7 +82,56 @@ fi echo "[run-bdd] red-proof OK — nothing passes without the plugin; the suite is real." PW_RC=0 -( cd "$BDD" && npx playwright test --config=playwright.config.ts ) || PW_RC=$? +PW_LOG="$(mktemp)" +( cd "$BDD" && npx playwright test --config=playwright.config.ts ) 2>&1 | tee "$PW_LOG" || true +PW_RC=${PIPESTATUS[0]} + +# ── Anti-weakening guards on the speed-up (DEC-070) ─────────────────────────── +# Capping the red-proof clock is only safe while a scenario that WOULD pass +# plugin-less still has room to pass. If the cap clipped such a scenario into a +# timeout-failure, the gate would report 0-passed and go green while a fake-green +# scenario sat undetected — a silently weakened gate. Two checks make that +# impossible to happen unnoticed. +PW_TOTAL=$(grep -oE 'Running [0-9]+ test' "$PW_LOG" | grep -oE '[0-9]+' | head -1 || true); PW_TOTAL=${PW_TOTAL:-0} + +# Guard A — coverage parity: the red-proof must exercise the SAME scenario count +# as the real run. Catches any future change that narrows the red-proof pass +# (a filter, a project split, a grep) rather than merely speeding it up. +if [ "$RP_TOTAL" -ne "$PW_TOTAL" ]; then + echo "::error::red-proof coverage MISMATCH — red-proof ran $RP_TOTAL scenario(s) but the real run ran $PW_TOTAL. The red-proof must cover every behaviour scenario (DEC-066/070)." + exit 1 +fi + +# Guard B — cap adequacy: every scenario that genuinely PASSES in the real run +# must complete well inside the red-proof cap. If the slowest honest pass needs +# more time than the cap allows, the cap is capable of masking a fake-green, so +# fail loudly with the value to raise it to. 80% leaves headroom for the extra +# waiting a plugin-less run does before its own failure. +# Parse the per-scenario durations Playwright prints at end-of-line, e.g. "(18.5s)". +# POSIX sed + shell arithmetic ONLY — deliberately no gawk-style 3-arg match(): +# the harness runs wherever the devkit image runs and mawk (Debian's default awk) +# rejects that GNU extension outright, which would make this guard a syntax error +# at runtime rather than a check. Verified against real CI log output. +SLOWEST_PASS_MS=0 +while read -r v u; do + case "$u" in + ms) ms=${v%.*} ;; + s) ms=$(( ${v%%.*} * 1000 + 10#$(printf '%s' "${v#*.}0" | cut -c1-1) * 100 )) ;; + m) ms=$(( ${v%%.*} * 60000 )) ;; + *) ms=0 ;; + esac + [ "$ms" -gt "$SLOWEST_PASS_MS" ] && SLOWEST_PASS_MS=$ms +done <= ${need}." + exit 1 +fi # One Playwright trace.zip per captured scenario → artifacts (network + DOM snapshots + # console + video + timeline in one file; open with `npx playwright show-trace ` or diff --git a/e2e/node_modules b/e2e/node_modules new file mode 120000 index 0000000..64fc00e --- /dev/null +++ b/e2e/node_modules @@ -0,0 +1 @@ +/a0/usr/workdir/.bdd-deps/node_modules \ No newline at end of file From c02a51367d986d5e35c42f717d32d893296d36ba Mon Sep 17 00:00:00 2001 From: Agent Zero Date: Sat, 1 Aug 2026 23:22:05 +0100 Subject: [PATCH 2/3] ci: publish the devkit harness image to GHCR (public) DEVKIT_IMAGE defaulted to ghcr.io/agent-zero-plugins/plugin-devkit:latest but nothing ever published it, so the package did not exist: every consumer make e2e died with manifest unknown, and CI only worked because each workflow rebuilt the image inline (~85s per run). Build devcontainer/Containerfile and push :latest + :sha-<12>, plus the version tag on v* tags. Least privilege: contents:read + packages:write, GITHUB_TOKEN only, no PAT. The final step logs OUT and pulls anonymously, failing with actionable instructions if the package is not public -- the no-auth consumer path is the reason to publish at all, so it is proven rather than assumed. --- .github/workflows/publish-devkit-image.yml | 89 ++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/publish-devkit-image.yml diff --git a/.github/workflows/publish-devkit-image.yml b/.github/workflows/publish-devkit-image.yml new file mode 100644 index 0000000..74e7249 --- /dev/null +++ b/.github/workflows/publish-devkit-image.yml @@ -0,0 +1,89 @@ +# publish-devkit-image — build & publish the devkit harness image to GHCR (PUBLIC). +# +# WHY THIS EXISTS (DEC-071): +# `e2e/Makefile.devkit` defaults DEVKIT_IMAGE to +# ghcr.io/agent-zero-plugins/plugin-devkit:latest, but nothing ever published +# it — the package did not exist, so every consumer's `make e2e` died with +# "manifest unknown" and CI only worked because each workflow rebuilt the +# image inline (~85s per run, every run). +# +# PUBLIC ON PURPOSE: +# The image is quay.io/podman/stable + Node + Playwright + Fedora browser +# libs. It contains NO secrets and NO first-party source. Publishing it +# publicly means an outside contributor can run the documented local loop +# with no GHCR auth at all — the single most important consumability +# property for this repo. +name: publish-devkit-image + +on: + push: + branches: [main] + paths: + - "devcontainer/Containerfile" + - ".github/workflows/publish-devkit-image.yml" + tags: ["v*"] + workflow_dispatch: + +# Least privilege: this job only needs to read the repo and write the package. +permissions: + contents: read + packages: write + +concurrency: + group: publish-devkit-image + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + # GITHUB_TOKEN suffices: the package lives in THIS repo's org and the + # job declares packages:write. No PAT, no long-lived secret. + run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Build + run: | + set -euo pipefail + podman build -t devkit:build -f devcontainer/Containerfile devcontainer + + - name: Tag and push + run: | + set -euo pipefail + IMAGE=ghcr.io/${{ github.repository_owner }}/plugin-devkit + IMAGE=$(echo "$IMAGE" | tr '[:upper:]' '[:lower:]') # GHCR requires lowercase + SHA_TAG="sha-${GITHUB_SHA::12}" + + podman tag devkit:build "$IMAGE:$SHA_TAG" + podman push "$IMAGE:$SHA_TAG" + echo "pushed $IMAGE:$SHA_TAG" + + # :latest tracks main. Tag builds additionally publish the version. + if [ "${GITHUB_REF_TYPE:-}" = "tag" ]; then + VER="${GITHUB_REF_NAME#v}" + podman tag devkit:build "$IMAGE:$VER" + podman push "$IMAGE:$VER" + echo "pushed $IMAGE:$VER" + fi + + podman tag devkit:build "$IMAGE:latest" + podman push "$IMAGE:latest" + echo "pushed $IMAGE:latest" + + - name: Verify the image is pullable ANONYMOUSLY + # The whole point of publishing is the no-auth consumer path, so prove + # it rather than assume it. A fresh logged-out store is the only honest + # check: a cached layer or a live login would mask a private package. + run: | + set -euo pipefail + IMAGE=ghcr.io/${{ github.repository_owner }}/plugin-devkit + IMAGE=$(echo "$IMAGE" | tr '[:upper:]' '[:lower:]') + podman logout ghcr.io || true + if podman pull --authfile /dev/null "$IMAGE:latest"; then + echo "anonymous pull OK — consumers need no GHCR auth" + else + echo "::error::$IMAGE:latest is NOT anonymously pullable. Set the package visibility to Public (Org -> Packages -> plugin-devkit -> Package settings -> Change visibility). Until then every consumer's 'make e2e' fails with 'manifest unknown'." + exit 1 + fi From 4d015ca05ad93fede2869ada69f6413a528a9f05 Mon Sep 17 00:00:00 2001 From: omar-nahhas-agent Date: Mon, 3 Aug 2026 01:19:09 +0100 Subject: [PATCH 3/3] fix(e2e): bound post-login landmark wait explicitly (DEC-072) LoginPage.login() asserted the top-nav "Plugins" button with Playwright's bare 5000ms expect default -- the only post-login landmark in the suite left on the default (PluginsPage uses explicit 8s/20s/10s). On a cold nested A0 the WebUI hydrates ~5s AFTER the post-login redirect, so that deadline straddles the render and flakes ~50%. Proven from run 30721063715's trace: page reached / at t=2.4s, top-nav rendered at t=7.755s, default deadline ~7.4s -- missed by ~350ms. The passing run on the very next commit touched no test/timeout/image config, confirming a coin-flip race rather than an image behavioural difference. The wait exists to surface a silent auth failure, not to measure hydration speed, so bound it generously at 30s. --- e2e/pages/LoginPage.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/e2e/pages/LoginPage.ts b/e2e/pages/LoginPage.ts index 4c88900..3cb38b8 100644 --- a/e2e/pages/LoginPage.ts +++ b/e2e/pages/LoginPage.ts @@ -34,8 +34,16 @@ export class LoginPage { // A0 redirects to `/` on success. We assert on a post-login landmark // (the top-nav "Plugins" button) rather than the URL alone, so a // silent auth failure shows up as a timeout on this line. + // + // Explicit 30s (DEC-072): on a cold nested A0 the WebUI hydrates ~5s AFTER + // the post-login redirect, straddling Playwright's 5s default and flaking + // ~50%. Measured from run 30721063715's trace: page reached `/` at t=2.4s, + // top-nav rendered at t=7.755s — the default deadline (~7.4s) missed it by + // ~350ms. This wait exists to surface a SILENT AUTH FAILURE, not to measure + // hydration speed, so bound it generously — consistent with PluginsPage's + // 8s/20s/10s landmark waits. await expect( this.page.getByRole("button", { name: "Plugins", exact: true }), - ).toBeVisible(); + ).toBeVisible({ timeout: 30_000 }); } }