From 0d6dd7412b8410c28f312fb4864bc91ac4f85232 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:23:47 +0100 Subject: [PATCH 1/5] fix(testing): Tear down relay validators started from PATH Zombienet starts the relay validators as `polkadot` from PATH, so the teardown pattern `target/testnet/polkadot` matched only their PVF workers and left both validators running after a run. Match the validators' command line instead; the workers exit with them. Checked on a local network: the old patterns killed 7 processes and left both validators running, the new ones leave nothing behind. --- testing/src/zombienet.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/src/zombienet.mjs b/testing/src/zombienet.mjs index 7838e48fa..b1cbe9ddd 100644 --- a/testing/src/zombienet.mjs +++ b/testing/src/zombienet.mjs @@ -15,7 +15,10 @@ import { ROOT, TESTING } from "./rehearsal-env.mjs"; export const CHAIN_BINARIES = [ "zombienet-macos", - "target/testnet/polkadot", + // Zombienet starts the relay validators as `polkadot` from PATH (see + // zombienet.toml), so match their command line wherever the binary lives. + // Their PVF workers exit with them. + "polkadot --chain", "target/release/pendulum-node", ]; From 9a3695e4b59660ab7c8d2d14815d791a74328a02 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:24:14 +0100 Subject: [PATCH 2/5] fix(testing): Start the monitor from the finalized block in rehearsals The rehearsal and the drills started the monitor at the best block plus one, with the nonce read there. The monitor ingests finalized blocks only, so until finality caught up it had no verified hash for its start and paged "no verified hash for Pendulum block N". Start it after the finalized head instead, with the nonce read at that head. Checked on a local network with the monitor build: best #3, finalized #1 gave one "no verified hash" page within 20 seconds from the old start and none from the new one. --- testing/src/drills.mjs | 9 +++++---- testing/src/rehearsal.mjs | 9 +++++---- testing/src/zombienet.mjs | 11 +++++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/testing/src/drills.mjs b/testing/src/drills.mjs index 6559f1578..43539b3e9 100644 --- a/testing/src/drills.mjs +++ b/testing/src/drills.mjs @@ -35,7 +35,9 @@ import { assertTestnet, buildContext, CONVERSION_FACTOR, loadEnv, PEN_12, ROOT, send, TESTING, } from "./rehearsal-env.mjs"; import { deployToSepolia } from "./rehearsal-deploy.mjs"; -import { discoverCollator, generateSpec, killMatching, sleep, spawnNetwork, teardown, waitForFinality } from "./zombienet.mjs"; +import { + discoverCollator, generateSpec, killMatching, monitorStart, sleep, spawnNetwork, teardown, waitForFinality, +} from "./zombienet.mjs"; const flags = new Set(process.argv.slice(2)); const DAEMONS = ["attestor1", "attestor2", "attestor3", "attestor4", ...(MONITOR_DIR ? ["monitor"] : []), "releaser"]; @@ -140,7 +142,7 @@ async function main() { }; const pendulumHead = (await api.query.system.number()).toString(); const baseHead = String(await ctx.pub.getBlockNumber()); - const monitorStartNonce = (await api.query.tokenMigration.nextNonce()).toString(); + const monitorFrom = await monitorStart(api); const attestorEnv = (i) => ({ ...baseEnv, PENDULUM_WS: pendulumWs, // The Zombienet chain has its own genesis, not Pendulum's. @@ -157,8 +159,7 @@ async function main() { PENDULUM_WS: pendulumWs, GRACE_SECONDS: "300", GUARDIAN_PRIVATE_KEY: env.GUARDIAN_PRIVATE_KEY, - PENDULUM_START_BLOCK: (BigInt(pendulumHead) + 1n).toString(), - PENDULUM_START_NONCE: monitorStartNonce, + ...monitorFrom, BASE_START_BLOCK: baseHead, }); start("releaser", "releaser", { ...baseEnv, RELEASER_PRIVATE_KEY: env.RELEASER_PRIVATE_KEY, START_BLOCK: baseHead }); diff --git a/testing/src/rehearsal.mjs b/testing/src/rehearsal.mjs index 7acac0287..7dfced496 100644 --- a/testing/src/rehearsal.mjs +++ b/testing/src/rehearsal.mjs @@ -35,7 +35,9 @@ import { assertTestnet, buildContext, CONVERSION_FACTOR, loadEnv, PEN_12, PEN_18, ROOT, send, TESTING, } from "./rehearsal-env.mjs"; import { deployToSepolia, rehearsalParams } from "./rehearsal-deploy.mjs"; -import { discoverCollator, killMatching, generateSpec, sleep, spawnNetwork, teardown, waitForFinality } from "./zombienet.mjs"; +import { + discoverCollator, killMatching, generateSpec, monitorStart, sleep, spawnNetwork, teardown, waitForFinality, +} from "./zombienet.mjs"; const flags = new Set(process.argv.slice(2)); const FUND = flags.has("--fund"); @@ -311,7 +313,7 @@ async function main() { }; const pendulumHead = (await api.query.system.number()).toString(); const baseHead = String(await ctx.pub.getBlockNumber()); - const monitorStartNonce = (await api.query.tokenMigration.nextNonce()).toString(); + const monitorFrom = await monitorStart(api); // The Zombienet chain has its own genesis, not Pendulum's. const pendulumGenesisHash = api.genesisHash.toHex(); for (let i = 0; i < 4; i++) { @@ -327,8 +329,7 @@ async function main() { PENDULUM_WS: pendulumWs, GRACE_SECONDS: "300", GUARDIAN_PRIVATE_KEY: env.GUARDIAN_PRIVATE_KEY, - PENDULUM_START_BLOCK: (BigInt(pendulumHead) + 1n).toString(), - PENDULUM_START_NONCE: monitorStartNonce, + ...monitorFrom, BASE_START_BLOCK: baseHead, }); start("releaser", "releaser", { ...baseEnv, RELEASER_PRIVATE_KEY: env.RELEASER_PRIVATE_KEY, START_BLOCK: baseHead }); diff --git a/testing/src/zombienet.mjs b/testing/src/zombienet.mjs index b1cbe9ddd..51d8df0a2 100644 --- a/testing/src/zombienet.mjs +++ b/testing/src/zombienet.mjs @@ -148,3 +148,14 @@ export async function waitForFinality(api, { timeoutMs = 300_000, log } = {}) { } throw new Error("parachain never finalised a block — the relay is not including candidates"); } + +/** Where the monitor starts on the local chain: the block after the finalized + * head, with the migration nonce at that head. The monitor reads finalized + * blocks only, so a start past them (the best block runs a few blocks ahead + * here) has no verified hash until finality catches up, and it pages. */ +export async function monitorStart(api) { + const hash = await api.rpc.chain.getFinalizedHead(); + const header = await api.rpc.chain.getHeader(hash); + const nonce = await (await api.at(hash)).query.tokenMigration.nextNonce(); + return { PENDULUM_START_BLOCK: String(header.number.toNumber() + 1), PENDULUM_START_NONCE: nonce.toString() }; +} From 70b2f980f992d11313ff53a78d0d57f20b42da84 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:24:28 +0100 Subject: [PATCH 3/5] fix(testing): Scan Base Sepolia logs in 200-block ranges in rehearsals Base Sepolia's public RPC refuses eth_getLogs over more than 200 blocks ("eth_getLogs is limited to a 200 range"), and the releaser and the monitor default to 10,000-block ranges. The rehearsal and the drills now pass MAX_BLOCK_RANGE and BASE_MAX_BLOCK_RANGE of 200 unless the shell sets them. --- testing/src/drills.mjs | 3 +++ testing/src/rehearsal.mjs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/testing/src/drills.mjs b/testing/src/drills.mjs index 43539b3e9..486448268 100644 --- a/testing/src/drills.mjs +++ b/testing/src/drills.mjs @@ -139,6 +139,9 @@ async function main() { const baseEnv = { BASE_RPC_URL: env.BASE_SEPOLIA_RPC_URL, VAULT_ADDRESS: vault, BASE_CHAIN_ID: "84532", POLL_INTERVAL_MS: "12000", + // Base Sepolia's public RPC serves eth_getLogs for at most 200 blocks. + MAX_BLOCK_RANGE: process.env.MAX_BLOCK_RANGE ?? "200", + BASE_MAX_BLOCK_RANGE: process.env.BASE_MAX_BLOCK_RANGE ?? "200", }; const pendulumHead = (await api.query.system.number()).toString(); const baseHead = String(await ctx.pub.getBlockNumber()); diff --git a/testing/src/rehearsal.mjs b/testing/src/rehearsal.mjs index 7dfced496..5cf2f8855 100644 --- a/testing/src/rehearsal.mjs +++ b/testing/src/rehearsal.mjs @@ -310,6 +310,9 @@ async function main() { // the starts so they do not align into bursts. In production each // attestor has its own node and this pressure does not arise. POLL_INTERVAL_MS: "12000", + // Base Sepolia's public RPC serves eth_getLogs for at most 200 blocks. + MAX_BLOCK_RANGE: process.env.MAX_BLOCK_RANGE ?? "200", + BASE_MAX_BLOCK_RANGE: process.env.BASE_MAX_BLOCK_RANGE ?? "200", }; const pendulumHead = (await api.query.system.number()).toString(); const baseHead = String(await ctx.pub.getBlockNumber()); From a6b8aa5656c163a37036bb1711978e9fc6344cd0 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 12:24:42 +0100 Subject: [PATCH 4/5] fix(testing): Refuse a rehearsal per-release cap above the daily cap The vault rejects perReleaseCap > dailyCap (CapsInverted), but the rehearsal found out only when the deploy reverted, after the network was up. The per-release cap now defaults to the daily cap (28,800 PEN each by default) instead of a fixed 28,800 that a smaller daily cap would fall below, and loading the rehearsal environment refuses an inverted pair with a message naming both values, before anything is spawned or deployed. --- testing/.env.rehearsal.example | 3 ++- testing/src/rehearsal-deploy.mjs | 7 ++----- testing/src/rehearsal-env.mjs | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/testing/.env.rehearsal.example b/testing/.env.rehearsal.example index 582678760..8ad889679 100644 --- a/testing/.env.rehearsal.example +++ b/testing/.env.rehearsal.example @@ -35,7 +35,8 @@ RELEASER_PRIVATE_KEY= # separately on Anvil in phase 1, where time can be warped. REHEARSAL_DAILY_CAP_PEN=28800 # Must not exceed the daily cap: the vault enforces perReleaseCap <= dailyCap, -# since anything in between could never fit the daily allowance. +# since anything in between could never fit the daily allowance, and the +# harness refuses such a pair at startup. Unset, it equals the daily cap. REHEARSAL_PER_RELEASE_CAP_PEN=28800 # How far ahead the immutable sweep floor sits. Short so the sweep path is diff --git a/testing/src/rehearsal-deploy.mjs b/testing/src/rehearsal-deploy.mjs index 5e9d6a227..8e23d930f 100644 --- a/testing/src/rehearsal-deploy.mjs +++ b/testing/src/rehearsal-deploy.mjs @@ -13,15 +13,12 @@ import { execFileSync } from "node:child_process"; import { readFileSync } from "node:fs"; import path from "node:path"; -import { BASE_SEPOLIA_CHAIN_ID, PEN_18, ROOT } from "./rehearsal-env.mjs"; +import { BASE_SEPOLIA_CHAIN_ID, PEN_18, rehearsalCapsPen, ROOT } from "./rehearsal-env.mjs"; const CONTRACTS = path.join(ROOT, "contracts"); export function rehearsalParams(env) { - const dailyCapPen = BigInt(env.REHEARSAL_DAILY_CAP_PEN ?? "28800"); - // Equal to the daily cap: the vault rejects perReleaseCap > dailyCap (that - // band could never release). - const perReleaseCapPen = BigInt(env.REHEARSAL_PER_RELEASE_CAP_PEN ?? "28800"); + const { daily: dailyCapPen, perRelease: perReleaseCapPen } = rehearsalCapsPen(env); return { maxIssuance: 150_000_000n * PEN_18, dailyCap: dailyCapPen * PEN_18, diff --git a/testing/src/rehearsal-env.mjs b/testing/src/rehearsal-env.mjs index 73fab4869..1fb0a40d0 100644 --- a/testing/src/rehearsal-env.mjs +++ b/testing/src/rehearsal-env.mjs @@ -73,9 +73,27 @@ export function loadEnv() { throw new Error(`${key} is not a 32-byte hex private key`); } } + rehearsalCapsPen(env); // refuses inverted caps before anything is spawned or deployed return env; } +/** The rehearsal caps in whole PEN, sized for wall clock (see + * .env.rehearsal.example). The per-release cap defaults to the daily cap. */ +export function rehearsalCapsPen(env) { + const daily = BigInt(env.REHEARSAL_DAILY_CAP_PEN ?? "28800"); + const perRelease = BigInt(env.REHEARSAL_PER_RELEASE_CAP_PEN ?? daily); + // The vault rejects perReleaseCap > dailyCap (CapsInverted): a release in + // between could never fit the daily allowance. + if (perRelease > daily) { + throw new Error( + `REHEARSAL_PER_RELEASE_CAP_PEN (${perRelease}) is above REHEARSAL_DAILY_CAP_PEN (${daily}), ` + + "which the vault rejects (CapsInverted). Set it to at most the daily cap, or leave it unset " + + "to use the daily cap (28800 PEN each by default).", + ); + } + return { daily, perRelease }; +} + export function buildContext(env) { const chain = defineChain({ id: BASE_SEPOLIA_CHAIN_ID, From 9e2f21a3da3da2675c8bc503affdfdf04fbbc0d4 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Tue, 27 Oct 2026 15:17:32 +0100 Subject: [PATCH 5/5] fix(testing): Accept the monitor's unverified-approval pages in phase 3 The monitor now reports an approval it cannot check against a finalized Pendulum burn as unverified and pauses as a precaution, rather than as a proven mismatch. Chopsticks' clock trails Anvil's, so phase 3 sees that verdict. Accept either one; the automatic pause stays the assertion. --- testing/src/phase3-e2e.mjs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/testing/src/phase3-e2e.mjs b/testing/src/phase3-e2e.mjs index 56238e07d..dc16fc799 100644 --- a/testing/src/phase3-e2e.mjs +++ b/testing/src/phase3-e2e.mjs @@ -248,17 +248,20 @@ await monitorCheck("a real vault deficit alerts and reaches a confirmed automati await monitorCheck("fabricated approvals are reported and pause the vault before they can reach quorum", async () => { // One attestor's approval cannot release on its own, so the monitor reports // it; a second attestor approving the same tuple pauses the vault before a - // third approval could release it. + // third approval could release it. Chopsticks' clock trails Anvil's, so the + // monitor may never see a Pendulum view late enough to prove the tuple + // fabricated; it then reports the approval as unverified once its grace runs + // out and pauses as a precaution. Either verdict passes. const fabricated = { ...V, functionName: "approve", args: [9_999_999n, admin.address, MIN] }; const before = logs("monitor").length; await send(attestors[0], fabricated); - await waitFor(() => logs("monitor").slice(before).includes("MISMATCHED APPROVAL"), { + await waitFor(() => /MISMATCHED APPROVAL|BASE EVENT NOT VERIFIED IN TIME/.test(logs("monitor").slice(before)), { timeoutMs: 30_000, label: "the monitor to report a fabricated approval", }); assertEq(await read(V, "paused"), false, "one fabricated approval leaves the vault running"); await send(attestors[1], fabricated); - await waitFor(() => logs("monitor").slice(before).includes("MIGRATION TUPLE VIOLATION"), { + await waitFor(() => /MIGRATION TUPLE VIOLATION|PRECAUTIONARY PAUSE/.test(logs("monitor").slice(before)), { timeoutMs: 30_000, label: "the monitor to reject the second fabricated approval", });