diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d22c2f96..15fb4c386 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ on: env: CARGO_TERM_COLOR: always + ZKM_TOOLCHAIN_VERSION: '20260917' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -61,6 +62,7 @@ jobs: java -jar "$JAR" -config GraphLifecycleFixed.cfg GraphLifecycle.tla java -jar "$JAR" -config GraphLifecycleFineGrainedFixed.cfg GraphLifecycleFineGrainedFixed.tla java -jar "$JAR" -config InstancePresignedFixed.cfg InstancePresigned.tla + java -jar "$JAR" -config InstanceLifecycle.cfg InstanceLifecycle.tla java -jar "$JAR" -config Take2DisproveRace.cfg Take2DisproveRace.tla java -jar "$JAR" -config MultiActorRace.cfg MultiActorRace.tla java -jar "$JAR" -config InstanceBridgeOutRaceFixed.cfg InstanceBridgeOutRace.tla @@ -141,10 +143,21 @@ jobs: timeout-minutes: 30 with: profile: minimal - toolchain: nightly-2025-12-11 + toolchain: nightly-2026-09-17 override: true components: clippy - - run: curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/ProjectZKM/toolchain/refs/heads/main/setup.sh | sh + - name: Install Ziren toolchain + run: | + set -euo pipefail + installer="$RUNNER_TEMP/zkmup" + curl --proto '=https' --tlsv1.2 -fsSL --retry 3 \ + https://raw.githubusercontent.com/ProjectZKM/toolchain/582f7927cac0668883a86cbd02fbc18b500b1b6f/zkmup \ + -o "$installer" + sh "$installer" install -v "$ZKM_TOOLCHAIN_VERSION" + SHELL=/bin/true sh "$installer" setup -v "$ZKM_TOOLCHAIN_VERSION" + source "$HOME/.zkm-toolchain/env" + rustc --version + cargo --version - name: Install Dependencies run: sudo apt update && sudo apt install protobuf-compiler - run: | @@ -154,14 +167,20 @@ jobs: name: Cargo Test needs: tla-plus runs-on: ubuntu-latest - strategy: - matrix: - toolchain: - - nightly-2025-12-11 steps: - uses: actions/checkout@v5 - name: Install Ziren toolchain - run: curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/ProjectZKM/toolchain/refs/heads/main/setup.sh | sh + run: | + set -euo pipefail + installer="$RUNNER_TEMP/zkmup" + curl --proto '=https' --tlsv1.2 -fsSL --retry 3 \ + https://raw.githubusercontent.com/ProjectZKM/toolchain/582f7927cac0668883a86cbd02fbc18b500b1b6f/zkmup \ + -o "$installer" + sh "$installer" install -v "$ZKM_TOOLCHAIN_VERSION" + SHELL=/bin/true sh "$installer" setup -v "$ZKM_TOOLCHAIN_VERSION" + source "$HOME/.zkm-toolchain/env" + rustc --version + cargo --version - name: Install Dependencies run: sudo apt update && sudo apt install protobuf-compiler - name: Launch the Regtest diff --git a/.github/workflows/e2e_localnet.yml b/.github/workflows/e2e_localnet.yml new file mode 100644 index 000000000..3cd4d967b --- /dev/null +++ b/.github/workflows/e2e_localnet.yml @@ -0,0 +1,70 @@ +name: Localnet e2e + +# The whole localnet e2e suite (localnet/README.md, localnet/e2e/README.md) on the self-hosted +# bitvm2-node runner: build the mock-gc binaries, rebuild a private localnet, run all 13 cases. +# ~2 h per run, so it is not a PR gate: same triggers as ci_time_consuming.yml. + +on: + push: + branches: ["main", "dev"] + workflow_dispatch: + +# One CI localnet per host: queue runs, never cancel one that is half way through. +concurrency: + group: localnet-e2e + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + # A localnet of its own next to any other on the host (ports PORT_BASE+0..999). + LOCALNET_DATA: /data/actions-runner-bitvm2-node/cache/localnet-e2e/data + PORT_BASE: "25000" + COMPOSE_PROJECT: bvci + # Dedicated target dir, kept between runs; only this workflow builds into it. + CARGO_TARGET_DIR: /data/actions-runner-bitvm2-node/cache/localnet-e2e/target + +jobs: + e2e: + name: Localnet e2e + runs-on: [self-hosted, bitvm2-node] + timeout-minutes: 210 + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + + - name: Build the mock-gc binaries + run: | + # results of earlier runs stay in the data dir: keep the last 10, never report a stale one + mkdir -p "$LOCALNET_DATA/results" + rm -f "$LOCALNET_DATA/results/latest" + find "$LOCALNET_DATA/results" -mindepth 1 -maxdepth 1 -type d | sort | head -n -10 | xargs -r rm -rf + localnet/build.sh + + - name: Reset the localnet + run: make localnet-reset + + - name: Run the e2e suite + run: make e2e + + - name: Report + if: always() + run: | + run=$(readlink -f "$LOCALNET_DATA/results/latest" 2>/dev/null || true) + [ -d "$run" ] || run="" + echo "E2E_RESULTS=$run" >> "$GITHUB_ENV" + if [ -f "$run/report.md" ]; then cat "$run/report.md" >> "$GITHUB_STEP_SUMMARY"; else echo "no e2e report" >> "$GITHUB_STEP_SUMMARY"; fi + + - name: Stop the localnet + if: always() + run: | + localnet/scripts/chaos.sh reset || true + localnet/scripts/down.sh infra + + - name: Upload the results + if: always() && env.E2E_RESULTS != '' + uses: actions/upload-artifact@v4 + with: + name: localnet-e2e-${{ github.run_id }} + path: ${{ env.E2E_RESULTS }} + retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eefb99acf..4ce5cdf53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,7 +80,6 @@ jobs: "target/release/send-rbf" "target/release/pegin-request" "target/release/pegout" - "target/release/update-db" ) if [ "${{ runner.os }}" = "Linux" ]; then diff --git a/.gitignore b/.gitignore index 3968c89af..f1e50a0d8 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,13 @@ local_docs/ local_scripts/ scripts/testnet/ scripts/devnet/ -*.DS_Store \ No newline at end of file +*.DS_Store +# localnet state (default LOCALNET_DATA) and build artifacts of its tooling +localnet/.local/ +localnet/subgraph/gateway/node_modules/ +localnet/subgraph/gateway/build/ +localnet/subgraph/gateway/generated/ +localnet/contracts-local/out/ +localnet/contracts-local/cache/ +localnet/contracts-local/broadcast/ +localnet/tools/__pycache__/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..b67bf64cc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "localnet/contracts"] + path = localnet/contracts + url = https://github.com/KSlashh/bitvm-L2-contracts.git diff --git a/Cargo.lock b/Cargo.lock index e96e998b7..2e2a74662 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3012,6 +3012,7 @@ dependencies = [ "tracing", "uuid", "verifiable-circuit-babe", + "zkm-sdk", ] [[package]] @@ -3041,6 +3042,7 @@ dependencies = [ "client", "commit-chain", "dotenv", + "engine", "esplora-client", "futures", "goat", @@ -4584,6 +4586,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "engine" +version = "0.4.0" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "hex", + "prometheus-client", + "serde", + "serde_json", + "sha2 0.10.9", + "store", + "tempfile", + "tokio", + "tokio-util", + "tracing", + "uuid", +] + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -6486,6 +6508,7 @@ dependencies = [ "libp2p-ping", "libp2p-quic", "libp2p-relay", + "libp2p-request-response", "libp2p-swarm", "libp2p-tcp", "libp2p-upnp", diff --git a/Cargo.toml b/Cargo.toml index 3737e052c..b704ae7ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/cbft-rpc", "crates/bitvm-gc", "crates/store", + "crates/engine", "crates/node-macros", "crates/client", "crates/util", @@ -118,6 +119,7 @@ soldering-host = { git = "https://github.com/GOATNetwork/bitvm-gc", features = [ #garbled-snark-verifier = { path = "../bitvm-gc/garbled-snark-verifier"} #soldering-host = { path = "../bitvm-gc/babe-programs/soldering/host" } store = { path = "crates/store" } +engine = { path = "crates/engine" } util = { path = "crates/util" } client = { path = "crates/client" } bitcoin-light-client-circuit = { path = "circuits/crates/bitcoin-light-client-circuit" } diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..5508d225e --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +# Repo-level shortcuts for the local mock testnet under localnet/ (see localnet/README.md). +# Every target honours LOCALNET_DATA, PORT_BASE and COMPOSE_PROJECT from the environment, e.g. +# LOCALNET_DATA=/tmp/bv2 PORT_BASE=23000 COMPOSE_PROJECT=bvlocal2 make localnet-reset +LOCALNET := localnet + +.PHONY: localnet-build localnet-up localnet-down localnet-reset localnet-status e2e-smoke e2e + +localnet-build: ## build the mock-gc node + helper binaries into the repo target + $(LOCALNET)/build.sh + +localnet-up: ## adopt / start an existing localnet (infra, miner, proof builder, 5 nodes) + $(LOCALNET)/scripts/up.sh + +localnet-down: ## stop nodes + support processes (infra keeps running; `scripts/down.sh infra` stops it too) + $(LOCALNET)/scripts/down.sh + +localnet-reset: ## DESTRUCTIVE: wipe $(LOCALNET_DATA) and rebuild chains, contracts, subgraph, nodes + $(LOCALNET)/scripts/reset.sh --yes + +localnet-status: + $(LOCALNET)/scripts/status.sh + +e2e-smoke: ## peg-in, peg-out and challenge/take-2 happy paths (~20 min) + $(LOCALNET)/e2e/run.sh pegin_happy pegout_happy challenge_take2 + +e2e: ## the whole suite (~110 min) + $(LOCALNET)/e2e/run.sh diff --git a/README.md b/README.md index 6ddf509ff..b55d53767 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ GOAT Network's BitVM bridge implementation. See [GOAT BitVM Whitepaper](https:// - `proof-builder-rpc/`: RPC server for offloading proof generation to a separate process - `crates/`: Shared Rust crates for common types and utilities - `deployment`: Deployment scripts and documentation +- `scripts/ops/`: Read-only operator helpers for a running node (health triage, log queries, instance / graph traces) +- `localnet/`: clone-and-run mock testnet (five `mock-gc` nodes on regtest + anvil + graph-node) and the on-demand e2e suite, see [localnet/README.md](localnet/README.md); `make localnet-reset`, `make e2e-smoke` ## Formal verification (TLA+) diff --git a/crates/bitvm-gc/Cargo.toml b/crates/bitvm-gc/Cargo.toml index 50ae4e378..0be4db450 100644 --- a/crates/bitvm-gc/Cargo.toml +++ b/crates/bitvm-gc/Cargo.toml @@ -22,11 +22,17 @@ ark-bn254 = { workspace = true } ark-groth16 = { workspace = true } ark-serialize = { workspace = true } ark-ff = { workspace = true } +ark-ec = { workspace = true } verifiable-circuit-babe = { workspace = true } garbled-snark-verifier = { workspace = true } soldering-host = { workspace = true } rayon = { workspace = true } +# Only needed by the compile-time `mock-gc` feature, to wrap the mocked soldered +# output in a structurally-real `ZKMProofWithPublicValues` so the node's payload +# plumbing (compact/expand, SolderingProof::output) keeps working unchanged. +zkm-sdk = { workspace = true, optional = true } + serde-big-array = { workspace = true } serde_json = { workspace = true } bincode.workspace = true @@ -46,3 +52,6 @@ tokio = { workspace = true, features = ["macros", "time"] } [features] default = [] ci-tests = [] +# Replace the real BABE/GC/soldering machinery with deterministic, on-chain-valid +# mocks. Localnet only. See node feature `mock-gc` which forwards to this one. +mock-gc = ["dep:zkm-sdk"] diff --git a/crates/bitvm-gc/src/babe_adapter.rs b/crates/bitvm-gc/src/babe_adapter.rs index ca9637e50..6bc67219d 100644 --- a/crates/bitvm-gc/src/babe_adapter.rs +++ b/crates/bitvm-gc/src/babe_adapter.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; +#[cfg(not(feature = "mock-gc"))] use std::panic::{AssertUnwindSafe, catch_unwind}; +#[cfg(not(feature = "mock-gc"))] use std::path::PathBuf; use crate::types::BitvmGcCircuitData; @@ -14,22 +16,36 @@ use goat::assert_scripts::{INPUT_WIRE_NUM, OperatorAssertSecretKey, WireHash, la use goat::wots::{Wots, Wots96}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +#[cfg(not(feature = "mock-gc"))] use soldering_host::BabeBundle; +#[cfg(not(feature = "mock-gc"))] pub use soldering_host::BabeBundleBuilder; +// Under `mock-gc`, the BABE builder, the four heavy setup/dispute entry points, and the operator +// assert-proof verdict are replaced by the deterministic mocks in `crate::mock_gc`. Every other +// helper here (label derivation, WOTS assert witness, compact/expand payload plumbing, the real +// `build_real_challenge_assert_witness`) stays real in both builds. +#[cfg(feature = "mock-gc")] +pub use crate::mock_gc::{ + BabeBundleBuilder, build_real_setup_package, open_real_setup_and_solder, + recover_real_wrongly_challenged_witness, verify_operator_assert_proof, verify_real_setup, +}; use verifiable_circuit_babe::babe::{ GC_INPUT_WIRES, WitnessEncSetupCt, build_challenge_assert_witness as babe_build_challenge_assert_witness, }; use verifiable_circuit_babe::cac::cac_finalize_indices; pub use verifiable_circuit_babe::cac::{CACSetupPackage, FinalizedInstanceData}; +#[cfg(not(feature = "mock-gc"))] use verifiable_circuit_babe::dre::N_PADDED; use verifiable_circuit_babe::gc::{SGC_PART1_CONSTANT_SIZE, SparseAdaptorTable}; pub use verifiable_circuit_babe::instance::commit::CACInstanceCommit; pub use verifiable_circuit_babe::instance::secret::Seed; +#[cfg(not(feature = "mock-gc"))] use verifiable_circuit_babe::prover::BABEProver; -use verifiable_circuit_babe::soldering::{ - SolderedLabelsData, SolderingData as RealSolderingData, SolderingProof as RealSolderingProof, -}; +use verifiable_circuit_babe::soldering::SolderedLabelsData; +#[cfg(not(feature = "mock-gc"))] +use verifiable_circuit_babe::soldering::SolderingData as RealSolderingData; +use verifiable_circuit_babe::soldering::SolderingProof as RealSolderingProof; pub use verifiable_circuit_babe::transactions::ChallengeAssertWitnessRaw; pub use verifiable_circuit_babe::transactions::TxAssertWitness; use verifiable_circuit_babe::utils::pi1_xd_to_wots96_msg; @@ -164,6 +180,7 @@ pub fn build_setup_package(n_cc: usize) -> Result { } /// Builds a real random BABE/CAC setup package bound to the supplied Groth16 statement. +#[cfg(not(feature = "mock-gc"))] pub fn build_real_setup_package( n_cc: usize, vk: &Groth16VerifyingKey, @@ -186,6 +203,7 @@ pub fn build_real_setup_package( } /// Reconstructs the real Verifier instances and creates CAC opening/soldering output data. +#[cfg(not(feature = "mock-gc"))] pub fn open_real_setup_and_solder( soldering_builder: &BabeBundleBuilder, private_state: &BabeVerifierPrivateState, @@ -210,6 +228,7 @@ pub fn open_real_setup_and_solder( } /// Verifies real CAC openings, commitments, and the native Ziren soldering proof. +#[cfg(not(feature = "mock-gc"))] #[allow(clippy::too_many_arguments)] pub fn verify_real_setup( soldering_builder: &BabeBundleBuilder, @@ -389,6 +408,17 @@ pub fn recover_operator_proof_from_assert_witness( Ok(Groth16Proof { a: pi1, b: pi2, c: pi3 }) } +/// Verifies the operator's asserted Groth16 proof (the real BN254 pairing check). Under `mock-gc` +/// this is replaced by a policy verdict (`crate::mock_gc::verify_operator_assert_proof`). +#[cfg(not(feature = "mock-gc"))] +pub fn verify_operator_assert_proof( + assert_witness: &TxAssertWitness, + vk: &Groth16VerifyingKey, + static_input: StaticInputs, +) -> bool { + assert_witness.verify_groth16_proof(vk, &static_input) +} + /// Builds a placeholder verifier challenge witness from an assert witness. pub fn build_challenge_assert_witness( verifier_state: &BabeVerifierState, @@ -454,6 +484,7 @@ pub fn build_wrongly_challenged_witness( } /// Evaluates the native BABE garbled circuit and returns a finalized hashlock preimages. +#[cfg(not(feature = "mock-gc"))] pub fn recover_real_wrongly_challenged_witness( prover_state: &BabeProverState, challenge_witness: &BabeChallengeAssertWitness, @@ -482,6 +513,7 @@ pub fn build_wrongly_challenged_witness_from_preimages( Ok(BabeWronglyChallengedWitness { verifier_index: challenge_witness.verifier_index, final_msg }) } +#[cfg(not(feature = "mock-gc"))] fn ensure_real_gc_assets_configured() -> Result<()> { for name in [ "FGC_GATES_PATH", @@ -504,7 +536,7 @@ fn ensure_real_gc_assets_configured() -> Result<()> { Ok(()) } -fn statement_digest( +pub(crate) fn statement_digest( vk: &Groth16VerifyingKey, static_inputs: StaticInputs, ) -> Result<[u8; 32]> { @@ -526,7 +558,7 @@ fn restore_real_verifier( .ok_or_else(|| anyhow::anyhow!("Cannot restore real verifier")) } -fn validate_finalized_indices( +pub(crate) fn validate_finalized_indices( package: &CACSetupPackage, finalized_indices: &[usize], ) -> Result<()> { @@ -543,6 +575,7 @@ fn validate_finalized_indices( Ok(()) } +#[cfg(not(feature = "mock-gc"))] fn from_real_soldering(soldering: &RealSolderingData) -> Result { let output = soldering.soldering_proof.output().map_err(anyhow::Error::msg)?; Ok(SolderingData { @@ -573,6 +606,7 @@ fn expand_compact_soldering_data(soldering: CompactSolderingData) -> Result Result { if soldering.proof.is_empty() { bail!("soldering proof is empty"); @@ -583,6 +617,7 @@ fn to_real_soldering(soldering: &SolderingData) -> Result { }) } +#[cfg(not(feature = "mock-gc"))] fn recover_a_valid_finalized_messages( prover_state: &BabeProverState, challenge_witness: &BabeChallengeAssertWitness, diff --git a/crates/bitvm-gc/src/lib.rs b/crates/bitvm-gc/src/lib.rs index f0383511a..3dee083e2 100644 --- a/crates/bitvm-gc/src/lib.rs +++ b/crates/bitvm-gc/src/lib.rs @@ -6,6 +6,8 @@ pub mod watchtower; pub mod actors; pub mod babe_adapter; pub mod keys; +#[cfg(feature = "mock-gc")] +pub mod mock_gc; pub mod pegin; pub mod timelocks; pub mod types; diff --git a/crates/bitvm-gc/src/mock_gc.rs b/crates/bitvm-gc/src/mock_gc.rs new file mode 100644 index 000000000..640a772ae --- /dev/null +++ b/crates/bitvm-gc/src/mock_gc.rs @@ -0,0 +1,637 @@ +//! Deterministic, on-chain-valid mocks that stand in for the real BABE / garbled-circuit +//! machinery when the `mock-gc` cargo feature is enabled. +//! +//! Design goal: keep **all on-chain-relevant data real** (the `epk` label commitments, the +//! finalized-message hashlocks, the soldered wire deltas) so that the Operator's Assert, the +//! Verifier's ChallengeAssert and the Operator's WronglyChallenged transactions all satisfy the +//! real Bitcoin scripts, while faking only the parts that would otherwise require the 3.3 GB gate +//! files and a Ziren STARK/Groth16 prover (the garbling, the soldering STARK proof, and the +//! Groth16 pairing verdict). +//! +//! The mock label commitments are derived with the **real** light-secret derivation +//! (`InstanceLightSecrets::from_seed`, ChaCha20 only), so the untouched, still-real +//! `build_real_challenge_assert_witness` keeps producing 768 labels whose HASH160 match the +//! committed `epk` bit-by-bit. The soldered output is produced by the **real** +//! `soldering_guest_compute`, so `verify_soldering_output_match_commitment` still passes; it is +//! then wrapped in a structurally-real (but empty) `ZKMProofWithPublicValues` so the node's +//! payload plumbing survives with zero call-site changes. +//! +//! What is intentionally *not* sound (localnet only): the finalized message preimage is placed in +//! plaintext in `ct_setup.ct3_masked_msg`, and the Groth16 pairing check is replaced by a policy +//! verdict shared by convention with the mock proof builder. This is fine for a functional +//! testnet and useless as a security test. + +use crate::babe_adapter::StaticInputs; + +use std::collections::HashSet; + +use anyhow::{Result, anyhow, bail}; +use ark_bn254::{Bn254, Fr, G1Affine}; +use ark_ec::AffineRepr; +use ark_groth16::VerifyingKey as Groth16VerifyingKey; +use ark_serialize::CanonicalSerialize; +use sha2::{Digest, Sha256}; + +use verifiable_circuit_babe::babe::{GC_INPUT_WIRES, WitnessEncSetupCt}; +use verifiable_circuit_babe::cac::verify_finalized_indices_in_bundle; +use verifiable_circuit_babe::dre::N_PADDED; +use verifiable_circuit_babe::prover::BABEProver; +use verifiable_circuit_babe::soldering::{build_soldered_wires_input, soldering_guest_compute}; +use verifiable_circuit_babe::utils::derive_hashlock; +use verifiable_circuit_babe::verifier::{BABEVerifier, InstanceLightSecrets}; +use zkm_sdk::{ZKMProof, ZKMProofWithPublicValues, ZKMPublicValues}; + +use crate::babe_adapter::{ + BABE_M_CC, BabeChallengeAssertWitness, BabeProverState, BabeVerifierPrivateState, + BabeWronglyChallengedWitness, CACInstanceCommit, CACSetupPackage, FinalizedInstanceData, + OpenedInstanceSeeds, Seed, SetupAndSolderingData, SolderingData, TxAssertWitness, + assert_wots_message, sample_finalized_instance_data, statement_digest, + validate_finalized_indices, +}; + +/// Version string stamped into the mocked soldering `ZKMProofWithPublicValues`. Not verified by +/// the node (only `SolderingProof::output()` / the public values are consumed for soldering). +const MOCK_ZKM_VERSION: &str = "mock-gc"; + +/// Domain tag for the deterministic per-instance finalized message preimage. +const MOCK_MSG_TAG: &[u8] = b"bitvm2-mock-gc/final-msg"; + +/// Drop-in replacement for `soldering_host::BabeBundleBuilder`. Under the real build that type runs +/// a Ziren `client.setup()` on the soldering guest ELF in `new()` (tens of seconds, the heaviest +/// startup item). The mock is a zero-cost unit struct. +#[derive(Debug, Default, Clone, Copy)] +pub struct BabeBundleBuilder; + +impl BabeBundleBuilder { + pub fn new() -> Self { + tracing::info!( + event = "mock_gc", + stage = "builder_init", + "mock-gc BabeBundleBuilder (no Ziren setup)" + ); + Self + } +} + +/// The proof verdict policy, read from `MOCK_GC_PROOF_VERDICT` at call time. It stands in for the +/// real Groth16 pairing check (which no mocked proof can satisfy for the embedded IMM VK). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Verdict { + /// Treat every structurally-valid operator proof as valid (happy path, no dispute). Default. + AlwaysValid, + /// Treat every operator proof as invalid (Verifier challenges; Operator cannot answer). + AlwaysInvalid, + /// Run the real pairing check (a genuinely valid Groth16 proof still passes). + Real, +} + +fn verdict() -> Verdict { + match std::env::var("MOCK_GC_PROOF_VERDICT").ok().as_deref() { + Some("always-invalid") => Verdict::AlwaysInvalid, + Some("real") => Verdict::Real, + _ => Verdict::AlwaysValid, + } +} + +fn sha256(data: &[u8]) -> [u8; 32] { + Sha256::digest(data).into() +} + +/// Deterministic per-instance filler commitment for fields the mock does not need to be real. +fn tag_commit(seed: &Seed, n: u8) -> [u8; 32] { + let mut bytes = Vec::with_capacity(seed.len() + 2); + bytes.extend_from_slice(seed); + bytes.extend_from_slice(&[0xf0, n]); + sha256(&bytes) +} + +/// Deterministic finalized-message preimage for an instance (kept in plaintext in the finalized +/// data so the Operator can recover it; its HASH160 is the on-chain finalized hashlock). +fn mock_msg(seed: &Seed) -> [u8; 32] { + let mut bytes = Vec::with_capacity(MOCK_MSG_TAG.len() + seed.len()); + bytes.extend_from_slice(MOCK_MSG_TAG); + bytes.extend_from_slice(seed); + sha256(&bytes) +} + +/// Builds a CAC instance commitment whose `epk` (label hashlocks) and `h_msg` are derived exactly +/// as the real `CACInstanceCommit::from_instance`, but from the cheap light secrets and a mock +/// message. Only `epk` and `h_msg` are consumed on-chain / by the still-real challenge path; the +/// remaining commitment fields are deterministic filler. +fn mock_commit_from_seed(seed: Seed) -> CACInstanceCommit { + let ls = InstanceLightSecrets::from_seed(seed); + let mut epk = Vec::with_capacity(GC_INPUT_WIRES); + for key in &ls.input_0labels[0] { + epk.push([derive_hashlock(&key.0), derive_hashlock(&(*key ^ ls.delta[0]).0)]); + } + for key in &ls.input_0labels[1] { + epk.push([derive_hashlock(&key.0), derive_hashlock(&(*key ^ ls.delta[1]).0)]); + } + debug_assert_eq!(epk.len(), GC_INPUT_WIRES); + + CACInstanceCommit { + epk, + constant_commits_fgc: [ + [tag_commit(&seed, 1), tag_commit(&seed, 2)], + [tag_commit(&seed, 3), tag_commit(&seed, 4)], + ], + // The real path stores 510 SGC constant commits here; nothing in the node checks the + // length, so keep the gossiped GenCircuits package small. + constant_commits_sgc: vec![], + b_blind_commit: tag_commit(&seed, 5), + h_msg: derive_hashlock(&mock_msg(&seed)), + h_ct_setup: tag_commit(&seed, 6), + com_adaptor: [tag_commit(&seed, 7), tag_commit(&seed, 8)], + com_gc: [tag_commit(&seed, 9), tag_commit(&seed, 10), tag_commit(&seed, 11)], + } +} + +fn serialize_static_input(static_inputs: StaticInputs) -> Result> { + let mut bytes = Vec::new(); + static_inputs.serialize_compressed(&mut bytes)?; + Ok(bytes) +} + +/// Bit `wire` of the 96-byte Wots assert message, using the same layout as +/// `InstanceLightSecrets::compute_labels_from_bytes` (π₁ wires `0..512` in bytes `0..64`, x_d wires +/// `512..768` in bytes `64..96`). +fn message_bit(msg: &[u8; 96], wire: usize) -> usize { + let (byte, bit) = if wire < 2 * N_PADDED { + (wire / 8, wire % 8) + } else { + let local = wire - 2 * N_PADDED; + (64 + local / 8, local % 8) + }; + ((msg[byte] >> bit) & 1) as usize +} + +/// Mock of `build_real_setup_package`: sample fresh seeds (exactly like the real +/// `BABEVerifier::new`), derive the real label commitments cheaply, and bind the statement digest. +/// Skips the ~181-instance garbling entirely. +pub fn build_real_setup_package( + n_cc: usize, + vk: &Groth16VerifyingKey, + static_inputs: StaticInputs, +) -> Result<(CACSetupPackage, BabeVerifierPrivateState)> { + if n_cc == 0 { + bail!("n_cc must be greater than zero"); + } + let seeds: Vec = (0..n_cc).map(|_| rand::random()).collect(); + let commits = seeds.iter().map(|seed| mock_commit_from_seed(*seed)).collect(); + let package = CACSetupPackage { commits }; + let private_state = BabeVerifierPrivateState { + instance_seeds: seeds, + statement_digest: statement_digest(vk, static_inputs)?, + }; + Ok((package, private_state)) +} + +/// Mock of `open_real_setup_and_solder`: reveal the real seeds for non-finalized instances, build +/// finalized data carrying the statement and the plaintext message preimage, and produce the real +/// soldered output wrapped in a structurally-real (empty) STARK proof. +pub fn open_real_setup_and_solder( + _soldering_builder: &BabeBundleBuilder, + private_state: &BabeVerifierPrivateState, + package: &CACSetupPackage, + finalized_indices: &[usize], + vk: &Groth16VerifyingKey, + static_inputs: StaticInputs, +) -> Result { + if private_state.statement_digest != statement_digest(vk, static_inputs)? { + bail!("BABE setup statement does not match persisted verifier state"); + } + validate_finalized_indices(package, finalized_indices)?; + if private_state.instance_seeds.len() != package.commits.len() { + bail!("persisted BABE verifier seeds do not match setup package size"); + } + let verifier = + BABEVerifier::from_state(&private_state.instance_seeds, package, vk, static_inputs) + .ok_or_else(|| anyhow!("cannot restore mock verifier from persisted state"))?; + + let finalized_set: HashSet = finalized_indices.iter().copied().collect(); + let opened: OpenedInstanceSeeds = (0..package.commits.len()) + .filter(|index| !finalized_set.contains(index)) + .map(|index| (index, private_state.instance_seeds[index])) + .collect(); + + let static_bytes = serialize_static_input(static_inputs)?; + let finalized: Vec = finalized_indices + .iter() + .map(|&index| { + let mut data = sample_finalized_instance_data(index); + data.ct_setup = WitnessEncSetupCt { + ct2_r_delta_g2: static_bytes.clone(), + ct3_masked_msg: mock_msg(&private_state.instance_seeds[index]).to_vec(), + }; + data.b = G1Affine::generator(); + data + }) + .collect(); + + let soldered_input = build_soldered_wires_input(&verifier, finalized_indices); + let soldered_output = soldering_guest_compute(&soldered_input); + let public_values = bincode::serialize(&soldered_output)?; + let proof = bincode::serialize(&ZKMProofWithPublicValues { + proof: ZKMProof::Core(vec![]), + public_values: ZKMPublicValues::from(&public_values), + zkm_version: MOCK_ZKM_VERSION.to_string(), + })?; + + Ok(( + opened, + finalized, + SolderingData { finalized_indices: finalized_indices.to_vec(), soldered_output, proof }, + )) +} + +/// Mock of `verify_real_setup`: run the real shape/partition and soldering-commitment checks, plus +/// the mock-specific statement binding and message-preimage checks. Skips only the Ziren +/// `client.verify` and the 174-instance re-garbling. +#[allow(clippy::too_many_arguments)] +pub fn verify_real_setup( + _soldering_builder: &BabeBundleBuilder, + package: &CACSetupPackage, + opened: &[(usize, Seed)], + finalized: &[FinalizedInstanceData], + soldering: &SolderingData, + _vk: &Groth16VerifyingKey, + claimed_finalized_indices: &[usize], + static_public_inputs: StaticInputs, +) -> Result<()> { + verify_finalized_indices_in_bundle( + package, + opened, + finalized, + claimed_finalized_indices, + &soldering.finalized_indices, + ) + .map_err(anyhow::Error::msg)?; + + let static_bytes = serialize_static_input(static_public_inputs)?; + for data in finalized { + let commit = package + .commits + .get(data.index) + .ok_or_else(|| anyhow!("finalized instance index {} out of range", data.index))?; + if data.ct_setup.ct2_r_delta_g2 != static_bytes { + bail!("finalized instance {} is not bound to the setup statement", data.index); + } + if derive_hashlock(&data.ct_setup.ct3_masked_msg) != commit.h_msg { + bail!("finalized instance {} message preimage does not match its hashlock", data.index); + } + } + + if verify_openings_enabled() { + for (index, seed) in opened { + let commit = package + .commits + .get(*index) + .ok_or_else(|| anyhow!("opened instance index {index} out of range"))?; + if &mock_commit_from_seed(*seed) != commit { + bail!("opened seed for instance {index} does not reproduce its commitment"); + } + } + } + + BABEProver::verify_soldering_output_match_commitment( + &soldering.soldered_output, + package, + &soldering.finalized_indices, + ) + .map_err(anyhow::Error::msg)?; + + Ok(()) +} + +/// Mock of `recover_real_wrongly_challenged_witness`: instead of evaluating the garbled circuit, it +/// (1) checks the on-chain-revealed ChallengeAssert labels really are the committed base-instance +/// labels for the asserted message (exactly what the ChallengeAssert Bitcoin script proves), (2) +/// applies the proof verdict, and (3) returns the plaintext finalized message preimage — whose +/// HASH160 is the on-chain WronglyChallenged hashlock. +pub fn recover_real_wrongly_challenged_witness( + prover_state: &BabeProverState, + challenge_witness: &BabeChallengeAssertWitness, + _proof: &ark_groth16::Proof, + _vk: Groth16VerifyingKey, + _dyn_pubin: Fr, +) -> Result { + if prover_state.finalized.is_empty() { + bail!("BABE prover state has no finalized instances"); + } + if prover_state.finalized.len() != prover_state.h_msgs.len() { + bail!("BABE prover state finalized data and hash count differ"); + } + if prover_state.h_msgs.len() != BABE_M_CC { + bail!("wrongly challenged setup must contain exactly {BABE_M_CC} finalized hashlocks"); + } + + let base = &prover_state.finalized[0]; + let base_commit = prover_state + .package + .commits + .get(base.index) + .ok_or_else(|| anyhow!("base finalized instance index {} out of range", base.index))?; + let input_labels = &challenge_witness.witness.input_labels; + if input_labels.len() != GC_INPUT_WIRES { + bail!("challenge witness exposes {} labels; expected {GC_INPUT_WIRES}", input_labels.len()); + } + + // The published ChallengeAssert labels must be the committed base-instance labels selected by + // the bits of the asserted 96-byte message. This is precisely the relation the on-chain + // ChallengeAssert script enforces, so it cannot pass with junk labels. + let msg = assert_wots_message(&TxAssertWitness { + wots_sig: challenge_witness.witness.wots_sig.clone(), + pi2: vec![], + pi3: vec![], + })?; + for (wire, label) in input_labels.iter().enumerate() { + let bit = message_bit(&msg, wire); + if derive_hashlock(label) != base_commit.epk[wire][bit] { + bail!("Cannot find any valid msg"); + } + } + + match verdict() { + Verdict::AlwaysInvalid => bail!("Cannot find any valid msg"), + // No mocked proof can satisfy the real pairing equation, so `real` cannot let the Operator + // legitimately answer; treat it as "cannot answer" to force the Disprove branch. + Verdict::Real => bail!("Cannot find any valid msg"), + Verdict::AlwaysValid => {} + } + + let final_msg = base.ct_setup.ct3_masked_msg.clone(); + if derive_hashlock(&final_msg) != prover_state.h_msgs[0] { + bail!("Cannot find any valid msg"); + } + + Ok(BabeWronglyChallengedWitness { verifier_index: challenge_witness.verifier_index, final_msg }) +} + +/// Replaces the Verifier's real `TxAssertWitness::verify_groth16_proof` pairing check. The witness +/// must be structurally recoverable (as the real assert path always produces), then the verdict +/// decides whether the Verifier accepts the proof (happy path) or challenges. +pub fn verify_operator_assert_proof( + assert_witness: &TxAssertWitness, + vk: &Groth16VerifyingKey, + static_input: StaticInputs, +) -> bool { + if assert_witness.try_recover_pi1_xd().is_none() || assert_witness.recover_pi2_pi3().is_none() { + return false; + } + match verdict() { + Verdict::AlwaysInvalid => false, + Verdict::Real => assert_witness.verify_groth16_proof(vk, &static_input), + Verdict::AlwaysValid => true, + } +} + +/// Whether `verify_real_setup` re-derives every opened commitment from its revealed seed. +/// Default on; set `MOCK_GC_VERIFY_OPENINGS=false`/`0` to skip (~174 instances) for speed. +fn verify_openings_enabled() -> bool { + !matches!( + std::env::var("MOCK_GC_VERIFY_OPENINGS").ok().as_deref(), + Some("false") | Some("0") | Some("no") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::babe_adapter::{ + BABE_M_CC, BabeProverState, build_assert_witness, build_real_challenge_assert_witness, + compact_soldering_proof_payload, derive_finalized_indices, + expand_compact_soldering_proof_payload, extract_gc_circuit_data, + }; + use ark_bn254::G2Affine; + use ark_crypto_primitives::snark::CircuitSpecificSetupSNARK; + use goat::assert_scripts::label_hash; + use goat::wots::{Wots, Wots96}; + use rand::SeedableRng; + use verifiable_circuit_babe::babe::DummyMulCircuit; + + // Small n_cc keeps the test quick while still exercising opened (n_cc - M_CC) instances. + const TEST_N_CC: usize = 12; + + fn test_vk() -> Groth16VerifyingKey { + let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(42); + let (_, vk) = ark_groth16::Groth16::::setup( + DummyMulCircuit:: { a: Some(Fr::from(3u64)), b: Some(Fr::from(7u64)) }, + &mut rng, + ) + .expect("groth16 setup"); + vk + } + + fn test_verifier_pubkey() -> bitcoin::PublicKey { + use bitcoin::secp256k1::{Secp256k1, SecretKey}; + let secp = Secp256k1::new(); + let sk = SecretKey::from_slice(&[7u8; 32]).unwrap(); + bitcoin::PublicKey::new(sk.public_key(&secp)) + } + + fn mock_operator_proof() -> ark_groth16::Proof { + // Structurally valid, non-infinity subgroup points; the mock never checks the pairing. + ark_groth16::Proof { + a: G1Affine::generator(), + b: G2Affine::generator(), + c: G1Affine::generator(), + } + } + + struct Fixture { + vk: Groth16VerifyingKey, + static_input: StaticInputs, + package: CACSetupPackage, + private_state: BabeVerifierPrivateState, + finalized_indices: Vec, + opened: Vec<(usize, Seed)>, + finalized: Vec, + soldering: SolderingData, + } + + fn build_fixture() -> Fixture { + let vk = test_vk(); + let static_input = [Fr::from(2024u64); crate::babe_adapter::STATIC_SLOTS.len()]; + let (package, private_state) = + build_real_setup_package(TEST_N_CC, &vk, static_input).expect("setup package"); + let finalized_indices = + derive_finalized_indices(&package, BABE_M_CC).expect("finalized indices"); + let (opened, finalized, soldering) = open_real_setup_and_solder( + &BabeBundleBuilder::new(), + &private_state, + &package, + &finalized_indices, + &vk, + static_input, + ) + .expect("open and solder"); + Fixture { + vk, + static_input, + package, + private_state, + finalized_indices, + opened, + finalized, + soldering, + } + } + + #[test] + fn setup_verifies_and_payload_round_trips() { + let f = build_fixture(); + // The verifier's setup verifies on the operator side. + verify_real_setup( + &BabeBundleBuilder::new(), + &f.package, + &f.opened, + &f.finalized, + &f.soldering, + &f.vk, + &f.finalized_indices, + f.static_input, + ) + .expect("verify_real_setup should accept an honest bundle"); + + // The compact payload survives a bincode round-trip (the ZKMProofWithPublicValues wrapper + // and finalized data are preserved), and still verifies. + let compact = compact_soldering_proof_payload(&f.opened, &f.finalized, &f.soldering) + .expect("compact payload"); + let bytes = bincode::serialize(&compact).expect("serialize compact payload"); + let restored: crate::babe_adapter::CompactSolderingProofPayload = + bincode::deserialize(&bytes).expect("deserialize compact payload"); + let (opened2, finalized2, soldering2) = + expand_compact_soldering_proof_payload(restored).expect("expand payload"); + verify_real_setup( + &BabeBundleBuilder::new(), + &f.package, + &opened2, + &finalized2, + &soldering2, + &f.vk, + &f.finalized_indices, + f.static_input, + ) + .expect("verify_real_setup should accept the round-tripped bundle"); + } + + #[test] + fn verify_rejects_tampered_statement_and_message() { + let f = build_fixture(); + + // Wrong statement: the finalized instances are no longer bound to it. + let wrong = verify_real_setup( + &BabeBundleBuilder::new(), + &f.package, + &f.opened, + &f.finalized, + &f.soldering, + &f.vk, + &f.finalized_indices, + f.static_input.map(|input| input + Fr::from(1u64)), + ); + assert!(wrong.is_err(), "a mismatched statement must be rejected"); + + // Tampered message preimage: HASH160 no longer matches the committed hashlock. + let mut finalized = f.finalized.clone(); + finalized[0].ct_setup.ct3_masked_msg = vec![0u8; 32]; + let tampered = verify_real_setup( + &BabeBundleBuilder::new(), + &f.package, + &f.opened, + &finalized, + &f.soldering, + &f.vk, + &f.finalized_indices, + f.static_input, + ); + assert!(tampered.is_err(), "a tampered finalized message must be rejected"); + } + + #[test] + fn challenge_labels_match_gc_data_and_wrongly_challenged_is_consistent() { + // This test owns MOCK_GC_PROOF_VERDICT; no other test reads the verdict, so mutating the + // process env here is race-free. + unsafe { std::env::remove_var("MOCK_GC_PROOF_VERDICT") }; + + let f = build_fixture(); + let base_idx = f.finalized_indices[0]; + let h_msgs: Vec<[u8; 20]> = + f.finalized.iter().map(|d| f.package.commits[d.index].h_msg).collect(); + let gc_data = extract_gc_circuit_data( + test_verifier_pubkey(), + &f.package.commits[base_idx].epk, + &h_msgs, + ) + .expect("extract gc data"); + assert_eq!(gc_data.final_msg_hashlocks.len(), BABE_M_CC); + + // Operator asserts, Verifier reveals the base-instance labels. + let proof = mock_operator_proof(); + let dynamic_input = Fr::from(9u64); + let secret = Wots96::generate_secret_key(); + let assert_witness = + build_assert_witness(&proof, &secret, dynamic_input).expect("assert witness"); + let msg = assert_wots_message(&assert_witness).expect("assert message"); + let challenge = build_real_challenge_assert_witness( + &f.private_state, + &f.package, + &f.finalized_indices, + &f.vk, + f.static_input, + &assert_witness, + 0, + ) + .expect("challenge assert witness"); + + // Every revealed label hashes to the gc_data wire hash selected by the asserted message bit + // — exactly the relation the on-chain ChallengeAssert script enforces. + assert_eq!(challenge.witness.input_labels.len(), GC_INPUT_WIRES); + for (wire, label) in challenge.witness.input_labels.iter().enumerate() { + let bit = message_bit(&msg, wire); + let expected = if bit == 1 { + gc_data.wire_hashes[wire].true_label_hash + } else { + gc_data.wire_hashes[wire].false_label_hash + }; + assert_eq!(label_hash(&label.to_vec()), expected, "label mismatch at wire {wire}"); + } + + // Operator recovers a valid WronglyChallenged message whose HASH160 is an on-chain hashlock. + let prover_state = BabeProverState { + package: f.package.clone(), + finalized: f.finalized.clone(), + soldering: f.soldering.clone(), + h_msgs: h_msgs.clone(), + }; + assert!(verify_operator_assert_proof(&assert_witness, &f.vk, f.static_input)); + let recovered = recover_real_wrongly_challenged_witness( + &prover_state, + &challenge, + &proof, + f.vk.clone(), + dynamic_input, + ) + .expect("wrongly challenged witness"); + assert!( + gc_data.final_msg_hashlocks.contains(&label_hash(&recovered.final_msg)), + "recovered message must satisfy an on-chain finalized hashlock" + ); + + // Forcing an invalid verdict makes the Verifier challenge and blocks the Operator's answer. + unsafe { std::env::set_var("MOCK_GC_PROOF_VERDICT", "always-invalid") }; + assert!(!verify_operator_assert_proof(&assert_witness, &f.vk, f.static_input)); + assert!( + recover_real_wrongly_challenged_witness( + &prover_state, + &challenge, + &proof, + f.vk.clone(), + dynamic_input, + ) + .is_err(), + "with an invalid verdict the operator cannot answer the challenge" + ); + unsafe { std::env::remove_var("MOCK_GC_PROOF_VERDICT") }; + } +} diff --git a/crates/bitvm-gc/src/timelocks.rs b/crates/bitvm-gc/src/timelocks.rs index dd38dae65..6c333c304 100644 --- a/crates/bitvm-gc/src/timelocks.rs +++ b/crates/bitvm-gc/src/timelocks.rs @@ -37,8 +37,11 @@ pub const NODE_SIGNET_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { operator_commit: 18, connector_f: 24, }; +/// Regtest keeps `connector_z` (T0) wide on purpose: the peg-in pre-signing +/// ceremony and the abandon buffer are measured in blocks, and a one-block T0 +/// would expire every localnet peg-in before the committee finishes signing. pub const NODE_REGTEST_TIMELOCK_CONFIG: TimelockConfig = TimelockConfig { - connector_z: 1, + connector_z: 60, connector_a: 2, prover_connector: 1, connector_d: 3, @@ -95,7 +98,9 @@ impl ProtocolTimingBudget { self.action_window().saturating_add(self.watchtower_proof_blocks) } - const fn confirmed_action_window(self) -> u32 { + /// Blocks needed for already-broadcast evidence to confirm and settle: + /// evidence depth plus inclusion plus reorg safety. + pub const fn confirmed_action_window(self) -> u32 { self.evidence_confirmations .saturating_add(self.inclusion_blocks) .saturating_add(self.safety_blocks) @@ -143,6 +148,28 @@ pub fn protocol_timing_budget(network: Network) -> ProtocolTimingBudget { } } +/// Regtest's timing budget has no confirmation, inclusion, or safety blocks, so +/// its `confirmed_action_window` is zero; the peg-in abandon buffer needs a +/// non-zero floor there so a Peg-In-Confirm broadcast right before T0 still +/// gets a few blocks to land before the instance is abandoned. +pub const NODE_REGTEST_PEGIN_ABANDON_BUFFER_BLOCKS: u32 = 6; + +/// Default peg-in abandon buffer, in blocks after `Prepare height + connector_z` (T0). +/// +/// Between T0 and the end of this buffer no node signs or broadcasts for the +/// instance; it only follows the chain. Once the buffer has elapsed and the +/// Peg-In-Confirm is neither confirmed nor visible in the mempool, the peg-in +/// is abandoned (`Timeout`). Defaults to the network's `confirmed_action_window` +/// (evidence + inclusion + safety), with a fixed floor on regtest. +pub fn default_pegin_abandon_buffer_blocks(network: Network) -> u32 { + match network { + Network::Regtest => NODE_REGTEST_PEGIN_ABANDON_BUFFER_BLOCKS, + Network::Bitcoin | Network::Testnet | Network::Testnet4 | Network::Signet => { + protocol_timing_budget(network).confirmed_action_window() + } + } +} + pub fn validate_timelock_config(network: Network, config: &TimelockConfig) -> Result<()> { let budget = protocol_timing_budget(network); for (name, value) in [ @@ -304,6 +331,31 @@ mod tests { } } + #[test] + fn pegin_abandon_buffer_defaults_are_non_zero_confirmed_action_windows() { + for (network, expected) in [ + (Network::Bitcoin, 18), + (Network::Testnet, 14), + (Network::Testnet4, 14), + (Network::Signet, 3), + (Network::Regtest, NODE_REGTEST_PEGIN_ABANDON_BUFFER_BLOCKS), + ] { + let buffer = default_pegin_abandon_buffer_blocks(network); + assert_eq!(buffer, expected, "{network:?}"); + assert!(buffer > 0, "{network:?} abandon buffer must not be zero"); + if network != Network::Regtest { + assert_eq!(buffer, protocol_timing_budget(network).confirmed_action_window()); + } + } + } + + #[test] + fn regtest_connector_z_leaves_room_for_presigning_and_abandon_buffer() { + let connector_z = default_connector_z_timelock_blocks(Network::Regtest); + assert_eq!(connector_z, 60); + assert!(connector_z > default_pegin_abandon_buffer_blocks(Network::Regtest)); + } + #[test] fn rejects_non_height_based_bip68_sequences() { assert!(ensure_bip68_height_based_sequence("test", BIP68_BLOCKS_MASK).is_ok()); diff --git a/crates/client/src/http_client/async_client.rs b/crates/client/src/http_client/async_client.rs index 0d2e9b452..e59cd6a11 100644 --- a/crates/client/src/http_client/async_client.rs +++ b/crates/client/src/http_client/async_client.rs @@ -1,3 +1,4 @@ +use crate::timeout_config::{DEFAULT_HTTP_CONNECT_TIMEOUT_SECS, get_http_request_timeout_secs}; use anyhow::anyhow; use esplora_client::Error; use reqwest::{Client, Response}; @@ -5,10 +6,12 @@ use std::time::Duration; const DEFAULT_MAX_RETRIES: usize = 6; const BASE_BACKOFF_MILLIS: Duration = Duration::from_millis(256); -const RETRYABLE_ERROR_CODES: [u16; 3] = [ +const RETRYABLE_ERROR_CODES: [u16; 5] = [ 429, // TOO_MANY_REQUESTS 500, // INTERNAL_SERVER_ERROR + 502, // BAD_GATEWAY (a proxy in front of the upstream) 503, // SERVICE_UNAVAILABLE + 504, // GATEWAY_TIMEOUT ]; pub struct HttpAsyncClient { @@ -17,8 +20,21 @@ pub struct HttpAsyncClient { } impl HttpAsyncClient { + /// A client with a connect timeout and a whole-request timeout + /// (`HTTP_REQUEST_TIMEOUT_SECS`), so a hung upstream fails the call instead + /// of parking the caller forever. pub fn new(max_retries: Option) -> Self { - Self { client: Client::new(), max_retries: max_retries.unwrap_or(DEFAULT_MAX_RETRIES) } + let client = Client::builder() + .connect_timeout(Duration::from_secs(DEFAULT_HTTP_CONNECT_TIMEOUT_SECS)) + .timeout(Duration::from_secs(get_http_request_timeout_secs())) + .build() + .unwrap_or_else(|err| { + tracing::warn!( + "failed to build the HTTP client with timeouts: {err}; using the default client" + ); + Client::new() + }); + Self { client, max_retries: max_retries.unwrap_or(DEFAULT_MAX_RETRIES) } } pub async fn get_response_json( diff --git a/crates/client/src/timeout_config.rs b/crates/client/src/timeout_config.rs index 6190daf23..ff057a228 100644 --- a/crates/client/src/timeout_config.rs +++ b/crates/client/src/timeout_config.rs @@ -2,6 +2,10 @@ pub const ENV_BTC_REQUEST_TIMEOUT_SECS: &str = "BTC_REQUEST_TIMEOUT_SECS"; pub const DEFAULT_BTC_REQUEST_TIMEOUT_SECS: u64 = 15; pub const ENV_GOAT_RPC_TIMEOUT_SECS: &str = "GOAT_RPC_TIMEOUT_SECS"; pub const DEFAULT_GOAT_RPC_TIMEOUT_SECS: u64 = 20; +/// Whole-request timeout of the generic HTTP client (proof builder and proof forwarding). +pub const ENV_HTTP_REQUEST_TIMEOUT_SECS: &str = "HTTP_REQUEST_TIMEOUT_SECS"; +pub const DEFAULT_HTTP_REQUEST_TIMEOUT_SECS: u64 = 60; +pub const DEFAULT_HTTP_CONNECT_TIMEOUT_SECS: u64 = 10; pub fn get_btc_request_timeout_secs() -> u64 { std::env::var(ENV_BTC_REQUEST_TIMEOUT_SECS) @@ -18,3 +22,11 @@ pub fn get_goat_rpc_timeout_secs() -> u64 { .map(|timeout| timeout.clamp(1, 120)) .unwrap_or(DEFAULT_GOAT_RPC_TIMEOUT_SECS) } + +pub fn get_http_request_timeout_secs() -> u64 { + std::env::var(ENV_HTTP_REQUEST_TIMEOUT_SECS) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|timeout| timeout.clamp(1, 600)) + .unwrap_or(DEFAULT_HTTP_REQUEST_TIMEOUT_SECS) +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml new file mode 100644 index 000000000..aea348050 --- /dev/null +++ b/crates/engine/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "engine" +version.workspace = true +edition.workspace = true + +[dependencies] +store = { workspace = true } +anyhow = { workspace = true } +async-trait = { workspace = true } +futures = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tokio-util = { workspace = true } +tracing = { workspace = true } +prometheus-client = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/engine/src/ack.rs b/crates/engine/src/ack.rs new file mode 100644 index 000000000..5f89762f0 --- /dev/null +++ b/crates/engine/src/ack.rs @@ -0,0 +1,28 @@ +//! ACK return channel: the aggregate worker hands ACKs to the network task +//! after its transaction committed. The channel is bounded; when it is full the +//! ACK is dropped and the sender's resend produces it again. + +use tokio::sync::mpsc; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AckRequest { + pub peer: String, + pub delivery_id: String, +} + +pub type AckReceiver = mpsc::Receiver; + +#[derive(Clone, Debug)] +pub struct AckSender(mpsc::Sender); + +impl AckSender { + pub fn channel(capacity: usize) -> (Self, AckReceiver) { + let (tx, rx) = mpsc::channel(capacity.max(1)); + (Self(tx), rx) + } + + /// Returns false when the ACK was dropped because the channel is full or closed. + pub fn try_send(&self, request: AckRequest) -> bool { + self.0.try_send(request).is_ok() + } +} diff --git a/crates/engine/src/aggregate.rs b/crates/engine/src/aggregate.rs new file mode 100644 index 000000000..c3bc2dc86 --- /dev/null +++ b/crates/engine/src/aggregate.rs @@ -0,0 +1,97 @@ +//! The aggregate contract: `apply(state, event) -> Effects` inside one transaction. + +use crate::effects::Effects; +use crate::event::AggregateRef; +use async_trait::async_trait; +use store::localdb::StorageProcessor; +use store::{EngineAggregate, EngineInboxEvent}; + +/// A dependency the apply could not satisfy. The event stays pending with its +/// payload and becomes runnable again when `wake_ref` changes or on the periodic +/// re-check. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WaitingFor { + pub wake_ref: String, + pub detail: String, +} + +impl WaitingFor { + pub fn on(aggregate: &AggregateRef, detail: impl Into) -> Self { + Self { wake_ref: aggregate.wake_ref(), detail: detail.into() } + } + + pub fn on_ref(wake_ref: impl Into, detail: impl Into) -> Self { + Self { wake_ref: wake_ref.into(), detail: detail.into() } + } +} + +#[derive(Debug)] +pub enum ApplyOutcome { + /// Material persisted; the event is done and its payload can go. + Applied, + /// A dependency is missing; effects are still committed (for example a + /// fetch job) and the event is re-applied later. + Waiting(WaitingFor), + /// Permanent business rejection. All writes of this apply are rolled back + /// and the event is recorded as `Failed` with the reason. + Rejected(String), +} + +#[derive(Debug)] +pub enum ApplyError { + /// Retry later with backoff; nothing of this apply is kept. + Transient(anyhow::Error), + /// An invariant is broken: the event is poisoned and the aggregate flagged. + Invariant(anyhow::Error), +} + +impl ApplyError { + pub fn transient(error: impl Into) -> Self { + ApplyError::Transient(error.into()) + } + + pub fn invariant(error: impl Into) -> Self { + ApplyError::Invariant(error.into()) + } +} + +/// Unclassified errors (typically storage) are retried rather than poisoned: +/// poison is reserved for panics and explicit invariant violations. +impl From for ApplyError { + fn from(error: anyhow::Error) -> Self { + ApplyError::Transient(error) + } +} + +pub struct ApplyContext<'a> { + pub now: i64, + pub aggregate: AggregateRef, + /// Bookkeeping row as it was when the apply started (`None` for a new aggregate). + pub current: Option, + /// The apply transaction; business tables are read and written through it. + pub storage: &'a mut StorageProcessor<'static>, + pub effects: Effects, +} + +impl ApplyContext<'_> { + pub fn status(&self) -> &str { + self.current.as_ref().map(|c| c.status.as_str()).unwrap_or("") + } + + pub fn phase(&self) -> &str { + self.current.as_ref().map(|c| c.phase.as_str()).unwrap_or("") + } + + pub fn state(&self) -> Option<&str> { + self.current.as_ref().and_then(|c| c.state.as_deref()) + } +} + +#[async_trait] +pub trait AggregateHandler: Send + Sync { + async fn apply( + &self, + ctx: &mut ApplyContext<'_>, + event: &EngineInboxEvent, + ) -> Result; +} diff --git a/crates/engine/src/clock.rs b/crates/engine/src/clock.rs new file mode 100644 index 000000000..8c4b44efc --- /dev/null +++ b/crates/engine/src/clock.rs @@ -0,0 +1,43 @@ +//! Wall clock abstraction so tests can drive deadlines and backoff deterministically. + +use std::sync::Arc; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub trait Clock: Send + Sync + 'static { + /// Unix time in seconds. + fn now(&self) -> i64; +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> i64 { + SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0) + } +} + +/// A clock that only moves when told to. +#[derive(Clone, Debug, Default)] +pub struct ManualClock(Arc); + +impl ManualClock { + pub fn new(now: i64) -> Self { + Self(Arc::new(AtomicI64::new(now))) + } + + pub fn set(&self, now: i64) { + self.0.store(now, Ordering::SeqCst); + } + + pub fn advance(&self, secs: i64) -> i64 { + self.0.fetch_add(secs, Ordering::SeqCst) + secs + } +} + +impl Clock for ManualClock { + fn now(&self) -> i64 { + self.0.load(Ordering::SeqCst) + } +} diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs new file mode 100644 index 000000000..591be513f --- /dev/null +++ b/crates/engine/src/config.rs @@ -0,0 +1,134 @@ +//! Engine tuning knobs. Every duration used for scheduling is a local timer; +//! protocol deadlines are block heights and belong to the aggregates. + +use std::time::Duration; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Backoff { + /// Fixed steps in seconds; the last step repeats. + Steps(Vec), + /// `base` doubled per attempt, capped at `cap`, in seconds. + Doubling { base: i64, cap: i64 }, +} + +impl Backoff { + /// Delay in seconds before attempt number `attempt` (1-based) is retried. + pub fn delay(&self, attempt: i64) -> i64 { + let attempt = attempt.max(1); + match self { + Backoff::Steps(steps) => { + if steps.is_empty() { + return 0; + } + let index = usize::try_from(attempt - 1).unwrap_or(usize::MAX).min(steps.len() - 1); + steps[index] + } + Backoff::Doubling { base, cap } => { + let shift = u32::try_from(attempt - 1).unwrap_or(u32::MAX).min(30); + base.saturating_mul(1i64 << shift).min(*cap) + } + } + } +} + +#[derive(Clone, Debug)] +pub struct EngineConfig { + /// Aggregate workers; aggregates are sharded by `shard_key % worker_count`. + /// SQLite has a single writer and every apply holds the write lock, so more + /// workers only add lock contention; keep 1 unless applies wait on I/O. + pub worker_count: usize, + /// Stack size for worker and CPU job threads; 0 runs them as tokio tasks. + pub thread_stack_bytes: usize, + /// Upper bound for one apply; a timed-out apply is retried as transient. + pub apply_timeout: Duration, + /// How often a worker polls when nothing woke it. + pub worker_poll_interval: Duration, + /// Full re-check of every waiting event, and refresh of row-count metrics. + pub recheck_interval: Duration, + /// Retry schedule for transient apply failures. + pub inbox_retry_backoff: Backoff, + /// Re-publish schedule for open outbox rows. + pub outbox_backoff: Backoff, + pub outbox_batch: i64, + pub outbox_lease_secs: i64, + pub outbox_poll_interval: Duration, + pub job_lease_secs: i64, + pub job_lease_renew_interval: Duration, + pub job_retry_backoff: Backoff, + pub job_poll_interval: Duration, + /// How long the n-th consecutive poison of one action (a job action or an + /// event kind of an aggregate) holds that action back. The first poison + /// holds nothing back; an action that panics on every block then waits out + /// growing steps instead of re-running on every block. A clean run of the + /// action or a manual clear starts the count over. + pub poison_backoff: Backoff, + /// Concurrent CPU-bound jobs (the `J` slots). + pub cpu_pool_slots: usize, + /// Concurrent CPU-bound peg-out jobs; separate slots so a graph build + /// never holds a challenge or an assert back. + pub pegout_pool_slots: usize, + /// Concurrent network-bound jobs. + pub network_pool_slots: usize, + /// Bounded ACK return channel; a full channel drops the ACK and relies on resend. + pub ack_channel_capacity: usize, + /// Terminal inbox / outbox / job rows are purged after this long. + pub terminal_retention_secs: i64, + pub housekeeping_interval: Duration, +} + +impl Default for EngineConfig { + fn default() -> Self { + Self { + worker_count: 1, + thread_stack_bytes: 32 * 1024 * 1024, + apply_timeout: Duration::from_secs(120), + worker_poll_interval: Duration::from_secs(1), + recheck_interval: Duration::from_secs(60), + inbox_retry_backoff: Backoff::Steps(vec![10, 30, 60, 120, 300, 600]), + outbox_backoff: Backoff::Doubling { base: 30, cap: 600 }, + outbox_batch: 16, + outbox_lease_secs: 60, + outbox_poll_interval: Duration::from_secs(1), + job_lease_secs: 300, + job_lease_renew_interval: Duration::from_secs(60), + job_retry_backoff: Backoff::Steps(vec![10, 30, 60, 120, 300, 600]), + job_poll_interval: Duration::from_secs(1), + poison_backoff: Backoff::Steps(vec![0, 600, 3600, 6 * 3600, 24 * 3600]), + cpu_pool_slots: 1, + pegout_pool_slots: 1, + network_pool_slots: 4, + ack_channel_capacity: 256, + terminal_retention_secs: 15 * 24 * 60 * 60, + housekeeping_interval: Duration::from_secs(60 * 60), + } + } +} + +impl EngineConfig { + /// Seconds a waiting event stays parked before the worker re-checks it on + /// its own (wakeups make it runnable earlier). + pub fn waiting_recheck_secs(&self) -> i64 { + self.recheck_interval.as_secs().max(1) as i64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backoff_schedules() { + let steps = Backoff::Steps(vec![10, 30, 60]); + assert_eq!(steps.delay(0), 10); + assert_eq!(steps.delay(1), 10); + assert_eq!(steps.delay(3), 60); + assert_eq!(steps.delay(99), 60); + let doubling = Backoff::Doubling { base: 30, cap: 600 }; + assert_eq!(doubling.delay(1), 30); + assert_eq!(doubling.delay(2), 60); + assert_eq!(doubling.delay(5), 480); + assert_eq!(doubling.delay(6), 600); + assert_eq!(doubling.delay(40), 600); + assert_eq!(Backoff::Steps(vec![]).delay(1), 0); + } +} diff --git a/crates/engine/src/effects.rs b/crates/engine/src/effects.rs new file mode 100644 index 000000000..1dc994469 --- /dev/null +++ b/crates/engine/src/effects.rs @@ -0,0 +1,206 @@ +//! Everything an apply may change besides the aggregate's own business rows. +//! The engine writes these in the same transaction as the apply. + +use crate::event::AggregateRef; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use store::EngineOutboxCloseTarget; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum JobPool { + /// CPU-bound setup work (garbled circuits, graph builds). + Cpu, + /// CPU-bound peg-out work, on its own slots: a challenge or an assert + /// answers a timelock and must never queue behind a graph build. + Pegout, + Network, +} + +impl JobPool { + pub const fn as_str(self) -> &'static str { + match self { + JobPool::Cpu => "cpu", + JobPool::Pegout => "pegout", + JobPool::Network => "network", + } + } +} + +/// Stable job id: `H(aggregate ‖ phase ‖ action ‖ params hash)`. +pub fn stable_job_id(aggregate: &AggregateRef, phase: &str, action: &str, params: &[u8]) -> String { + let params_hash = Sha256::digest(params); + let mut hasher = Sha256::new(); + hasher.update(aggregate.kind.as_bytes()); + hasher.update(b":"); + hasher.update(aggregate.id.as_bytes()); + hasher.update(b":"); + hasher.update(phase.as_bytes()); + hasher.update(b":"); + hasher.update(action.as_bytes()); + hasher.update(b":"); + hasher.update(params_hash); + hex::encode(hasher.finalize()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OutboxMessage { + pub outbox_id: String, + pub phase_ref: String, + pub kind: String, + pub topic: String, + pub payload: Vec, + /// Heavy message: outstanding recipients (peer id -> delivery id). + pub ack_set: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JobSpec { + pub job_id: String, + pub phase: String, + pub action: String, + pub pool: JobPool, + pub params: Vec, + /// Unix time after which the job is failed instead of run. + pub deadline_at: Option, + /// Unix time before which the job is not claimed (a local timer, never a + /// protocol deadline). + pub run_after: Option, +} + +impl JobSpec { + /// A job with the stable id derived from its identity. + pub fn new( + aggregate: &AggregateRef, + phase: impl Into, + action: impl Into, + pool: JobPool, + params: Vec, + ) -> Self { + let phase = phase.into(); + let action = action.into(); + let job_id = stable_job_id(aggregate, &phase, &action, ¶ms); + Self { job_id, phase, action, pool, params, deadline_at: None, run_after: None } + } + + pub fn with_deadline(mut self, deadline_at: i64) -> Self { + self.deadline_at = Some(deadline_at); + self + } + + pub fn with_run_after(mut self, run_after: i64) -> Self { + self.run_after = Some(run_after); + self + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum JobCancel { + Job(String), + Phase(String), + /// Every job of another aggregate (for example the graphs of a failed instance). + Aggregate { + aggregate_kind: String, + aggregate_id: String, + }, + /// One phase of another aggregate's jobs. + AggregatePhase { + aggregate_kind: String, + aggregate_id: String, + phase: String, + }, +} + +#[derive(Debug, Default)] +pub struct Effects { + pub status: Option, + pub phase: Option, + pub state: Option>, + pub outbox: Vec, + pub outbox_closes: Vec<(EngineOutboxCloseTarget, String)>, + pub jobs: Vec, + pub job_cancels: Vec, + pub wake_refs: Vec, + pub acks: Vec, +} + +impl Effects { + pub fn set_status(&mut self, status: impl Into) -> &mut Self { + self.status = Some(status.into()); + self + } + + pub fn set_phase(&mut self, phase: impl Into) -> &mut Self { + self.phase = Some(phase.into()); + self + } + + /// Replace the opaque aggregate state (`None` clears it). + pub fn set_state(&mut self, state: Option) -> &mut Self { + self.state = Some(state); + self + } + + pub fn send(&mut self, message: OutboxMessage) -> &mut Self { + self.outbox.push(message); + self + } + + pub fn close_outbox( + &mut self, + target: EngineOutboxCloseTarget, + reason: impl Into, + ) -> &mut Self { + self.outbox_closes.push((target, reason.into())); + self + } + + pub fn enqueue_job(&mut self, job: JobSpec) -> &mut Self { + self.jobs.push(job); + self + } + + pub fn cancel_job(&mut self, job_id: impl Into) -> &mut Self { + self.job_cancels.push(JobCancel::Job(job_id.into())); + self + } + + pub fn cancel_phase_jobs(&mut self, phase: impl Into) -> &mut Self { + self.job_cancels.push(JobCancel::Phase(phase.into())); + self + } + + pub fn cancel_jobs_for_aggregate(&mut self, aggregate: &AggregateRef) -> &mut Self { + self.job_cancels.push(JobCancel::Aggregate { + aggregate_kind: aggregate.kind.clone(), + aggregate_id: aggregate.id.clone(), + }); + self + } + + /// Cancel the jobs of one phase of another aggregate. + pub fn cancel_phase_jobs_for_aggregate( + &mut self, + aggregate: &AggregateRef, + phase: impl Into, + ) -> &mut Self { + self.job_cancels.push(JobCancel::AggregatePhase { + aggregate_kind: aggregate.kind.clone(), + aggregate_id: aggregate.id.clone(), + phase: phase.into(), + }); + self + } + + /// Wake events waiting on `wake_ref` (status and phase changes wake the + /// aggregate's own ref automatically). + pub fn wake(&mut self, wake_ref: impl Into) -> &mut Self { + self.wake_refs.push(wake_ref.into()); + self + } + + pub fn ack(&mut self, peer: impl Into, delivery_id: impl Into) -> &mut Self { + self.acks + .push(store::EngineAckTarget { peer: peer.into(), delivery_id: delivery_id.into() }); + self + } +} diff --git a/crates/engine/src/event.rs b/crates/engine/src/event.rs new file mode 100644 index 000000000..47bb57ed0 --- /dev/null +++ b/crates/engine/src/event.rs @@ -0,0 +1,176 @@ +//! Inbox events and aggregate references. + +use serde::{Deserialize, Serialize}; +use store::{EngineAckTarget, EngineInboxEvent, engine_shard_key}; + +/// Identity of an aggregate: its kind (`instance`, `graph`, ...) and id. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct AggregateRef { + pub kind: String, + pub id: String, +} + +impl AggregateRef { + pub fn new(kind: impl Into, id: impl Into) -> Self { + Self { kind: kind.into(), id: id.into() } + } + + pub fn from_event(event: &EngineInboxEvent) -> Self { + Self { kind: event.aggregate_kind.clone(), id: event.aggregate_id.clone() } + } + + /// The reference other aggregates wait on when this one changes. + pub fn wake_ref(&self) -> String { + format!("{}:{}", self.kind, self.id) + } + + pub fn shard_key(&self) -> i64 { + engine_shard_key(&self.kind, &self.id) + } + + pub fn shard(&self, shard_count: usize) -> usize { + (self.shard_key() % shard_count.max(1) as i64) as usize + } +} + +impl std::fmt::Display for AggregateRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.kind, self.id) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum EventSource { + Network, + Chain, + Height, + Job, + Local, +} + +impl EventSource { + pub const fn as_str(self) -> &'static str { + match self { + EventSource::Network => "network", + EventSource::Chain => "chain", + EventSource::Height => "height", + EventSource::Job => "job", + EventSource::Local => "local", + } + } +} + +/// An event to record in the inbox. `event_id` is the dedup key and must be +/// derived from the content (content id, chain event id, height key, job +/// result id), never from arrival. +#[derive(Clone, Debug)] +pub struct NewEvent { + pub event_id: String, + pub aggregate: AggregateRef, + pub kind: String, + pub source: EventSource, + pub author: String, + /// How admission classified the author of a network event; the storage + /// quota is accounted per class. Empty for every other source. + pub admission_class: String, + pub payload: Vec, + /// For heavy messages: who to ACK once the material is durably applied or parked. + pub ack: Option, +} + +impl NewEvent { + pub fn new( + event_id: impl Into, + aggregate: AggregateRef, + kind: impl Into, + source: EventSource, + payload: Vec, + ) -> Self { + Self { + event_id: event_id.into(), + aggregate, + kind: kind.into(), + source, + author: String::new(), + admission_class: String::new(), + payload, + ack: None, + } + } + + pub fn with_author(mut self, author: impl Into) -> Self { + self.author = author.into(); + self + } + + pub fn with_admission_class(mut self, class: impl Into) -> Self { + self.admission_class = class.into(); + self + } + + pub fn with_ack(mut self, peer: impl Into, delivery_id: impl Into) -> Self { + self.ack = Some(EngineAckTarget { peer: peer.into(), delivery_id: delivery_id.into() }); + self + } + + /// The store row for this event, for producers that record it inside + /// their own transaction (see [`crate::EngineHandle::submit_in`]). + pub fn into_store(self) -> store::NewEngineInboxEvent { + store::NewEngineInboxEvent { + event_id: self.event_id, + aggregate_kind: self.aggregate.kind, + aggregate_id: self.aggregate.id, + kind: self.kind, + source: self.source.as_str().to_string(), + author: self.author, + admission_class: self.admission_class, + payload: self.payload, + ack: self.ack, + } + } +} + +/// Event kind used for job completions. +pub const JOB_RESULT_KIND: &str = "job_result"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum JobResultOutcome { + Done(Vec), + Failed(String), + Cancelled, + Poisoned(String), +} + +/// Payload of a `job_result` event. `stale` is true when the completing holder +/// had lost its lease: the job row was not updated, but the external effect may +/// have happened, so the aggregate reconciles it against current state. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct JobResultEvent { + pub job_id: String, + pub phase: String, + pub action: String, + pub generation: i64, + pub stale: bool, + pub outcome: JobResultOutcome, +} + +impl JobResultEvent { + pub fn event_id(job_id: &str, generation: i64) -> String { + format!("job-result:{job_id}:{generation}") + } + + pub fn decode(payload: &[u8]) -> anyhow::Result { + Ok(serde_json::from_slice(payload)?) + } + + pub fn into_new_event(self, aggregate: AggregateRef) -> anyhow::Result { + let payload = serde_json::to_vec(&self)?; + Ok(NewEvent::new( + Self::event_id(&self.job_id, self.generation), + aggregate, + JOB_RESULT_KIND, + EventSource::Job, + payload, + )) + } +} diff --git a/crates/engine/src/jobs.rs b/crates/engine/src/jobs.rs new file mode 100644 index 000000000..d5ad77f45 --- /dev/null +++ b/crates/engine/src/jobs.rs @@ -0,0 +1,516 @@ +//! Leased jobs with generation fencing. A job is claimed with a lease and a new +//! generation; the holder renews the lease while running and reports with its +//! generation. A holder that lost its lease is fenced at completion, but its +//! external result still reaches the aggregate as a stale `job_result` event. + +use crate::clock::Clock; +use crate::config::EngineConfig; +use crate::effects::JobPool; +use crate::event::{AggregateRef, JobResultEvent, JobResultOutcome}; +use crate::metrics::EngineMetrics; +use crate::wake::{EngineWake, panic_message, spawn_on_thread}; +use async_trait::async_trait; +use futures::FutureExt; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use store::localdb::LocalDB; +use store::{ + EngineInboxInsert, EngineJob, EngineJobFinish, EngineJobFinishOutcome, EngineJobState, +}; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +pub struct JobContext { + pub now: i64, + pub job: EngineJob, + /// Cancelled when the lease is lost or the engine shuts down. + pub cancel: CancellationToken, +} + +#[derive(Debug)] +pub enum JobOutcome { + Done(Vec), + /// Retry after `retry_after_secs`, or the configured backoff when `None`. + Transient { + error: String, + retry_after_secs: Option, + }, + /// Permanent business failure; recorded and reported to the aggregate. + Failed(String), +} + +#[async_trait] +pub trait JobRunner: Send + Sync { + async fn run(&self, ctx: JobContext) -> JobOutcome; +} + +pub(crate) type JobRegistry = HashMap<(String, String), Arc>; + +#[derive(Clone)] +pub(crate) struct ExecutorDeps { + pub db: LocalDB, + pub clock: Arc, + pub config: Arc, + pub registry: Arc, + pub metrics: EngineMetrics, + pub wake: EngineWake, +} + +pub(crate) async fn run_executor( + deps: ExecutorDeps, + notify: Arc, + cancel: CancellationToken, +) { + let pools = [ + (JobPool::Cpu, Arc::new(Semaphore::new(deps.config.cpu_pool_slots.max(1)))), + (JobPool::Pegout, Arc::new(Semaphore::new(deps.config.pegout_pool_slots.max(1)))), + (JobPool::Network, Arc::new(Semaphore::new(deps.config.network_pool_slots.max(1)))), + ]; + loop { + if cancel.is_cancelled() { + return; + } + if let Err(error) = finish_lapsed_cancelled(&deps).await { + warn!(event = "engine_jobs", outcome = "cancel_sweep_failed", error = %error, "could not finish lapsed cancelled jobs"); + } + for (pool, semaphore) in &pools { + if let Err(error) = claim_and_spawn(&deps, *pool, semaphore, &cancel).await { + warn!(event = "engine_jobs", outcome = "claim_failed", pool = pool.as_str(), error = %error, "job claim failed"); + } + } + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = notify.notified() => {} + _ = tokio::time::sleep(deps.config.job_poll_interval) => {} + } + } +} + +/// A cancelled job whose holder went away (crash, restart) is not run again: +/// it finishes as `Cancelled` here and its result reaches the aggregate. +async fn finish_lapsed_cancelled(deps: &ExecutorDeps) -> anyhow::Result<()> { + let now = deps.clock.now(); + let mut tx = deps.db.start_immediate_transaction().await?; + let jobs = tx.finish_lapsed_cancelled_engine_jobs(now).await?; + if jobs.is_empty() { + tx.commit().await?; + return Ok(()); + } + let mut aggregates = Vec::with_capacity(jobs.len()); + for job in &jobs { + let aggregate = AggregateRef::new(&job.aggregate_kind, &job.aggregate_id); + let event = JobResultEvent { + job_id: job.job_id.clone(), + phase: job.phase.clone(), + action: job.action.clone(), + generation: job.generation, + stale: false, + outcome: JobResultOutcome::Cancelled, + } + .into_new_event(aggregate.clone())?; + tx.insert_engine_inbox_event(&event.into_store(), now).await?; + aggregates.push(aggregate); + } + tx.commit().await?; + for (job, aggregate) in jobs.iter().zip(&aggregates) { + deps.metrics.record_job_run(&job.action, "cancelled"); + info!( + event = "engine_jobs", + outcome = "cancelled", + job_id = %job.job_id, + action = %job.action, + generation = job.generation, + "cancelled job lost its holder; finished without re-running" + ); + deps.wake.worker_for(aggregate); + } + Ok(()) +} + +async fn claim_and_spawn( + deps: &ExecutorDeps, + pool: JobPool, + semaphore: &Arc, + cancel: &CancellationToken, +) -> anyhow::Result<()> { + loop { + let available = semaphore.available_permits(); + if available == 0 { + return Ok(()); + } + let now = deps.clock.now(); + let jobs = { + let mut storage = deps.db.acquire().await?; + storage + .claim_due_engine_jobs( + pool.as_str(), + now, + now + deps.config.job_lease_secs, + available as i64, + ) + .await? + }; + if jobs.is_empty() { + return Ok(()); + } + let claimed = jobs.len(); + for job in jobs { + let permit = semaphore.clone().acquire_owned().await?; + spawn_job(deps.clone(), job, pool, permit, cancel.child_token()); + } + if claimed < available { + return Ok(()); + } + } +} + +fn spawn_job( + deps: ExecutorDeps, + job: EngineJob, + pool: JobPool, + permit: OwnedSemaphorePermit, + cancel: CancellationToken, +) { + let name = format!("engine-job-{}", &job.job_id[..job.job_id.len().min(12)]); + let stack = match pool { + JobPool::Cpu | JobPool::Pegout => deps.config.thread_stack_bytes, + JobPool::Network => 0, + }; + spawn_on_thread(name, stack, async move { + let _permit = permit; + execute_job(deps, job, cancel).await; + }); +} + +async fn execute_job(deps: ExecutorDeps, job: EngineJob, cancel: CancellationToken) { + let started = Instant::now(); + let now = deps.clock.now(); + let generation = job.generation; + let key = (job.aggregate_kind.clone(), job.action.clone()); + debug!( + event = "engine_jobs", + outcome = "started", + job_id = %job.job_id, + action = %job.action, + generation, + attempt = job.attempts, + "job claimed" + ); + let mut renewal: Option> = None; + let finish = if job.deadline_at.is_some_and(|deadline| deadline < now) { + EngineJobFinish::Failed(format!( + "deadline {} passed at {now}", + job.deadline_at.unwrap_or(0) + )) + } else { + match deps.registry.get(&key) { + None => EngineJobFinish::Failed(format!( + "no runner registered for {}:{}", + job.aggregate_kind, job.action + )), + Some(runner) => { + renewal = Some(spawn_lease_renewal( + deps.clone(), + job.job_id.clone(), + generation, + cancel.clone(), + )); + let ctx = JobContext { now, job: job.clone(), cancel: cancel.clone() }; + let outcome = std::panic::AssertUnwindSafe(runner.run(ctx)).catch_unwind().await; + match outcome { + Ok(JobOutcome::Done(result)) => EngineJobFinish::Done(result), + Ok(JobOutcome::Transient { error, retry_after_secs }) => { + // An explicit wait names its own delay; an error path falls + // back to the backoff and is the one worth a WARN. + let delay = match retry_after_secs { + Some(secs) => secs, + None => { + warn!( + event = "engine_jobs", + outcome = "attempt_error", + job_id = %job.job_id, + action = %job.action, + generation, + attempts = job.attempts + 1, + error = %error, + "job attempt failed; retrying with backoff" + ); + deps.config.job_retry_backoff.delay(job.attempts) + } + }; + EngineJobFinish::Retry { error, next_at: deps.clock.now() + delay } + } + Ok(JobOutcome::Failed(error)) => EngineJobFinish::Failed(error), + Err(payload) => EngineJobFinish::Poisoned(panic_message(payload.as_ref())), + } + } + } + }; + // Retry completion recording while renewing the lease to avoid rerunning work + // whose result was not saved. After lease loss, make bounded attempts to record + // the result for reconciliation. + let mut attempt: u32 = 0; + let mut attempts_after_loss: u32 = 0; + let last_error = loop { + match record_job_finish(&deps, &job, generation, &finish, started).await { + Ok(()) => { + if let Some(renewal) = renewal { + renewal.abort(); + } + return; + } + Err(error) => { + attempt += 1; + let cancelled = cancel.is_cancelled(); + if cancelled { + attempts_after_loss += 1; + } + warn!( + event = "engine_jobs", + outcome = "finish_retry", + job_id = %job.job_id, + generation, + attempt, + cancelled, + error = %error, + "could not record job completion; retrying" + ); + if attempts_after_loss >= JOB_FINISH_ATTEMPTS_AFTER_LEASE_LOSS { + break error; + } + let delay = JOB_FINISH_RETRY_BASE.saturating_mul(attempt.min(8)); + tokio::time::sleep(delay).await; + } + } + }; + if let Some(renewal) = renewal { + renewal.abort(); + } + error!( + event = "engine_jobs", + outcome = "finish_failed", + job_id = %job.job_id, + generation, + attempts = attempt, + error = %last_error, + "could not record job completion before the lease was lost or the engine stopped; a newer holder re-runs the job" + ); +} + +const JOB_FINISH_RETRY_BASE: Duration = Duration::from_millis(250); +const JOB_FINISH_ATTEMPTS_AFTER_LEASE_LOSS: u32 = 2; + +fn spawn_lease_renewal( + deps: ExecutorDeps, + job_id: String, + generation: i64, + cancel: CancellationToken, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(deps.config.job_lease_renew_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval.tick().await; + loop { + interval.tick().await; + let now = deps.clock.now(); + let renewed = match deps.db.acquire().await { + Ok(mut storage) => { + storage + .renew_engine_job_lease( + &job_id, + generation, + now + deps.config.job_lease_secs, + now, + ) + .await + } + Err(error) => Err(error), + }; + match renewed { + Ok(true) => {} + Ok(false) => { + // Our own finish may have committed just before this tick. + let finished = match deps.db.acquire().await { + Ok(mut storage) => storage.find_engine_job(&job_id).await.ok().flatten(), + Err(_) => None, + } + .is_some_and(|row| { + row.generation == generation + && row.state != EngineJobState::Running.to_string() + }); + if finished { + debug!(event = "engine_jobs", outcome = "renewal_done", job_id = %job_id, generation, "job finished; lease renewal stopped"); + return; + } + warn!( + event = "engine_jobs", + outcome = "lease_lost", + job_id = %job_id, + generation, + "job lease was taken over by a newer generation; cancelling this holder" + ); + cancel.cancel(); + return; + } + Err(error) => { + warn!(event = "engine_jobs", outcome = "renew_failed", job_id = %job_id, error = %error, "lease renewal failed; retrying"); + } + } + } + }) +} + +async fn record_job_finish( + deps: &ExecutorDeps, + job: &EngineJob, + generation: i64, + finish: &EngineJobFinish, + started: Instant, +) -> anyhow::Result<()> { + let now = deps.clock.now(); + let aggregate = AggregateRef::new(&job.aggregate_kind, &job.aggregate_id); + let mut tx = deps.db.start_immediate_transaction().await?; + let outcome = tx.finish_engine_job(&job.job_id, generation, finish, now).await?; + let (stale, result_outcome, label) = match (&outcome, finish) { + (EngineJobFinishOutcome::Applied { state }, finish) => { + let result_outcome = match (state, finish) { + (EngineJobState::Done, EngineJobFinish::Done(result)) => { + Some(JobResultOutcome::Done(result.clone())) + } + (EngineJobState::Cancelled, _) => Some(JobResultOutcome::Cancelled), + (EngineJobState::Failed, EngineJobFinish::Failed(error)) => { + Some(JobResultOutcome::Failed(error.clone())) + } + (EngineJobState::Poisoned, EngineJobFinish::Poisoned(error)) => { + Some(JobResultOutcome::Poisoned(error.clone())) + } + _ => None, + }; + (false, result_outcome, state.to_string().to_ascii_lowercase()) + } + (EngineJobFinishOutcome::Stale { .. }, EngineJobFinish::Done(result)) => { + (true, Some(JobResultOutcome::Done(result.clone())), "stale".to_string()) + } + (EngineJobFinishOutcome::Stale { .. }, EngineJobFinish::Failed(error)) => { + (true, Some(JobResultOutcome::Failed(error.clone())), "stale".to_string()) + } + (EngineJobFinishOutcome::Stale { .. }, EngineJobFinish::Poisoned(error)) => { + (true, Some(JobResultOutcome::Poisoned(error.clone())), "stale".to_string()) + } + (EngineJobFinishOutcome::Stale { .. }, EngineJobFinish::Retry { .. }) => { + (true, None, "stale".to_string()) + } + (EngineJobFinishOutcome::Missing, _) => (true, None, "missing".to_string()), + }; + match (&outcome, finish) { + (EngineJobFinishOutcome::Applied { .. }, EngineJobFinish::Poisoned(reason)) => { + let poison = tx + .mark_engine_aggregate_poisoned( + &job.aggregate_kind, + &job.aggregate_id, + &job.action, + reason, + now, + &|count| deps.config.poison_backoff.delay(count), + ) + .await?; + deps.metrics.record_poisoned(); + error!( + event = "engine_poisoned", + job_id = %job.job_id, + aggregate = %aggregate, + action = %job.action, + poison_count = poison.poison_count, + blocked_until = poison.blocked_until, + reason = %reason, + "job panicked; this action waits for the backoff, a clean run or a manual clear" + ); + } + (EngineJobFinishOutcome::Applied { .. }, _) => { + // The action ran to a result without panicking: its poison is over. + tx.release_engine_poison(&job.aggregate_kind, &job.aggregate_id, &job.action).await?; + } + _ => {} + } + let mut wake_worker = false; + if let Some(result_outcome) = result_outcome { + let event = JobResultEvent { + job_id: job.job_id.clone(), + phase: job.phase.clone(), + action: job.action.clone(), + generation, + stale, + outcome: result_outcome, + } + .into_new_event(aggregate.clone())?; + let inserted = tx.insert_engine_inbox_event(&event.into_store(), now).await?; + wake_worker = matches!(inserted, EngineInboxInsert::Inserted { .. }); + } + tx.commit().await?; + deps.metrics.record_job_run(&job.action, &label); + let elapsed_ms = started.elapsed().as_millis() as u64; + match (&outcome, finish) { + // A retried or failed attempt names its reason here: the row keeps it in + // `last_error`, but an operator reading the log must see why. + (EngineJobFinishOutcome::Applied { state }, EngineJobFinish::Retry { error, next_at }) => { + info!( + event = "engine_jobs", + outcome = %label, + job_id = %job.job_id, + action = %job.action, + generation, + state = %state, + attempts = job.attempts + 1, + next_at, + elapsed_ms, + reason = %error, + "job attempt ended; retrying" + ) + } + (EngineJobFinishOutcome::Applied { state }, EngineJobFinish::Failed(error)) => warn!( + event = "engine_jobs", + outcome = %label, + job_id = %job.job_id, + action = %job.action, + generation, + state = %state, + elapsed_ms, + error = %error, + "job failed" + ), + (EngineJobFinishOutcome::Applied { state }, _) => info!( + event = "engine_jobs", + outcome = %label, + job_id = %job.job_id, + action = %job.action, + generation, + state = %state, + elapsed_ms, + "job finished" + ), + (EngineJobFinishOutcome::Stale { current_generation, state }, _) => warn!( + event = "engine_jobs", + outcome = "stale_completion", + job_id = %job.job_id, + action = %job.action, + generation, + current_generation, + state = %state, + elapsed_ms, + "late completion fenced; the external result is recorded for reconciliation" + ), + (EngineJobFinishOutcome::Missing, _) => warn!( + event = "engine_jobs", + outcome = "missing", + job_id = %job.job_id, + "job row disappeared before completion" + ), + } + if wake_worker { + deps.wake.worker_for(&aggregate); + } + Ok(()) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs new file mode 100644 index 000000000..4b86d35da --- /dev/null +++ b/crates/engine/src/lib.rs @@ -0,0 +1,36 @@ +//! Execution engine for persisted events, jobs and P2P messages. +//! +//! Aggregate handlers apply events in a transaction that also records jobs, +//! outbox messages and dependency waits. Job results return as inbox events. +//! The node supplies [`AggregateHandler`]s, [`JobRunner`]s and a [`Transport`]. + +pub mod ack; +pub mod aggregate; +pub mod clock; +pub mod config; +pub mod effects; +pub mod event; +pub mod jobs; +pub mod metrics; +pub mod outbox; +pub mod runtime; +mod wake; +mod worker; + +pub use ack::{AckReceiver, AckRequest, AckSender}; +pub use aggregate::{AggregateHandler, ApplyContext, ApplyError, ApplyOutcome, WaitingFor}; +pub use clock::{Clock, ManualClock, SystemClock}; +pub use config::{Backoff, EngineConfig}; +pub use effects::{Effects, JobCancel, JobPool, JobSpec, OutboxMessage, stable_job_id}; +pub use event::{ + AggregateRef, EventSource, JOB_RESULT_KIND, JobResultEvent, JobResultOutcome, NewEvent, +}; +pub use jobs::{JobContext, JobOutcome, JobRunner}; +pub use metrics::EngineMetrics; +pub use outbox::Transport; +pub use runtime::{Engine, EngineBuilder, EngineHandle, EngineSnapshot, SubmitOutcome}; +pub use store::{ + EngineAggregate, EngineInboxEvent, EngineInboxState, EngineJob, EngineJobInsert, + EngineJobState, EngineOutboxAck, EngineOutboxCloseTarget, EngineOutboxMessage, + EngineOutboxState, EnginePoison, EngineWaitingEvent, +}; diff --git a/crates/engine/src/metrics.rs b/crates/engine/src/metrics.rs new file mode 100644 index 000000000..68e701778 --- /dev/null +++ b/crates/engine/src/metrics.rs @@ -0,0 +1,148 @@ +//! Engine metrics: counts by kind and outcome, apply latency, and row counts by +//! state for the four tables. + +use prometheus_client::encoding::EncodeLabelSet; +use prometheus_client::metrics::counter::Counter; +use prometheus_client::metrics::family::Family; +use prometheus_client::metrics::gauge::Gauge; +use prometheus_client::metrics::histogram::{Histogram, exponential_buckets}; +use prometheus_client::registry::Registry; +use std::time::Duration; + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct KindOutcomeLabels { + kind: String, + outcome: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct KindLabels { + kind: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct OutcomeLabels { + outcome: String, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +struct TableStateLabels { + table: String, + state: String, +} + +#[derive(Clone, Debug)] +pub struct EngineMetrics { + events_total: Family, + apply_duration_seconds: Family, + outbox_publish_total: Family, + job_runs_total: Family, + rows: Family, + acks_dropped_total: Counter, + poisoned_total: Counter, +} + +impl Default for EngineMetrics { + fn default() -> Self { + Self::new() + } +} + +impl EngineMetrics { + pub fn new() -> Self { + Self { + events_total: Family::default(), + apply_duration_seconds: Family::new_with_constructor(|| { + Histogram::new(exponential_buckets(0.005, 2.0, 15)) + }), + outbox_publish_total: Family::default(), + job_runs_total: Family::default(), + rows: Family::default(), + acks_dropped_total: Counter::default(), + poisoned_total: Counter::default(), + } + } + + pub fn register(&self, registry: &mut Registry) { + registry.register( + "bitvm_node_engine_events", + "Engine inbox events by kind and apply outcome", + self.events_total.clone(), + ); + registry.register( + "bitvm_node_engine_apply_duration_seconds", + "Engine apply duration by event kind", + self.apply_duration_seconds.clone(), + ); + registry.register( + "bitvm_node_engine_outbox_publish", + "Engine outbox publish attempts by outcome", + self.outbox_publish_total.clone(), + ); + registry.register( + "bitvm_node_engine_job_runs", + "Engine job runs by action and outcome", + self.job_runs_total.clone(), + ); + registry.register( + "bitvm_node_engine_rows", + "Engine table rows by state", + self.rows.clone(), + ); + registry.register( + "bitvm_node_engine_acks_dropped", + "ACKs dropped because the return channel was full", + self.acks_dropped_total.clone(), + ); + registry.register( + "bitvm_node_engine_poisoned", + "Events and jobs poisoned by a panic or invariant violation", + self.poisoned_total.clone(), + ); + } + + pub fn record_event(&self, kind: &str, outcome: &str) { + self.events_total + .get_or_create(&KindOutcomeLabels { + kind: kind.to_string(), + outcome: outcome.to_string(), + }) + .inc(); + } + + pub fn record_apply_duration(&self, kind: &str, elapsed: Duration) { + self.apply_duration_seconds + .get_or_create(&KindLabels { kind: kind.to_string() }) + .observe(elapsed.as_secs_f64()); + } + + pub fn record_publish(&self, success: bool) { + let outcome = if success { "success" } else { "failed" }; + self.outbox_publish_total + .get_or_create(&OutcomeLabels { outcome: outcome.to_string() }) + .inc(); + } + + pub fn record_job_run(&self, action: &str, outcome: &str) { + self.job_runs_total + .get_or_create(&KindOutcomeLabels { + kind: action.to_string(), + outcome: outcome.to_string(), + }) + .inc(); + } + + pub fn set_rows(&self, table: &str, state: &str, count: i64) { + self.rows + .get_or_create(&TableStateLabels { table: table.to_string(), state: state.to_string() }) + .set(count); + } + + pub fn record_ack_dropped(&self) { + self.acks_dropped_total.inc(); + } + + pub fn record_poisoned(&self) { + self.poisoned_total.inc(); + } +} diff --git a/crates/engine/src/outbox.rs b/crates/engine/src/outbox.rs new file mode 100644 index 000000000..5c04fdd42 --- /dev/null +++ b/crates/engine/src/outbox.rs @@ -0,0 +1,118 @@ +//! The publisher: the only sender. It re-publishes open outbox rows with +//! backoff until an aggregate closes them (ACKs collected, receiver evidence, +//! instance failed, graph unusable). + +use crate::clock::Clock; +use crate::config::EngineConfig; +use crate::metrics::EngineMetrics; +use async_trait::async_trait; +use std::sync::Arc; +use store::localdb::LocalDB; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; + +/// Wire transport. The engine hands over topic and encoded payload; the +/// transport decides how they reach the network. +#[async_trait] +pub trait Transport: Send + Sync { + async fn publish(&self, topic: &str, payload: &[u8]) -> anyhow::Result<()>; +} + +#[derive(Clone)] +pub(crate) struct PublisherDeps { + pub db: LocalDB, + pub clock: Arc, + pub config: Arc, + pub transport: Arc, + pub metrics: EngineMetrics, +} + +pub(crate) async fn run_publisher( + deps: PublisherDeps, + notify: Arc, + cancel: CancellationToken, +) { + loop { + if cancel.is_cancelled() { + return; + } + if let Err(error) = drain_outbox(&deps).await { + warn!(event = "engine_outbox", outcome = "drain_failed", error = %error, "outbox drain failed"); + } + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = notify.notified() => {} + _ = tokio::time::sleep(deps.config.outbox_poll_interval) => {} + } + } +} + +/// Publish every due row once. Returns the number of successful publishes. +pub(crate) async fn drain_outbox(deps: &PublisherDeps) -> anyhow::Result { + let mut published = 0; + loop { + let now = deps.clock.now(); + let batch = { + let mut storage = deps.db.acquire().await?; + storage + .claim_due_engine_outbox_messages( + now, + now + deps.config.outbox_lease_secs, + deps.config.outbox_batch, + ) + .await? + }; + if batch.is_empty() { + return Ok(published); + } + let claimed = batch.len() as i64; + for row in batch { + let result = deps.transport.publish(&row.topic, &row.payload).await; + let now = deps.clock.now(); + let mut storage = deps.db.acquire().await?; + match result { + Ok(()) => { + let next_at = now + deps.config.outbox_backoff.delay(row.publish_count + 1); + storage.record_engine_outbox_publish(&row.outbox_id, next_at, now).await?; + deps.metrics.record_publish(true); + published += 1; + debug!( + event = "engine_outbox", + outcome = "published", + outbox_id = %row.outbox_id, + kind = %row.kind, + publish_count = row.publish_count + 1, + next_at, + "outbox row published" + ); + } + Err(error) => { + let next_at = now + deps.config.outbox_backoff.delay(1); + storage + .record_engine_outbox_publish_failure( + &row.outbox_id, + &error.to_string(), + next_at, + now, + ) + .await?; + deps.metrics.record_publish(false); + warn!( + event = "engine_outbox", + outcome = "publish_failed", + outbox_id = %row.outbox_id, + kind = %row.kind, + error = %error, + next_at, + "outbox publish failed; will retry" + ); + } + } + } + if claimed < deps.config.outbox_batch { + return Ok(published); + } + } +} diff --git a/crates/engine/src/runtime.rs b/crates/engine/src/runtime.rs new file mode 100644 index 000000000..39197fb0b --- /dev/null +++ b/crates/engine/src/runtime.rs @@ -0,0 +1,464 @@ +//! Engine assembly: builder, handle for producers, and the run loop that owns +//! the workers, publisher, executor, re-check timer and housekeeping. + +use crate::ack::{AckReceiver, AckRequest, AckSender}; +use crate::aggregate::AggregateHandler; +use crate::clock::{Clock, SystemClock}; +use crate::config::EngineConfig; +use crate::event::{AggregateRef, NewEvent}; +use crate::jobs::{ExecutorDeps, JobRegistry, JobRunner, run_executor}; +use crate::metrics::EngineMetrics; +use crate::outbox::{PublisherDeps, Transport, run_publisher}; +use crate::wake::{EngineWake, spawn_on_thread}; +use crate::worker::{WorkerDeps, run_worker}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use store::localdb::{LocalDB, StorageProcessor}; +use store::{ + EngineAggregate, EngineInboxInsert, EngineInboxState, EngineOutboxAck, EnginePoison, + EngineStateCount, EngineWaitingEvent, +}; +use tokio_util::sync::CancellationToken; +use tracing::{info, warn}; + +struct EngineInner { + db: LocalDB, + config: Arc, + clock: Arc, + handlers: Arc>>, + runners: Arc, + transport: Arc, + metrics: EngineMetrics, + wake: EngineWake, + ack: AckSender, +} + +/// A transport that drops everything; used when no network is attached. +struct NullTransport; + +#[async_trait::async_trait] +impl Transport for NullTransport { + async fn publish(&self, _topic: &str, _payload: &[u8]) -> anyhow::Result<()> { + Err(anyhow::anyhow!("no transport attached to the engine")) + } +} + +pub struct EngineBuilder { + db: LocalDB, + config: EngineConfig, + clock: Arc, + handlers: HashMap>, + runners: JobRegistry, + transport: Option>, + metrics: Option, +} + +impl EngineBuilder { + pub fn new(db: LocalDB, config: EngineConfig) -> Self { + Self { + db, + config, + clock: Arc::new(SystemClock), + handlers: HashMap::new(), + runners: HashMap::new(), + transport: None, + metrics: None, + } + } + + pub fn clock(mut self, clock: impl Clock) -> Self { + self.clock = Arc::new(clock); + self + } + + pub fn handler(mut self, kind: impl Into, handler: Arc) -> Self { + self.handlers.insert(kind.into(), handler); + self + } + + pub fn job_runner( + mut self, + kind: impl Into, + action: impl Into, + runner: Arc, + ) -> Self { + self.runners.insert((kind.into(), action.into()), runner); + self + } + + pub fn transport(mut self, transport: Arc) -> Self { + self.transport = Some(transport); + self + } + + pub fn metrics(mut self, metrics: EngineMetrics) -> Self { + self.metrics = Some(metrics); + self + } + + /// Build the engine and the receiving end of the ACK channel. + pub fn build(self) -> (Engine, AckReceiver) { + let (ack, ack_rx) = AckSender::channel(self.config.ack_channel_capacity); + let wake = EngineWake::new(self.config.worker_count); + let inner = EngineInner { + db: self.db, + config: Arc::new(self.config), + clock: self.clock, + handlers: Arc::new(self.handlers), + runners: Arc::new(self.runners), + transport: self.transport.unwrap_or_else(|| Arc::new(NullTransport)), + metrics: self.metrics.unwrap_or_default(), + wake, + ack, + }; + (Engine { inner: Arc::new(inner) }, ack_rx) + } +} + +pub struct Engine { + inner: Arc, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SubmitOutcome { + Inserted, + Duplicate { state: EngineInboxState, ack_sent: bool, rearmed: bool }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EngineSnapshot { + pub counts: Vec, + /// Parked events, oldest park first. + pub waiting: Vec, + /// Actions held back by a panic, oldest first. + pub poisoned: Vec, +} + +/// Producer-side handle: record events, apply ACKs, wake waiters. +#[derive(Clone)] +pub struct EngineHandle { + inner: Arc, +} + +impl EngineHandle { + /// Record an event once and wake its shard. A duplicate heavy message whose + /// material is already applied or parked is ACKed again. + pub async fn submit(&self, event: NewEvent) -> anyhow::Result { + let now = self.inner.clock.now(); + let aggregate = event.aggregate.clone(); + let kind = event.kind.clone(); + let ack = event.ack.clone(); + let inserted = { + let mut storage = self.inner.db.acquire().await?; + storage.insert_engine_inbox_event(&event.into_store(), now).await? + }; + match inserted { + EngineInboxInsert::Inserted { .. } => { + self.inner.wake.worker_for(&aggregate); + Ok(SubmitOutcome::Inserted) + } + EngineInboxInsert::Duplicate { state, ack_again, rearmed } => { + let mut ack_sent = false; + if ack_again && let Some(ack) = ack { + ack_sent = self + .inner + .ack + .try_send(AckRequest { peer: ack.peer, delivery_id: ack.delivery_id }); + if !ack_sent { + self.inner.metrics.record_ack_dropped(); + } + } + if rearmed { + self.inner.wake.worker_for(&aggregate); + } + self.inner.metrics.record_event(&kind, "duplicate"); + Ok(SubmitOutcome::Duplicate { state, ack_sent, rearmed }) + } + } + } + + /// Record an event inside the caller's transaction. Nothing is woken and + /// no ACK is sent: the caller commits and then calls [`Self::wake_for`], + /// or relies on the worker poll. + pub async fn submit_in( + &self, + storage: &mut StorageProcessor<'_>, + event: NewEvent, + ) -> anyhow::Result { + let now = self.inner.clock.now(); + let kind = event.kind.clone(); + match storage.insert_engine_inbox_event(&event.into_store(), now).await? { + EngineInboxInsert::Inserted { .. } => Ok(SubmitOutcome::Inserted), + EngineInboxInsert::Duplicate { state, rearmed, .. } => { + self.inner.metrics.record_event(&kind, "duplicate"); + Ok(SubmitOutcome::Duplicate { state, ack_sent: false, rearmed }) + } + } + } + + /// Wake the worker owning `aggregate` (for events recorded with + /// [`Self::submit_in`] after their transaction committed). + pub fn wake_for(&self, aggregate: &AggregateRef) { + self.inner.wake.worker_for(aggregate); + } + + /// Apply an ACK received from `peer`. + pub async fn acknowledge( + &self, + peer: &str, + delivery_id: &str, + ) -> anyhow::Result { + let now = self.inner.clock.now(); + // Read-modify-write of the row's outstanding set: the immediate + // transaction serializes concurrent ACKs for the same row. + let mut tx = self.inner.db.start_immediate_transaction().await?; + let ack = tx.acknowledge_engine_outbox(peer, delivery_id, now).await?; + tx.commit().await?; + Ok(ack) + } + + /// Wake events waiting on any of `refs` (for changes made outside an apply). + pub async fn notify(&self, refs: &[String]) -> anyhow::Result { + let now = self.inner.clock.now(); + let woken = { + let mut storage = self.inner.db.acquire().await?; + storage.wake_engine_inbox_waiting(refs, now).await? + }; + if woken > 0 { + self.inner.wake.workers_all(); + } + Ok(woken) + } + + pub fn wake_all(&self) { + self.inner.wake.workers_all(); + self.inner.wake.publisher(); + self.inner.wake.executor(); + } + + /// Manual clear of one poisoned action, or of every action of the aggregate. + /// Returns how many actions were released. + pub async fn clear_poison( + &self, + aggregate: &AggregateRef, + action: Option<&str>, + ) -> anyhow::Result { + let mut storage = self.inner.db.acquire().await?; + let cleared = + storage.clear_engine_aggregate_poison(&aggregate.kind, &aggregate.id, action).await?; + drop(storage); + if cleared > 0 { + self.inner.wake.executor(); + self.inner.wake.worker_for(aggregate); + } + Ok(cleared) + } + + /// Actions of `aggregate` currently poisoned. + pub async fn poisons(&self, aggregate: &AggregateRef) -> anyhow::Result> { + let mut storage = self.inner.db.acquire().await?; + storage.list_engine_poisons(Some((&aggregate.kind, &aggregate.id)), 256).await + } + + pub async fn aggregate( + &self, + aggregate: &AggregateRef, + ) -> anyhow::Result> { + let mut storage = self.inner.db.acquire().await?; + storage.find_engine_aggregate(&aggregate.kind, &aggregate.id).await + } + + /// Row counts by table and state, without the parked and poisoned listings. + pub async fn state_counts(&self) -> anyhow::Result> { + let mut storage = self.inner.db.acquire().await?; + storage.engine_state_counts().await + } + + pub async fn snapshot(&self) -> anyhow::Result { + let mut storage = self.inner.db.acquire().await?; + Ok(EngineSnapshot { + counts: storage.engine_state_counts().await?, + waiting: storage.list_engine_inbox_waiting(256).await?, + poisoned: storage.list_engine_poisons(None, 256).await?, + }) + } +} + +impl Engine { + pub fn handle(&self) -> EngineHandle { + EngineHandle { inner: self.inner.clone() } + } + + /// Run until `cancel` fires. Returns an error if a core loop stops early. + pub async fn run(self, cancel: CancellationToken) -> anyhow::Result<()> { + let inner = self.inner; + startup_recovery(&inner).await?; + let mut tasks = Vec::new(); + let mut names = Vec::new(); + let worker_deps = WorkerDeps { + db: inner.db.clone(), + clock: inner.clock.clone(), + config: inner.config.clone(), + handlers: inner.handlers.clone(), + metrics: inner.metrics.clone(), + ack: inner.ack.clone(), + wake: inner.wake.clone(), + }; + for shard in 0..inner.wake.shard_count() { + tasks.push(spawn_on_thread( + format!("engine-worker-{shard}"), + inner.config.thread_stack_bytes, + run_worker( + shard, + worker_deps.clone(), + inner.wake.worker_notify(shard), + cancel.clone(), + ), + )); + names.push(format!("worker-{shard}")); + } + tasks.push(tokio::spawn(run_publisher( + PublisherDeps { + db: inner.db.clone(), + clock: inner.clock.clone(), + config: inner.config.clone(), + transport: inner.transport.clone(), + metrics: inner.metrics.clone(), + }, + inner.wake.publisher_notify(), + cancel.clone(), + ))); + names.push("publisher".to_string()); + tasks.push(tokio::spawn(run_executor( + ExecutorDeps { + db: inner.db.clone(), + clock: inner.clock.clone(), + config: inner.config.clone(), + registry: inner.runners.clone(), + metrics: inner.metrics.clone(), + wake: inner.wake.clone(), + }, + inner.wake.executor_notify(), + cancel.clone(), + ))); + names.push("executor".to_string()); + tasks.push(tokio::spawn(run_recheck(inner.clone(), cancel.clone()))); + names.push("recheck".to_string()); + tasks.push(tokio::spawn(run_housekeeping(inner.clone(), cancel.clone()))); + names.push("housekeeping".to_string()); + info!( + event = "engine_started", + workers = inner.wake.shard_count(), + handlers = inner.handlers.len(), + runners = inner.runners.len(), + "execution engine started" + ); + + let mut pending = tasks; + let stopped_early = tokio::select! { + biased; + _ = cancel.cancelled() => None, + (result, index, remaining) = futures::future::select_all(pending.drain(..)) => { + pending = remaining; + Some((names[index].clone(), result)) + } + }; + cancel.cancel(); + for task in pending { + let _ = task.await; + } + match stopped_early { + None => { + info!(event = "engine_stopped", "execution engine stopped"); + Ok(()) + } + Some((name, result)) => { + let detail = match result { + Ok(()) => "exited".to_string(), + Err(error) => format!("failed: {error}"), + }; + Err(anyhow::anyhow!("engine loop {name} stopped early: {detail}")) + } + } + } +} + +async fn startup_recovery(inner: &EngineInner) -> anyhow::Result<()> { + let now = inner.clock.now(); + let mut storage = inner.db.acquire().await?; + let rearmed = storage.rearm_engine_inbox_pending(now).await?; + let outbox_leases = storage.reset_engine_outbox_leases().await?; + let job_leases = storage.reset_engine_job_leases().await?; + info!( + event = "engine_recovery", + rearmed_events = rearmed, + outbox_leases, + job_leases, + "engine startup recovery completed" + ); + Ok(()) +} + +async fn run_recheck(inner: Arc, cancel: CancellationToken) { + let mut interval = tokio::time::interval(inner.config.recheck_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = interval.tick() => {} + } + let now = inner.clock.now(); + match inner.db.acquire().await { + Ok(mut storage) => { + match storage.rearm_engine_inbox_waiting(now).await { + Ok(rearmed) if rearmed > 0 => { + inner.wake.workers_all(); + info!(event = "engine_recheck", rearmed, "re-checking waiting events"); + } + Ok(_) => {} + Err(error) => { + warn!(event = "engine_recheck", error = %error, "re-check failed") + } + } + match storage.engine_state_counts().await { + Ok(counts) => { + for count in counts { + inner.metrics.set_rows(&count.table, &count.state, count.count); + } + } + Err(error) => { + warn!(event = "engine_recheck", error = %error, "row count refresh failed") + } + } + } + Err(error) => warn!(event = "engine_recheck", error = %error, "storage unavailable"), + } + } +} + +async fn run_housekeeping(inner: Arc, cancel: CancellationToken) { + let mut interval = tokio::time::interval(inner.config.housekeeping_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = interval.tick() => {} + } + let before = inner.clock.now() - inner.config.terminal_retention_secs; + match inner.db.acquire().await { + Ok(mut storage) => match storage.purge_engine_terminal_rows(before).await { + Ok(purged) if purged > 0 => { + info!(event = "engine_housekeeping", purged, "purged terminal engine rows") + } + Ok(_) => {} + Err(error) => warn!(event = "engine_housekeeping", error = %error, "purge failed"), + }, + Err(error) => { + warn!(event = "engine_housekeeping", error = %error, "storage unavailable") + } + } + } +} diff --git a/crates/engine/src/wake.rs b/crates/engine/src/wake.rs new file mode 100644 index 000000000..1926f99db --- /dev/null +++ b/crates/engine/src/wake.rs @@ -0,0 +1,95 @@ +//! In-memory wakeups. They only accelerate the workers, publisher and executor; +//! the database is the source of truth and every loop also polls. + +use crate::event::AggregateRef; +use std::sync::Arc; +use tokio::sync::Notify; + +#[derive(Clone, Debug)] +pub(crate) struct EngineWake { + workers: Arc>>, + publisher: Arc, + executor: Arc, +} + +impl EngineWake { + pub(crate) fn new(shard_count: usize) -> Self { + Self { + workers: Arc::new((0..shard_count.max(1)).map(|_| Arc::new(Notify::new())).collect()), + publisher: Arc::new(Notify::new()), + executor: Arc::new(Notify::new()), + } + } + + pub(crate) fn shard_count(&self) -> usize { + self.workers.len() + } + + pub(crate) fn worker_notify(&self, shard: usize) -> Arc { + self.workers[shard % self.workers.len()].clone() + } + + pub(crate) fn worker_for(&self, aggregate: &AggregateRef) { + self.workers[aggregate.shard(self.workers.len())].notify_one(); + } + + pub(crate) fn workers_all(&self) { + for worker in self.workers.iter() { + worker.notify_one(); + } + } + + pub(crate) fn publisher_notify(&self) -> Arc { + self.publisher.clone() + } + + pub(crate) fn publisher(&self) { + self.publisher.notify_one(); + } + + pub(crate) fn executor_notify(&self) -> Arc { + self.executor.clone() + } + + pub(crate) fn executor(&self) { + self.executor.notify_one(); + } +} + +/// Run `future` on a dedicated thread with `stack_bytes` of stack (0 runs it as +/// a tokio task). The returned handle resolves when the future finished. +pub(crate) fn spawn_on_thread( + name: String, + stack_bytes: usize, + future: F, +) -> tokio::task::JoinHandle<()> +where + F: std::future::Future + Send + 'static, +{ + if stack_bytes == 0 { + return tokio::spawn(future); + } + let runtime = tokio::runtime::Handle::current(); + let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>(); + std::thread::Builder::new() + .name(name) + .stack_size(stack_bytes) + .spawn(move || { + runtime.block_on(future); + let _ = done_tx.send(()); + }) + .expect("spawn engine thread"); + tokio::spawn(async move { + let _ = done_rx.await; + }) +} + +pub(crate) fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_owned() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "non-string panic payload".to_owned() + } +} diff --git a/crates/engine/src/worker.rs b/crates/engine/src/worker.rs new file mode 100644 index 000000000..b63d79cda --- /dev/null +++ b/crates/engine/src/worker.rs @@ -0,0 +1,479 @@ +//! Aggregate workers: one per shard, single writer for every aggregate in the +//! shard. An apply runs inside one transaction; the outcome decides whether +//! the event is applied, parked, rejected, retried or poisoned. + +use crate::ack::{AckRequest, AckSender}; +use crate::aggregate::{AggregateHandler, ApplyContext, ApplyError, ApplyOutcome, WaitingFor}; +use crate::clock::Clock; +use crate::config::EngineConfig; +use crate::effects::{Effects, JobCancel}; +use crate::event::AggregateRef; +use crate::metrics::EngineMetrics; +use crate::wake::{EngineWake, panic_message}; +use futures::FutureExt; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Instant; +use store::localdb::{LocalDB, StorageProcessor}; +use store::{ + EngineAggregate, EngineAggregateUpdate, EngineInboxEvent, EngineJobInsert, EngineJobState, + NewEngineJob, NewEngineOutboxMessage, +}; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +#[derive(Clone)] +pub(crate) struct WorkerDeps { + pub db: LocalDB, + pub clock: Arc, + pub config: Arc, + pub handlers: Arc>>, + pub metrics: EngineMetrics, + pub ack: AckSender, + pub wake: EngineWake, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ProcessOutcome { + Applied, + Waiting, + Rejected, + Transient, + TimedOut, + Poisoned, + NoHandler, +} + +impl ProcessOutcome { + const fn label(self) -> &'static str { + match self { + ProcessOutcome::Applied => "applied", + ProcessOutcome::Waiting => "waiting", + ProcessOutcome::Rejected => "rejected", + ProcessOutcome::Transient => "transient", + ProcessOutcome::TimedOut => "timed_out", + ProcessOutcome::Poisoned => "poisoned", + ProcessOutcome::NoHandler => "no_handler", + } + } +} + +pub(crate) async fn run_worker( + shard: usize, + deps: WorkerDeps, + notify: Arc, + cancel: CancellationToken, +) { + let shard_count = deps.wake.shard_count(); + loop { + if cancel.is_cancelled() { + return; + } + match drain_shard(shard, shard_count, &deps, &cancel).await { + Ok(_) => {} + Err(error) => { + warn!(event = "engine_worker", shard, error = %error, "worker pass failed; backing off"); + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = tokio::time::sleep(deps.config.worker_poll_interval) => {} + } + } + } + tokio::select! { + biased; + _ = cancel.cancelled() => return, + _ = notify.notified() => {} + _ = tokio::time::sleep(deps.config.worker_poll_interval) => {} + } + } +} + +/// Apply runnable events of `shard` until none is left. Returns how many were +/// processed. +pub(crate) async fn drain_shard( + shard: usize, + shard_count: usize, + deps: &WorkerDeps, + cancel: &CancellationToken, +) -> anyhow::Result { + let mut processed = 0; + loop { + if cancel.is_cancelled() { + return Ok(processed); + } + let now = deps.clock.now(); + let next = { + let mut storage = deps.db.acquire().await?; + storage.next_runnable_engine_inbox_event(shard_count as i64, shard as i64, now).await? + }; + let Some(event) = next else { return Ok(processed) }; + process_event(deps, event).await?; + processed += 1; + } +} + +struct PostCommit { + acks: Vec, + woke: bool, + outbox: bool, + jobs: bool, +} + +impl PostCommit { + fn dispatch(self, deps: &WorkerDeps) { + for ack in self.acks { + if !deps + .ack + .try_send(AckRequest { peer: ack.peer.clone(), delivery_id: ack.delivery_id }) + { + deps.metrics.record_ack_dropped(); + warn!(event = "engine_ack", outcome = "dropped", peer = %ack.peer, "ACK channel full; sender will resend"); + } + } + if self.woke { + deps.wake.workers_all(); + } + if self.outbox { + deps.wake.publisher(); + } + if self.jobs { + deps.wake.executor(); + } + } +} + +pub(crate) async fn process_event( + deps: &WorkerDeps, + event: EngineInboxEvent, +) -> anyhow::Result { + let aggregate = AggregateRef::from_event(&event); + let kind = event.kind.clone(); + let Some(handler) = deps.handlers.get(&aggregate.kind).cloned() else { + let now = deps.clock.now(); + let mut storage = deps.db.acquire().await?; + storage + .mark_engine_inbox_failed( + &event.event_id, + &format!("no handler for aggregate kind {}", aggregate.kind), + now, + ) + .await?; + warn!( + event = "engine_apply", + outcome = "no_handler", + event_id = %event.event_id, + aggregate = %aggregate, + kind = %kind, + "no handler registered for this aggregate kind" + ); + deps.metrics.record_event(&kind, ProcessOutcome::NoHandler.label()); + return Ok(ProcessOutcome::NoHandler); + }; + + let started = Instant::now(); + let now = deps.clock.now(); + // BEGIN IMMEDIATE: SQLite has one writer, so the apply takes the write lock + // up front instead of failing with a stale snapshot at its first write. + let mut tx = deps.db.start_immediate_transaction().await?; + let current = tx.find_engine_aggregate(&aggregate.kind, &aggregate.id).await?; + let (result, effects) = { + let mut ctx = ApplyContext { + now, + aggregate: aggregate.clone(), + current: current.clone(), + storage: &mut tx, + effects: Effects::default(), + }; + let result = std::panic::AssertUnwindSafe(tokio::time::timeout( + deps.config.apply_timeout, + handler.apply(&mut ctx, &event), + )) + .catch_unwind() + .await; + (result, std::mem::take(&mut ctx.effects)) + }; + deps.metrics.record_apply_duration(&kind, started.elapsed()); + + let outcome = match result { + Ok(Ok(Ok(ApplyOutcome::Applied))) => { + let post = + commit_apply(deps, tx, &event, &aggregate, current.as_ref(), effects, None, now) + .await?; + post.dispatch(deps); + debug!(event = "engine_apply", outcome = "applied", event_id = %event.event_id, aggregate = %aggregate, kind = %kind, "event applied"); + ProcessOutcome::Applied + } + Ok(Ok(Ok(ApplyOutcome::Waiting(waiting)))) => { + let detail = waiting.detail.clone(); + let wake_ref = waiting.wake_ref.clone(); + let post = commit_apply( + deps, + tx, + &event, + &aggregate, + current.as_ref(), + effects, + Some(waiting), + now, + ) + .await?; + post.dispatch(deps); + info!(event = "engine_apply", outcome = "waiting", event_id = %event.event_id, aggregate = %aggregate, kind = %kind, wake_ref = %wake_ref, detail = %detail, "event parked on a dependency"); + ProcessOutcome::Waiting + } + Ok(Ok(Ok(ApplyOutcome::Rejected(reason)))) => { + drop(tx); + let now = deps.clock.now(); + let mut storage = deps.db.acquire().await?; + storage.mark_engine_inbox_failed(&event.event_id, &reason, now).await?; + storage.release_engine_poison(&aggregate.kind, &aggregate.id, &kind).await?; + drop(storage); + PostCommit { + acks: event.ack_target().into_iter().collect(), + woke: false, + outbox: false, + jobs: false, + } + .dispatch(deps); + warn!(event = "engine_apply", outcome = "rejected", event_id = %event.event_id, aggregate = %aggregate, kind = %kind, reason = %reason, "event rejected"); + ProcessOutcome::Rejected + } + Ok(Ok(Err(ApplyError::Transient(error)))) => { + drop(tx); + if is_busy_error(&error) { + // Lock contention is not a business failure: try again shortly. + schedule_retry_at(deps, &event, &error.to_string(), deps.clock.now() + 1).await?; + } else { + schedule_retry(deps, &event, &error.to_string()).await?; + } + warn!(event = "engine_apply", outcome = "transient", event_id = %event.event_id, aggregate = %aggregate, kind = %kind, attempt = event.attempts + 1, error = %error, "event apply failed transiently; will retry"); + ProcessOutcome::Transient + } + Ok(Ok(Err(ApplyError::Invariant(error)))) => { + drop(tx); + poison(deps, &event, &aggregate, &format!("invariant violated: {error}")).await?; + ProcessOutcome::Poisoned + } + Ok(Err(_elapsed)) => { + drop(tx); + let message = format!("apply timed out after {:?}", deps.config.apply_timeout); + schedule_retry(deps, &event, &message).await?; + warn!(event = "engine_apply", outcome = "timed_out", event_id = %event.event_id, aggregate = %aggregate, kind = %kind, "event apply timed out; will retry"); + ProcessOutcome::TimedOut + } + Err(payload) => { + drop(tx); + poison( + deps, + &event, + &aggregate, + &format!("panic: {}", panic_message(payload.as_ref())), + ) + .await?; + ProcessOutcome::Poisoned + } + }; + deps.metrics.record_event(&kind, outcome.label()); + Ok(outcome) +} + +#[allow(clippy::too_many_arguments)] +async fn commit_apply( + deps: &WorkerDeps, + mut tx: StorageProcessor<'static>, + event: &EngineInboxEvent, + aggregate: &AggregateRef, + current: Option<&EngineAggregate>, + effects: Effects, + waiting: Option, + now: i64, +) -> anyhow::Result { + for message in &effects.outbox { + tx.insert_engine_outbox_message( + &NewEngineOutboxMessage { + outbox_id: message.outbox_id.clone(), + aggregate_kind: aggregate.kind.clone(), + aggregate_id: aggregate.id.clone(), + phase_ref: message.phase_ref.clone(), + kind: message.kind.clone(), + topic: message.topic.clone(), + payload: message.payload.clone(), + ack_set: message.ack_set.clone(), + first_at: now, + }, + now, + ) + .await?; + } + for (target, reason) in &effects.outbox_closes { + tx.close_engine_outbox(target, reason, now).await?; + } + for job in &effects.jobs { + let inserted = tx + .insert_engine_job( + &NewEngineJob { + job_id: job.job_id.clone(), + aggregate_kind: aggregate.kind.clone(), + aggregate_id: aggregate.id.clone(), + phase: job.phase.clone(), + action: job.action.clone(), + pool: job.pool.as_str().to_string(), + params: job.params.clone(), + deadline_at: job.deadline_at, + first_at: job.run_after.unwrap_or(now).max(now), + }, + now, + ) + .await?; + match inserted { + EngineJobInsert::Inserted => {} + EngineJobInsert::Existing(EngineJobState::Pending | EngineJobState::Running) => { + debug!(event = "engine_jobs", outcome = "already_open", job_id = %job.job_id, action = %job.action, "job is already queued or running"); + } + EngineJobInsert::Existing( + state @ (EngineJobState::Done | EngineJobState::Cancelled), + ) => { + // Same id means same params: the job ran (or was cancelled) + // before and the engine will not run it again. Aggregates that + // derive their jobs from chain state ask for the same job on + // every scan, so a finished one is routine. + debug!(event = "engine_jobs", outcome = "reenqueue_ignored", job_id = %job.job_id, action = %job.action, event_id = %event.event_id, state = %state, "job already ran; not run again"); + } + EngineJobInsert::Existing(state) => { + // A job that failed or panicked is asked for again: it stays + // where it is, which is worth a look. + warn!(event = "engine_jobs", outcome = "reenqueue_ignored", job_id = %job.job_id, action = %job.action, event_id = %event.event_id, state = %state, "job re-enqueued after a terminal failure; not run again"); + } + } + } + for cancel in &effects.job_cancels { + match cancel { + JobCancel::Job(job_id) => { + tx.cancel_engine_job(job_id, now).await?; + } + JobCancel::Phase(phase) => { + tx.cancel_engine_jobs_for_phase(&aggregate.kind, &aggregate.id, phase, now).await?; + } + JobCancel::Aggregate { aggregate_kind, aggregate_id } => { + tx.cancel_engine_jobs_for_aggregate(aggregate_kind, aggregate_id, now).await?; + } + JobCancel::AggregatePhase { aggregate_kind, aggregate_id, phase } => { + tx.cancel_engine_jobs_for_phase(aggregate_kind, aggregate_id, phase, now).await?; + } + } + } + // The handler ran this kind to a decision: a poison it carried is over. + tx.release_engine_poison(&aggregate.kind, &aggregate.id, &event.kind).await?; + let update = EngineAggregateUpdate { + status: effects.status.clone(), + phase: effects.phase.clone(), + state: effects.state.clone(), + last_event_id: Some(event.event_id.clone()), + }; + let updated = tx.upsert_engine_aggregate(&aggregate.kind, &aggregate.id, &update, now).await?; + let changed = + current.map(|c| c.status != updated.status || c.phase != updated.phase).unwrap_or(true); + let mut wake_refs = effects.wake_refs.clone(); + if changed { + wake_refs.push(aggregate.wake_ref()); + } + let woke = if wake_refs.is_empty() { + 0 + } else { + tx.wake_engine_inbox_waiting(&wake_refs, now).await? + }; + let marked = match &waiting { + None => tx.mark_engine_inbox_applied(&event.event_id, now).await?, + Some(w) => { + tx.mark_engine_inbox_waiting( + &event.event_id, + &w.wake_ref, + &w.detail, + now + deps.config.waiting_recheck_secs(), + now, + ) + .await? + } + }; + if !marked { + warn!(event = "engine_apply", outcome = "mark_skipped", event_id = %event.event_id, "inbox row was no longer pending when the apply committed"); + } + tx.commit().await?; + let mut acks = effects.acks.clone(); + acks.extend(event.ack_target()); + Ok(PostCommit { + acks, + woke: woke > 0, + outbox: !effects.outbox.is_empty(), + jobs: !effects.jobs.is_empty(), + }) +} + +async fn schedule_retry( + deps: &WorkerDeps, + event: &EngineInboxEvent, + error: &str, +) -> anyhow::Result<()> { + let now = deps.clock.now(); + let next_at = now + deps.config.inbox_retry_backoff.delay(event.attempts + 1); + schedule_retry_at(deps, event, error, next_at).await +} + +async fn schedule_retry_at( + deps: &WorkerDeps, + event: &EngineInboxEvent, + error: &str, + next_at: i64, +) -> anyhow::Result<()> { + let now = deps.clock.now(); + let mut storage = deps.db.acquire().await?; + storage.mark_engine_inbox_retry(&event.event_id, error, next_at, now).await?; + Ok(()) +} + +/// SQLite lock contention surfaces as "database is locked"; it means another +/// writer held the lock past the busy timeout, not that the apply is wrong. +pub(crate) fn is_busy_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + let message = cause.to_string().to_ascii_lowercase(); + message.contains("database is locked") + || message.contains("database is busy") + || message.contains("sqlite_busy") + }) +} + +async fn poison( + deps: &WorkerDeps, + event: &EngineInboxEvent, + aggregate: &AggregateRef, + reason: &str, +) -> anyhow::Result<()> { + let now = deps.clock.now(); + let mut tx = deps.db.start_immediate_transaction().await?; + tx.mark_engine_inbox_poisoned(&event.event_id, reason, now).await?; + let poison = tx + .mark_engine_aggregate_poisoned( + &aggregate.kind, + &aggregate.id, + &event.kind, + reason, + now, + &|count| deps.config.poison_backoff.delay(count), + ) + .await?; + tx.commit().await?; + deps.metrics.record_poisoned(); + error!( + event = "engine_poisoned", + event_id = %event.event_id, + aggregate = %aggregate, + kind = %event.kind, + poison_count = poison.poison_count, + blocked_until = poison.blocked_until, + reason = %reason, + "event poisoned; later events of this kind wait for the backoff, a clean apply or a manual clear" + ); + Ok(()) +} diff --git a/crates/engine/tests/engine.rs b/crates/engine/tests/engine.rs new file mode 100644 index 000000000..6f2dd9ef8 --- /dev/null +++ b/crates/engine/tests/engine.rs @@ -0,0 +1,1126 @@ +//! Engine integration tests using file-backed SQLite: transactional apply, +//! deduplication, ACKs, dependency waits, delivery retries and job recovery. + +use async_trait::async_trait; +use engine::{ + AckReceiver, AckRequest, AggregateHandler, AggregateRef, ApplyContext, ApplyError, + ApplyOutcome, Backoff, Clock, Engine, EngineBuilder, EngineConfig, EngineHandle, + EngineInboxEvent, EngineInboxState, EngineOutboxAck, EngineOutboxCloseTarget, EventSource, + JOB_RESULT_KIND, JobContext, JobOutcome, JobPool, JobResultEvent, JobResultOutcome, JobRunner, + JobSpec, ManualClock, NewEvent, OutboxMessage, SubmitOutcome, Transport, WaitingFor, +}; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use store::localdb::LocalDB; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; + +const KIND: &str = "test"; +const WAIT: Duration = Duration::from_secs(10); + +// ------------------------------------------------------------------ fixtures + +#[derive(Default)] +struct Probe { + concurrent: AtomicUsize, + max_concurrent: AtomicUsize, +} + +struct TestHandler { + probe: Arc, +} + +fn payload_str(event: &EngineInboxEvent) -> String { + String::from_utf8_lossy(event.payload.as_deref().unwrap_or_default()).to_string() +} + +#[async_trait] +impl AggregateHandler for TestHandler { + async fn apply( + &self, + ctx: &mut ApplyContext<'_>, + event: &EngineInboxEvent, + ) -> Result { + let payload = payload_str(event); + match event.kind.as_str() { + "set_status" => { + ctx.effects.set_status(payload); + Ok(ApplyOutcome::Applied) + } + "set_phase" => { + ctx.effects.set_phase(payload); + Ok(ApplyOutcome::Applied) + } + "wait_on" => { + // payload: ":|" + let (dep, required) = payload.split_once('|').expect("wait_on payload"); + let (dep_kind, dep_id) = dep.split_once(':').expect("dep ref"); + let dep_row = ctx.storage.find_engine_aggregate(dep_kind, dep_id).await?; + if dep_row.is_some_and(|row| row.status == required) { + ctx.effects.set_status("unblocked"); + Ok(ApplyOutcome::Applied) + } else { + Ok(ApplyOutcome::Waiting(WaitingFor::on_ref(dep, format!("need {required}")))) + } + } + "transient" => { + let failures: i64 = payload.parse().expect("transient count"); + if event.attempts < failures { + Err(ApplyError::transient(anyhow::anyhow!("flaky attempt {}", event.attempts))) + } else { + ctx.effects.set_status("recovered"); + Ok(ApplyOutcome::Applied) + } + } + "effects_then_fail" => { + ctx.effects.set_status("should_not_persist_on_failure"); + ctx.effects.send(OutboxMessage { + outbox_id: format!("{}:out", event.event_id), + phase_ref: "p".into(), + kind: "Msg".into(), + topic: "t".into(), + payload: b"x".to_vec(), + ack_set: None, + }); + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + "p", + "echo", + JobPool::Cpu, + b"x".to_vec(), + )); + if event.attempts < 1 { + Err(ApplyError::transient(anyhow::anyhow!("fail once"))) + } else { + Ok(ApplyOutcome::Applied) + } + } + "reject" => Ok(ApplyOutcome::Rejected("nope".into())), + "panic" => panic!("boom in apply"), + "panic_unless_status" => { + if ctx.current.as_ref().is_some_and(|row| row.status == payload) { + ctx.effects.set_status("survived"); + Ok(ApplyOutcome::Applied) + } else { + panic!("boom until {payload}") + } + } + "invariant" => Err(ApplyError::invariant(anyhow::anyhow!("broken"))), + "slow" => { + let millis: u64 = payload.parse().expect("slow millis"); + tokio::time::sleep(Duration::from_millis(millis)).await; + ctx.effects.set_status("slow_done"); + Ok(ApplyOutcome::Applied) + } + "send" => { + // payload: "||" (ack part optional) + let mut parts = payload.splitn(3, '|'); + let outbox_id = parts.next().unwrap().to_string(); + let phase_ref = parts.next().unwrap().to_string(); + let ack_set = parts.next().filter(|s| !s.is_empty()).map(|acks| { + acks.split(',') + .map(|pair| { + let (peer, delivery) = pair.split_once(':').unwrap(); + (peer.to_string(), delivery.to_string()) + }) + .collect::>() + }); + ctx.effects.send(OutboxMessage { + outbox_id, + phase_ref, + kind: "Msg".into(), + topic: "topic".into(), + payload: payload.clone().into_bytes(), + ack_set, + }); + Ok(ApplyOutcome::Applied) + } + "close_phase" => { + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Phase { + aggregate_kind: ctx.aggregate.kind.clone(), + aggregate_id: ctx.aggregate.id.clone(), + phase_ref: payload, + }, + "evidence", + ); + Ok(ApplyOutcome::Applied) + } + "job" => { + // payload: "||" + let mut parts = payload.splitn(3, '|'); + let action = parts.next().unwrap(); + let phase = parts.next().unwrap(); + let params = parts.next().unwrap_or("").as_bytes().to_vec(); + ctx.effects.set_phase(phase); + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + phase, + action, + JobPool::Cpu, + params, + )); + Ok(ApplyOutcome::Applied) + } + k if k == JOB_RESULT_KIND => { + let result = JobResultEvent::decode(event.payload.as_deref().unwrap_or_default()) + .map_err(ApplyError::invariant)?; + if result.stale { + ctx.effects.set_state(Some("stale_seen".into())); + return Ok(ApplyOutcome::Applied); + } + if result.phase != ctx.phase() { + return Ok(ApplyOutcome::Rejected(format!( + "result for phase {} but aggregate is in {}", + result.phase, + ctx.phase() + ))); + } + let status = match result.outcome { + JobResultOutcome::Done(bytes) => { + format!("job_done:{}", String::from_utf8_lossy(&bytes)) + } + JobResultOutcome::Failed(error) => format!("job_failed:{error}"), + JobResultOutcome::Cancelled => "job_cancelled".to_string(), + JobResultOutcome::Poisoned(error) => format!("job_poisoned:{error}"), + }; + ctx.effects.set_status(status); + Ok(ApplyOutcome::Applied) + } + "serial_probe" => { + let now = self.probe.concurrent.fetch_add(1, Ordering::SeqCst) + 1; + self.probe.max_concurrent.fetch_max(now, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(20)).await; + self.probe.concurrent.fetch_sub(1, Ordering::SeqCst); + Ok(ApplyOutcome::Applied) + } + other => Ok(ApplyOutcome::Rejected(format!("unknown kind {other}"))), + } + } +} + +struct EchoRunner; + +#[async_trait] +impl JobRunner for EchoRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + JobOutcome::Done(ctx.job.params.unwrap_or_default()) + } +} + +struct FlakyRunner; + +#[async_trait] +impl JobRunner for FlakyRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + if ctx.job.attempts < 3 { + JobOutcome::Transient { + error: format!("attempt {}", ctx.job.attempts), + retry_after_secs: None, + } + } else { + JobOutcome::Done(b"flaky-ok".to_vec()) + } + } +} + +struct BoomRunner; + +#[async_trait] +impl JobRunner for BoomRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + if ctx.job.params.as_deref() == Some(b"ok") { + return JobOutcome::Done(b"boom-ok".to_vec()); + } + panic!("boom in job") + } +} + +struct FailRunner; + +#[async_trait] +impl JobRunner for FailRunner { + async fn run(&self, _ctx: JobContext) -> JobOutcome { + JobOutcome::Failed("permanent".into()) + } +} + +/// Waits for the gate before answering with its generation. +struct GatedRunner { + gate: watch::Receiver, +} + +#[async_trait] +impl JobRunner for GatedRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let mut gate = self.gate.clone(); + while !*gate.borrow() { + if gate.changed().await.is_err() { + return JobOutcome::Failed("gate dropped".into()); + } + } + JobOutcome::Done(format!("gen{}", ctx.job.generation).into_bytes()) + } +} + +#[derive(Default)] +struct RecordingTransport { + published: Mutex)>>, + fail: AtomicBool, +} + +impl RecordingTransport { + fn count(&self) -> usize { + self.published.lock().unwrap().len() + } + + fn count_for(&self, needle: &str) -> usize { + self.published + .lock() + .unwrap() + .iter() + .filter(|(_, payload)| String::from_utf8_lossy(payload).starts_with(needle)) + .count() + } +} + +#[async_trait] +impl Transport for RecordingTransport { + async fn publish(&self, topic: &str, payload: &[u8]) -> anyhow::Result<()> { + if self.fail.load(Ordering::SeqCst) { + anyhow::bail!("transport down"); + } + self.published.lock().unwrap().push((topic.to_string(), payload.to_vec())); + Ok(()) + } +} + +struct TestEngine { + db: LocalDB, + _dir: tempfile::TempDir, + clock: ManualClock, + transport: Arc, + probe: Arc, + handle: EngineHandle, + acks: AckReceiver, + cancel: CancellationToken, + run: tokio::task::JoinHandle>, + gate: watch::Sender, +} + +fn fast_config() -> EngineConfig { + EngineConfig { + worker_count: 2, + thread_stack_bytes: 0, + apply_timeout: Duration::from_millis(300), + worker_poll_interval: Duration::from_millis(30), + recheck_interval: Duration::from_millis(250), + inbox_retry_backoff: Backoff::Steps(vec![1]), + outbox_backoff: Backoff::Doubling { base: 30, cap: 600 }, + outbox_batch: 16, + outbox_lease_secs: 60, + outbox_poll_interval: Duration::from_millis(30), + job_lease_secs: 1, + job_lease_renew_interval: Duration::from_secs(30), + job_retry_backoff: Backoff::Steps(vec![1]), + job_poll_interval: Duration::from_millis(30), + poison_backoff: Backoff::Steps(vec![0]), + cpu_pool_slots: 2, + pegout_pool_slots: 1, + network_pool_slots: 2, + ack_channel_capacity: 8, + terminal_retention_secs: 15 * 24 * 3600, + housekeeping_interval: Duration::from_secs(3600), + } +} + +async fn test_db(busy_timeout: Duration) -> (LocalDB, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("engine.db"); + let db = + LocalDB::new_with_busy_timeout(&format!("sqlite:{}", path.display()), false, busy_timeout) + .await; + db.migrate().await; + (db, dir) +} + +async fn start_with(config: EngineConfig, prepare: impl AsyncFnOnce(&LocalDB)) -> TestEngine { + start_with_busy_timeout(config, store::DEFAULT_BUSY_TIMEOUT, prepare).await +} + +async fn start_with_busy_timeout( + config: EngineConfig, + busy_timeout: Duration, + prepare: impl AsyncFnOnce(&LocalDB), +) -> TestEngine { + let (db, dir) = test_db(busy_timeout).await; + prepare(&db).await; + let clock = ManualClock::new(1_000_000); + let transport = Arc::new(RecordingTransport::default()); + let probe = Arc::new(Probe::default()); + let (gate, gate_rx) = watch::channel(false); + let (engine, acks): (Engine, AckReceiver) = EngineBuilder::new(db.clone(), config) + .clock(clock.clone()) + .transport(transport.clone()) + .handler(KIND, Arc::new(TestHandler { probe: probe.clone() })) + .job_runner(KIND, "echo", Arc::new(EchoRunner)) + .job_runner(KIND, "flaky", Arc::new(FlakyRunner)) + .job_runner(KIND, "boom", Arc::new(BoomRunner)) + .job_runner(KIND, "fail", Arc::new(FailRunner)) + .job_runner(KIND, "gated", Arc::new(GatedRunner { gate: gate_rx })) + .build(); + let handle = engine.handle(); + let cancel = CancellationToken::new(); + let run = tokio::spawn(engine.run(cancel.clone())); + TestEngine { db, _dir: dir, clock, transport, probe, handle, acks, cancel, run, gate } +} + +async fn start() -> TestEngine { + start_with(fast_config(), async |_| {}).await +} + +impl TestEngine { + async fn stop(self) { + self.cancel.cancel(); + self.run.await.unwrap().unwrap(); + } + + fn agg(&self, id: &str) -> AggregateRef { + AggregateRef::new(KIND, id) + } + + async fn submit(&self, id: &str, event_id: &str, kind: &str, payload: &str) -> SubmitOutcome { + self.handle + .submit(NewEvent::new( + event_id, + self.agg(id), + kind, + EventSource::Local, + payload.as_bytes().to_vec(), + )) + .await + .unwrap() + } + + async fn submit_acked( + &self, + id: &str, + event_id: &str, + kind: &str, + payload: &str, + peer: &str, + delivery: &str, + ) -> SubmitOutcome { + self.handle + .submit( + NewEvent::new( + event_id, + self.agg(id), + kind, + EventSource::Network, + payload.as_bytes().to_vec(), + ) + .with_author(peer) + .with_ack(peer, delivery), + ) + .await + .unwrap() + } + + async fn inbox(&self, event_id: &str) -> Option { + let mut s = self.db.acquire().await.unwrap(); + s.find_engine_inbox_event(event_id).await.unwrap() + } + + async fn wait_inbox_state(&self, event_id: &str, state: EngineInboxState) -> EngineInboxEvent { + wait_for(WAIT, || async { + self.inbox(event_id).await.filter(|row| row.state == state.to_string()) + }) + .await + } + + async fn wait_inbox_attempts(&self, event_id: &str, attempts: i64) -> EngineInboxEvent { + wait_for(WAIT, || async { + self.inbox(event_id).await.filter(|row| row.attempts >= attempts) + }) + .await + } + + async fn wait_status(&self, id: &str, status: &str) -> store::EngineAggregate { + let agg = self.agg(id); + wait_for(WAIT, || async { + self.handle.aggregate(&agg).await.unwrap().filter(|row| row.status == status) + }) + .await + } + + async fn wait_job_state(&self, job_id: &str, state: &str) -> store::EngineJob { + wait_for(WAIT, || async { + let mut s = self.db.acquire().await.unwrap(); + s.find_engine_job(job_id).await.unwrap().filter(|job| job.state == state) + }) + .await + } + + async fn wait_ack(&mut self) -> AckRequest { + tokio::time::timeout(WAIT, self.acks.recv()) + .await + .expect("ack in time") + .expect("ack channel open") + } + + async fn wait_publishes(&self, needle: &str, count: usize) { + wait_for(WAIT, || async { (self.transport.count_for(needle) >= count).then_some(()) }) + .await; + } +} + +async fn wait_for(timeout: Duration, mut probe: F) -> T +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let started = tokio::time::Instant::now(); + loop { + if let Some(value) = probe().await { + return value; + } + assert!(started.elapsed() < timeout, "condition not met within {timeout:?}"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn settle() { + tokio::time::sleep(Duration::from_millis(250)).await; +} + +// --------------------------------------------------------------------- tests + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn applies_events_once_and_acks_after_commit() { + let mut t = start().await; + assert_eq!( + t.submit_acked("a", "e1", "set_status", "ready", "peer-1", "d-1").await, + SubmitOutcome::Inserted + ); + t.wait_status("a", "ready").await; + assert_eq!(t.wait_ack().await, AckRequest { peer: "peer-1".into(), delivery_id: "d-1".into() }); + let row = t.wait_inbox_state("e1", EngineInboxState::Applied).await; + assert!(row.payload.is_none(), "payload cleared once applied"); + assert_eq!(row.attempts, 0); + // Duplicate of an applied heavy message: ignored, ACKed again. + assert_eq!( + t.submit_acked("a", "e1", "set_status", "ready", "peer-1", "d-1").await, + SubmitOutcome::Duplicate { + state: EngineInboxState::Applied, + ack_sent: true, + rearmed: false + } + ); + assert_eq!(t.wait_ack().await.delivery_id, "d-1"); + let agg = t.agg("a"); + let bookkeeping = t.handle.aggregate(&agg).await.unwrap().unwrap(); + assert_eq!(bookkeeping.last_event_id.as_deref(), Some("e1")); + assert_eq!(bookkeeping.version, 1); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn failed_apply_rolls_back_effects_and_retries_as_one_transaction() { + let t = start().await; + t.submit("a", "e1", "effects_then_fail", "").await; + let row = t.wait_inbox_attempts("e1", 1).await; + assert_eq!(row.state, "Pending"); + assert_eq!(row.next_at, t.clock.now() + 1); + assert!(row.last_error.as_deref().unwrap_or("").contains("fail once")); + // Nothing from the failed attempt survived. + { + let mut s = t.db.acquire().await.unwrap(); + assert!(s.find_engine_outbox_message("e1:out").await.unwrap().is_none()); + assert!(s.list_engine_jobs_for_aggregate(KIND, "a").await.unwrap().is_empty()); + assert!(s.find_engine_aggregate(KIND, "a").await.unwrap().is_none()); + } + assert_eq!(t.transport.count(), 0); + // The retry commits everything together. + t.clock.advance(1); + t.wait_inbox_state("e1", EngineInboxState::Applied).await; + t.wait_status("a", "should_not_persist_on_failure").await; + { + let mut s = t.db.acquire().await.unwrap(); + assert!(s.find_engine_outbox_message("e1:out").await.unwrap().is_some()); + assert_eq!(s.list_engine_jobs_for_aggregate(KIND, "a").await.unwrap().len(), 1); + } + t.wait_publishes("x", 1).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn transient_failures_back_off_and_never_poison() { + let t = start().await; + t.submit("a", "e1", "transient", "3").await; + for attempt in 1..=3 { + let row = t.wait_inbox_attempts("e1", attempt).await; + assert_eq!(row.state, "Pending", "attempt {attempt}"); + assert!(t.handle.poisons(&t.agg("a")).await.unwrap().is_empty()); + t.clock.advance(1); + } + t.wait_status("a", "recovered").await; + let row = t.wait_inbox_state("e1", EngineInboxState::Applied).await; + assert_eq!(row.attempts, 3); + assert!(t.handle.snapshot().await.unwrap().poisoned.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn apply_timeout_is_a_transient_failure() { + let t = start().await; + t.submit("a", "e1", "slow", "2000").await; + let row = t.wait_inbox_attempts("e1", 1).await; + assert_eq!(row.state, "Pending"); + assert!(row.last_error.as_deref().unwrap_or("").contains("timed out")); + assert!(t.handle.snapshot().await.unwrap().poisoned.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn repeated_poisons_of_a_kind_wait_out_their_backoff() { + let config = EngineConfig { poison_backoff: Backoff::Steps(vec![0, 3_600]), ..fast_config() }; + let t = start_with(config, async |_| {}).await; + // The first poison holds nothing back; other kinds are never held back. + t.submit("a", "e1", "panic", "").await; + t.wait_inbox_state("e1", EngineInboxState::Poisoned).await; + let poisons = t.handle.poisons(&t.agg("a")).await.unwrap(); + assert_eq!(poisons.len(), 1); + assert_eq!((poisons[0].action.as_str(), poisons[0].poison_count), ("panic", 1)); + assert_eq!(poisons[0].blocked_until, poisons[0].poisoned_at); + t.submit("a", "e2", "set_status", "alive").await; + t.wait_status("a", "alive").await; + assert_eq!(t.handle.poisons(&t.agg("a")).await.unwrap().len(), 1, "other kinds do not lift it"); + // The second poison of the kind holds it back for an hour. + t.submit("a", "e3", "panic", "").await; + t.wait_inbox_state("e3", EngineInboxState::Poisoned).await; + let poisons = t.handle.poisons(&t.agg("a")).await.unwrap(); + assert_eq!(poisons[0].poison_count, 2); + assert_eq!(poisons[0].blocked_until, poisons[0].poisoned_at + 3_600); + t.submit("a", "e4", "panic", "").await; + t.submit("a", "e5", "set_status", "still").await; + t.wait_status("a", "still").await; + settle().await; + let held = t.inbox("e4").await.unwrap(); + assert_eq!((held.state.as_str(), held.attempts), ("Pending", 0), "the kind is held back"); + // Once the backoff passed the kind runs again (and poisons again). + t.clock.advance(3_600); + t.wait_inbox_state("e4", EngineInboxState::Poisoned).await; + assert_eq!(t.handle.poisons(&t.agg("a")).await.unwrap()[0].poison_count, 3); + // A manual clear releases the kind and starts the count over. + assert_eq!(t.handle.clear_poison(&t.agg("a"), None).await.unwrap(), 1); + t.submit("a", "e6", "panic", "").await; + t.wait_inbox_state("e6", EngineInboxState::Poisoned).await; + assert_eq!(t.handle.poisons(&t.agg("a")).await.unwrap()[0].poison_count, 1); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_clean_apply_of_the_kind_releases_its_poison() { + let config = EngineConfig { poison_backoff: Backoff::Steps(vec![0]), ..fast_config() }; + let t = start_with(config, async |_| {}).await; + t.submit("a", "e1", "panic_unless_status", "ready").await; + t.wait_inbox_state("e1", EngineInboxState::Poisoned).await; + assert_eq!(t.handle.poisons(&t.agg("a")).await.unwrap().len(), 1); + t.submit("a", "e2", "set_status", "ready").await; + t.wait_status("a", "ready").await; + t.submit("a", "e3", "panic_unless_status", "ready").await; + t.wait_status("a", "survived").await; + assert!(t.handle.poisons(&t.agg("a")).await.unwrap().is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn panics_and_invariants_poison_their_kind() { + let t = start().await; + t.submit("a", "e1", "panic", "").await; + let row = t.wait_inbox_state("e1", EngineInboxState::Poisoned).await; + assert!(row.last_error.as_deref().unwrap_or("").contains("boom in apply")); + let poisons = t.handle.poisons(&t.agg("a")).await.unwrap(); + assert_eq!(poisons.len(), 1); + assert_eq!(poisons[0].action, "panic"); + assert_eq!(t.handle.snapshot().await.unwrap().poisoned.len(), 1); + // Other kinds of the aggregate still apply. + t.submit("a", "e2", "set_status", "alive").await; + t.wait_status("a", "alive").await; + // An explicit invariant violation poisons its kind too; a manual clear + // releases every action of the aggregate. + t.submit("a", "e3", "invariant", "").await; + t.wait_inbox_state("e3", EngineInboxState::Poisoned).await; + assert_eq!(t.handle.poisons(&t.agg("a")).await.unwrap().len(), 2); + assert_eq!(t.handle.clear_poison(&t.agg("a"), None).await.unwrap(), 2); + assert!(t.handle.poisons(&t.agg("a")).await.unwrap().is_empty()); + // Poisoned rows are terminal: a duplicate is ignored. + assert_eq!( + t.submit("a", "e1", "panic", "").await, + SubmitOutcome::Duplicate { + state: EngineInboxState::Poisoned, + ack_sent: false, + rearmed: false + } + ); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn rejected_events_are_failed_acked_and_rearmed_by_a_duplicate() { + let mut t = start().await; + t.submit_acked("a", "e1", "reject", "", "peer-1", "d-1").await; + let row = t.wait_inbox_state("e1", EngineInboxState::Failed).await; + assert_eq!(row.last_error.as_deref(), Some("nope")); + assert!(row.payload.is_some(), "rejected payload kept for inspection"); + assert_eq!(t.wait_ack().await.delivery_id, "d-1"); + // A duplicate re-arms the failed row; it is applied again and fails again. + assert_eq!( + t.submit_acked("a", "e1", "reject", "", "peer-1", "d-1").await, + SubmitOutcome::Duplicate { + state: EngineInboxState::Failed, + ack_sent: false, + rearmed: true + } + ); + let row = t.wait_inbox_attempts("e1", 2).await; + assert_eq!(row.state, "Failed"); + assert_eq!(t.wait_ack().await.delivery_id, "d-1"); + // Unknown aggregate kinds have no handler and fail without an ACK. + t.handle + .submit(NewEvent::new( + "e9", + AggregateRef::new("nobody", "x"), + "set_status", + EventSource::Local, + vec![], + )) + .await + .unwrap(); + let row = t.wait_inbox_state("e9", EngineInboxState::Failed).await; + assert!(row.last_error.as_deref().unwrap_or("").contains("no handler")); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn waiting_events_wake_when_the_dependency_changes() { + let mut t = start().await; + t.submit_acked("a", "e1", "wait_on", "test:dep|ready", "peer-1", "d-1").await; + let row = + wait_for(WAIT, || async { t.inbox("e1").await.filter(|row| row.waiting_ref.is_some()) }) + .await; + assert_eq!(row.state, "Pending"); + assert_eq!(row.waiting_ref.as_deref(), Some("test:dep")); + assert!(row.payload.is_some(), "parked events keep their payload"); + // Parked material is durable, so the heavy message is ACKed already. + assert_eq!(t.wait_ack().await.delivery_id, "d-1"); + let snapshot = t.handle.snapshot().await.unwrap(); + assert_eq!(snapshot.waiting.len(), 1); + assert_eq!(snapshot.waiting[0].waiting_detail.as_deref(), Some("need ready")); + // A duplicate of a parked heavy message is ACKed again. + assert_eq!( + t.submit_acked("a", "e1", "wait_on", "test:dep|ready", "peer-1", "d-1").await, + SubmitOutcome::Duplicate { + state: EngineInboxState::Pending, + ack_sent: true, + rearmed: false + } + ); + assert_eq!(t.wait_ack().await.delivery_id, "d-1"); + // The dependency changing status wakes the waiter through the same commit. + t.submit("dep", "e2", "set_status", "ready").await; + t.wait_status("a", "unblocked").await; + let row = t.wait_inbox_state("e1", EngineInboxState::Applied).await; + assert!(row.waiting_ref.is_none()); + assert!(t.handle.snapshot().await.unwrap().waiting.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn waiting_events_are_rechecked_periodically_without_a_wakeup() { + let t = start().await; + t.submit("a", "e1", "wait_on", "test:dep|ready").await; + wait_for(WAIT, || async { t.inbox("e1").await.and_then(|row| row.waiting_ref).map(|_| ()) }) + .await; + // Change the dependency behind the engine's back: no wakeup is generated. + { + let mut s = t.db.acquire().await.unwrap(); + let update = + store::EngineAggregateUpdate { status: Some("ready".into()), ..Default::default() }; + s.upsert_engine_aggregate(KIND, "dep", &update, t.clock.now()).await.unwrap(); + } + t.wait_status("a", "unblocked").await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn outbox_republishes_with_backoff_until_acked_or_closed() { + let t = start().await; + t.submit("g", "e1", "send", "heavy|setup|p1:d1,p2:d2").await; + t.submit("g", "e2", "send", "small|setup|").await; + t.wait_publishes("heavy", 1).await; + t.wait_publishes("small", 1).await; + settle().await; + assert_eq!(t.transport.count(), 2, "one publish per row until the backoff passes"); + // Backoff: 30 s after the first publish, 60 s after the second. + t.clock.advance(30); + t.wait_publishes("heavy", 2).await; + t.wait_publishes("small", 2).await; + t.clock.advance(30); + settle().await; + assert_eq!(t.transport.count(), 4); + t.clock.advance(30); + t.wait_publishes("heavy", 3).await; + // ACKs close the heavy row only when the whole set answered. + assert_eq!( + t.handle.acknowledge("p1", "d1").await.unwrap(), + EngineOutboxAck::Matched { outbox_id: "heavy".into(), closed: false } + ); + assert_eq!(t.handle.acknowledge("p2", "wrong").await.unwrap(), EngineOutboxAck::NoMatch); + assert_eq!( + t.handle.acknowledge("p2", "d2").await.unwrap(), + EngineOutboxAck::Matched { outbox_id: "heavy".into(), closed: true } + ); + let heavy_before = t.transport.count_for("heavy"); + // Receiver evidence closes the small row through an apply. + t.submit("g", "e3", "close_phase", "setup").await; + t.wait_inbox_state("e3", EngineInboxState::Applied).await; + t.clock.advance(600); + settle().await; + assert_eq!(t.transport.count_for("heavy"), heavy_before); + assert_eq!(t.transport.count_for("small"), 3); + { + let mut s = t.db.acquire().await.unwrap(); + let small = s.find_engine_outbox_message("small").await.unwrap().unwrap(); + assert_eq!( + (small.state.as_str(), small.close_reason.as_deref()), + ("Closed", Some("evidence")) + ); + let heavy = s.find_engine_outbox_message("heavy").await.unwrap().unwrap(); + assert_eq!( + (heavy.state.as_str(), heavy.close_reason.as_deref()), + ("Closed", Some("acked")) + ); + } + // Transport failures are recorded and retried. + t.transport.fail.store(true, Ordering::SeqCst); + t.submit("g", "e4", "send", "later|other|").await; + let failed = wait_for(WAIT, || async { + let mut s = t.db.acquire().await.unwrap(); + s.find_engine_outbox_message("later").await.unwrap().filter(|row| row.last_error.is_some()) + }) + .await; + assert_eq!(failed.publish_count, 0); + t.transport.fail.store(false, Ordering::SeqCst); + t.clock.advance(30); + t.wait_publishes("later", 1).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn job_results_come_back_as_events_and_are_validated_against_the_phase() { + let t = start().await; + t.submit("a", "e1", "job", "echo|phase1|hello").await; + let job_id = engine::stable_job_id(&t.agg("a"), "phase1", "echo", b"hello"); + let job = t.wait_job_state(&job_id, "Done").await; + assert_eq!(job.result.as_deref(), Some(b"hello".as_slice())); + assert_eq!(job.generation, 1); + t.wait_status("a", "job_done:hello").await; + let result_event = + t.wait_inbox_state(&JobResultEvent::event_id(&job_id, 1), EngineInboxState::Applied).await; + assert_eq!(result_event.source, "job"); + // Re-creating the same job is a no-op. + t.submit("a", "e2", "job", "echo|phase1|hello").await; + t.wait_inbox_state("e2", EngineInboxState::Applied).await; + settle().await; + assert_eq!(t.wait_job_state(&job_id, "Done").await.attempts, 1); + // A result that arrives after the phase moved on is rejected by the aggregate. + t.submit("a", "e3", "job", "gated|phase2|slow").await; + let gated_id = engine::stable_job_id(&t.agg("a"), "phase2", "gated", b"slow"); + t.wait_job_state(&gated_id, "Running").await; + t.submit("a", "e4", "set_phase", "phase3").await; + t.wait_inbox_state("e4", EngineInboxState::Applied).await; + t.gate.send(true).unwrap(); + t.wait_job_state(&gated_id, "Done").await; + let stale_phase = + t.wait_inbox_state(&JobResultEvent::event_id(&gated_id, 1), EngineInboxState::Failed).await; + assert!(stale_phase.last_error.as_deref().unwrap_or("").contains("phase2")); + assert_eq!(t.handle.aggregate(&t.agg("a")).await.unwrap().unwrap().status, "job_done:hello"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn late_holder_is_fenced_and_its_external_result_is_reconciled() { + let t = start().await; + t.submit("a", "e1", "job", "gated|p|x").await; + let job_id = engine::stable_job_id(&t.agg("a"), "p", "gated", b"x"); + let first = t.wait_job_state(&job_id, "Running").await; + assert_eq!(first.generation, 1); + // The lease (1 s) lapses without renewal; the executor claims generation 2 + // while the first holder is still running. + t.clock.advance(2); + let second = wait_for(WAIT, || async { + let mut s = t.db.acquire().await.unwrap(); + s.find_engine_job(&job_id).await.unwrap().filter(|job| job.generation == 2) + }) + .await; + assert_eq!((second.state.as_str(), second.attempts), ("Running", 2)); + // Both holders finish; only generation 2 may write the row. + t.gate.send(true).unwrap(); + let done = t.wait_job_state(&job_id, "Done").await; + assert_eq!(done.generation, 2); + assert_eq!(done.result.as_deref(), Some(b"gen2".as_slice())); + // Both completions reach the aggregate as events: the fresh one advances + // the status, the stale one is reconciled (recorded in the state). + let fresh = + t.wait_inbox_state(&JobResultEvent::event_id(&job_id, 2), EngineInboxState::Applied).await; + assert_eq!(fresh.source, "job"); + let stale = + t.wait_inbox_state(&JobResultEvent::event_id(&job_id, 1), EngineInboxState::Applied).await; + assert_eq!(stale.source, "job"); + let agg = t.wait_status("a", "job_done:gen2").await; + assert_eq!(agg.state.as_deref(), Some("stale_seen"), "stale result reached the aggregate"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn job_completion_waits_out_a_long_write_lock_instead_of_rerunning() { + // A short busy timeout makes every blocked statement fail fast, like a + // long apply would make them fail against the 5 s default. + let t = start_with_busy_timeout(fast_config(), Duration::from_millis(300), async |_| {}).await; + t.submit("a", "e1", "job", "gated|p|x").await; + let job_id = engine::stable_job_id(&t.agg("a"), "p", "gated", b"x"); + t.wait_job_state(&job_id, "Running").await; + // Hold the write lock well past the old retry budget while the job finishes. + let blocker = t.db.start_immediate_transaction().await.unwrap(); + t.gate.send(true).unwrap(); + tokio::time::sleep(Duration::from_secs(4)).await; + assert_eq!(t.wait_job_state(&job_id, "Running").await.generation, 1, "still the first holder"); + drop(blocker); + let done = t.wait_job_state(&job_id, "Done").await; + assert_eq!( + (done.generation, done.attempts), + (1, 1), + "recorded by the original holder, not re-run" + ); + t.wait_status("a", "job_done:gen1").await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_acks_close_the_row_exactly_once() { + let t = start().await; + let acks: Vec = (0..8).map(|i| format!("p{i}:d{i}")).collect(); + t.submit("g", "e1", "send", &format!("heavy|setup|{}", acks.join(","))).await; + t.wait_publishes("heavy", 1).await; + let mut tasks = Vec::new(); + for i in 0..8 { + let handle = t.handle.clone(); + tasks.push(tokio::spawn(async move { + handle.acknowledge(&format!("p{i}"), &format!("d{i}")).await.unwrap() + })); + } + let mut closed = 0; + for task in tasks { + match task.await.unwrap() { + EngineOutboxAck::Matched { outbox_id, closed: was_closed } => { + assert_eq!(outbox_id, "heavy"); + closed += usize::from(was_closed); + } + EngineOutboxAck::NoMatch => panic!("every ACK must match"), + } + } + assert_eq!(closed, 1, "exactly one ACK closes the row"); + let mut s = t.db.acquire().await.unwrap(); + let row = s.find_engine_outbox_message("heavy").await.unwrap().unwrap(); + assert_eq!((row.state.as_str(), row.close_reason.as_deref()), ("Closed", Some("acked"))); + assert!(row.ack_set.unwrap().is_empty()); + drop(s); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn job_transient_failures_retry_and_panics_poison_the_action() { + let config = EngineConfig { poison_backoff: Backoff::Steps(vec![3_600]), ..fast_config() }; + let t = start_with(config, async |_| {}).await; + t.submit("a", "e1", "job", "flaky|p|").await; + let flaky_id = engine::stable_job_id(&t.agg("a"), "p", "flaky", b""); + for attempt in 1..=2 { + let job = wait_for(WAIT, || async { + let mut s = t.db.acquire().await.unwrap(); + s.find_engine_job(&flaky_id) + .await + .unwrap() + .filter(|job| job.attempts == attempt && job.state == "Pending") + }) + .await; + assert!(job.last_error.as_deref().unwrap_or("").contains("attempt")); + t.clock.advance(1); + } + let job = t.wait_job_state(&flaky_id, "Done").await; + assert_eq!(job.attempts, 3); + t.wait_status("a", "job_done:flaky-ok").await; + // Permanent failure is reported, not retried. + t.submit("a", "e2", "job", "fail|p|").await; + let fail_id = engine::stable_job_id(&t.agg("a"), "p", "fail", b""); + assert_eq!(t.wait_job_state(&fail_id, "Failed").await.attempts, 1); + t.wait_status("a", "job_failed:permanent").await; + // A panic poisons the job and holds the action back on the aggregate. + t.submit("a", "e3", "job", "boom|p|first").await; + let boom_id = engine::stable_job_id(&t.agg("a"), "p", "boom", b"first"); + t.wait_job_state(&boom_id, "Poisoned").await; + let poisons = wait_for(WAIT, || async { + Some(t.handle.poisons(&t.agg("a")).await.unwrap()).filter(|p| !p.is_empty()) + }) + .await; + assert_eq!((poisons[0].action.as_str(), poisons[0].poison_count), ("boom", 1)); + t.wait_status("a", "job_poisoned:boom in job").await; + assert_eq!( + t.handle.poisons(&t.agg("a")).await.unwrap().len(), + 1, + "a job result does not lift the poison" + ); + // A second job for the poisoned action, created without a new event, is + // held back by the executor while the backoff runs. + let second_id = engine::stable_job_id(&t.agg("a"), "p", "boom", b"second"); + { + let mut s = t.db.acquire().await.unwrap(); + assert!( + s.insert_engine_job( + &store::NewEngineJob { + job_id: second_id.clone(), + aggregate_kind: KIND.into(), + aggregate_id: "a".into(), + phase: "p".into(), + action: "boom".into(), + pool: "cpu".into(), + params: b"second".to_vec(), + deadline_at: None, + first_at: t.clock.now(), + }, + t.clock.now(), + ) + .await + .unwrap() + .is_inserted() + ); + } + t.handle.wake_all(); + settle().await; + assert_eq!( + t.wait_job_state(&second_id, "Pending").await.attempts, + 0, + "same action is held back" + ); + // Other actions keep running meanwhile. + t.submit("a", "e5", "job", "echo|p|still-works").await; + let echo_id = engine::stable_job_id(&t.agg("a"), "p", "echo", b"still-works"); + t.wait_job_state(&echo_id, "Done").await; + t.wait_inbox_state(&JobResultEvent::event_id(&echo_id, 1), EngineInboxState::Applied).await; + assert_eq!(t.wait_job_state(&second_id, "Pending").await.attempts, 0); + // Once the backoff passed the held job runs (and poisons the action again). + t.clock.advance(3_600); + t.wait_job_state(&second_id, "Poisoned").await; + assert_eq!(t.handle.poisons(&t.agg("a")).await.unwrap()[0].poison_count, 2); + // A manual clear lets the action run; a clean run releases the poison. + assert_eq!(t.handle.clear_poison(&t.agg("a"), Some("boom")).await.unwrap(), 1); + t.submit("a", "e6", "job", "boom|p|ok").await; + let ok_id = engine::stable_job_id(&t.agg("a"), "p", "boom", b"ok"); + t.wait_job_state(&ok_id, "Done").await; + t.wait_status("a", "job_done:boom-ok").await; + assert!(t.handle.poisons(&t.agg("a")).await.unwrap().is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn events_of_one_aggregate_apply_strictly_in_order() { + let t = start().await; + for index in 0..12 { + t.submit("serial", &format!("s{index}"), "serial_probe", "").await; + } + for index in 0..12 { + t.wait_inbox_state(&format!("s{index}"), EngineInboxState::Applied).await; + } + assert_eq!(t.probe.max_concurrent.load(Ordering::SeqCst), 1); + // Applied in arrival order. + let mut s = t.db.acquire().await.unwrap(); + let rows = s.list_engine_inbox_events_for_aggregate(KIND, "serial", 20).await.unwrap(); + let applied: Vec = rows.iter().map(|row| row.applied_at.unwrap()).collect(); + let mut sorted = applied.clone(); + sorted.sort_unstable_by(|a, b| b.cmp(a)); + assert_eq!(applied, sorted); + drop(s); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn startup_recovery_rearms_events_and_reclaims_leases() { + let job_id = engine::stable_job_id(&AggregateRef::new(KIND, "a"), "p", "echo", b"recover"); + let t = start_with(fast_config(), async |db: &LocalDB| { + let mut s = db.acquire().await.unwrap(); + // A pending event parked far in the future by a previous process. + s.insert_engine_inbox_event( + &store::NewEngineInboxEvent { + event_id: "old".into(), + aggregate_kind: KIND.into(), + aggregate_id: "a".into(), + kind: "set_status".into(), + source: "local".into(), + author: String::new(), + admission_class: String::new(), + payload: b"recovered".to_vec(), + ack: None, + }, + 1, + ) + .await + .unwrap(); + s.mark_engine_inbox_retry("old", "busy", i64::MAX / 2, 1).await.unwrap(); + // A job claimed by a dead process, lease far in the future. + s.insert_engine_job( + &store::NewEngineJob { + job_id: job_id.clone(), + aggregate_kind: KIND.into(), + aggregate_id: "a".into(), + phase: "p".into(), + action: "echo".into(), + pool: "cpu".into(), + params: b"recover".to_vec(), + deadline_at: None, + first_at: 1, + }, + 1, + ) + .await + .unwrap(); + let claimed = s.claim_due_engine_jobs("cpu", 1, i64::MAX / 2, 1).await.unwrap(); + assert_eq!(claimed[0].generation, 1); + // An outbox row leased by the dead process. + s.insert_engine_outbox_message( + &store::NewEngineOutboxMessage { + outbox_id: "leased".into(), + aggregate_kind: KIND.into(), + aggregate_id: "a".into(), + phase_ref: "p".into(), + kind: "Msg".into(), + topic: "t".into(), + payload: b"leased".to_vec(), + ack_set: None, + first_at: 1, + }, + 1, + ) + .await + .unwrap(); + assert_eq!(s.claim_due_engine_outbox_messages(1, i64::MAX / 2, 1).await.unwrap().len(), 1); + }) + .await; + t.wait_status("a", "recovered").await; + let job = t.wait_job_state(&job_id, "Done").await; + assert_eq!(job.generation, 2, "re-claimed with a new generation"); + t.wait_publishes("leased", 1).await; + t.stop().await; +} diff --git a/crates/store/.sqlx/query-0ab4a3e44fd041e0367f3654664d2162af691ab32529033816847dfe418c752d.json b/crates/store/.sqlx/query-0ab4a3e44fd041e0367f3654664d2162af691ab32529033816847dfe418c752d.json deleted file mode 100644 index 72a8bf22f..000000000 --- a/crates/store/.sqlx/query-0ab4a3e44fd041e0367f3654664d2162af691ab32529033816847dfe418c752d.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? WHERE status = ? AND updated_at < ?", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - }, - "hash": "0ab4a3e44fd041e0367f3654664d2162af691ab32529033816847dfe418c752d" -} diff --git a/crates/store/.sqlx/query-c43b2fa8472dddd5ba6e2845a3760b10f683547e1ddbe8b9165f6d5edb53e625.json b/crates/store/.sqlx/query-c43b2fa8472dddd5ba6e2845a3760b10f683547e1ddbe8b9165f6d5edb53e625.json deleted file mode 100644 index 7466e1eb6..000000000 --- a/crates/store/.sqlx/query-c43b2fa8472dddd5ba6e2845a3760b10f683547e1ddbe8b9165f6d5edb53e625.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE message\n SET state = 'Expired'\n WHERE state IN ('Pending', 'Processing')\n AND updated_at < ?", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "c43b2fa8472dddd5ba6e2845a3760b10f683547e1ddbe8b9165f6d5edb53e625" -} diff --git a/crates/store/.sqlx/query-e05adfcd50f53709318a8408a78d4905cea100c746e12f357518b7c73569706c.json b/crates/store/.sqlx/query-e05adfcd50f53709318a8408a78d4905cea100c746e12f357518b7c73569706c.json deleted file mode 100644 index 2623a5ffa..000000000 --- a/crates/store/.sqlx/query-e05adfcd50f53709318a8408a78d4905cea100c746e12f357518b7c73569706c.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "DELETE FROM message WHERE updated_at < ?", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "e05adfcd50f53709318a8408a78d4905cea100c746e12f357518b7c73569706c" -} diff --git a/crates/store/.sqlx/query-ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4.json b/crates/store/.sqlx/query-ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4.json deleted file mode 100644 index 73cd38143..000000000 --- a/crates/store/.sqlx/query-ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? WHERE instance_id = ? AND status = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - }, - "hash": "ede88a2e872505ac5e58b2e71280dc8081e776358af4302a88f2f683c8d97ad4" -} diff --git a/crates/store/migrations/20250810113519_set_auto_vacuum.sql b/crates/store/migrations/20250810113519_set_auto_vacuum.sql deleted file mode 100644 index 6cafbbfb2..000000000 --- a/crates/store/migrations/20250810113519_set_auto_vacuum.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Add migration script here -PRAGMA -auto_vacuum = 1; \ No newline at end of file diff --git a/crates/store/migrations/20250814113519_create_graph_table.sql b/crates/store/migrations/20250814113519_create_graph_table.sql deleted file mode 100644 index 7289917f3..000000000 --- a/crates/store/migrations/20250814113519_create_graph_table.sql +++ /dev/null @@ -1,38 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `graph`; -CREATE TABLE graph -( - `graph_id` TEXT NOT NULL DEFAULT '', - `instance_id` TEXT NOT NULL, - `kickoff_index` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `from_addr` TEXT NOT NULL DEFAULT '', - `to_addr` TEXT NOT NULL DEFAULT '', - `graph_ipfs_base_url` TEXT NOT NULL DEFAULT '', - `amount` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `challenge_amount` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `status` TEXT NOT NULL DEFAULT '', - `sub_status` TEXT NOT NULL DEFAULT '', - `operator_pubkey` TEXT NOT NULL DEFAULT '', - `cur_prekickoff_txid` TEXT, - `next_prekickoff` TEXT, - `force_skip_kickoff_txid` TEXT, - `quick_challenge_txid` TEXT, - `challenge_incomplete_kickoff_txid` TEXT, - `pegin_txid` TEXT, - `kickoff_txid` TEXT, - `take1_txid` TEXT, - `challenge_txid` TEXT, - `take2_txid` TEXT, - `watchtower_challenge_init_txid` TEXT, - `operator_assert_txid` TEXT, - `verifier_assert_txids` TEXT NOT NULL DEFAULT '[]', - `disprove_txids` TEXT NOT NULL DEFAULT '[]', - `init_withdraw_tx_hash` TEXT, - `bridge_out_start_at` BIGINT NOT NULL DEFAULT 0, - `zkm_version` TEXT NOT NULL DEFAULT '', - `status_updated_at` BIGINT NOT NULL DEFAULT 0, - `proceed_withdraw_height` BIGINT NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`graph_id`) -); diff --git a/crates/store/migrations/20250814113702_create_message_table.sql b/crates/store/migrations/20250814113702_create_message_table.sql deleted file mode 100644 index 6c7de81e3..000000000 --- a/crates/store/migrations/20250814113702_create_message_table.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `message`; -CREATE TABLE message -( - `message_id` TEXT NOT NULL, - `business_id` TEXT NOT NULL, - `from_peer` TEXT NOT NULL DEFAULT 'self', - `actor` TEXT NOT NULL, - `msg_type` TEXT NOT NULL, - `content` BLOB NOT NULL, - `state` TEXT NOT NULL DEFAULT 'pending', - `lock_time_until` BIGINT NOT NULL DEFAULT 0, - `weight` BIGINT NOT NULL DEFAULT 0, - `message_version` BIGINT NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`message_id`) -) \ No newline at end of file diff --git a/crates/store/migrations/20250814114142_create_goat_tx_record_table.sql b/crates/store/migrations/20250814114142_create_goat_tx_record_table.sql deleted file mode 100644 index 4e613940a..000000000 --- a/crates/store/migrations/20250814114142_create_goat_tx_record_table.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `goat_tx_record`; -CREATE TABLE goat_tx_record -( - `instance_id` TEXT NOT NULL, - `graph_id` TEXT NOT NULL, - `tx_type` TEXT NOT NULL DEFAULT 'Normal', - `tx_hash` TEXT NOT NULL DEFAULT '', - `height` BIGINT NOT NULL DEFAULT 0, - `is_local` BOOL NOT NULL DEFAULT 1, - `extra` TEXT, - `created_at` BIGINT NOT NULL DEFAULT 0, - `processing_status` TEXT NOT NULL DEFAULT 'Skipped', - PRIMARY KEY (`instance_id`, `graph_id`, `tx_type`) -); -CREATE INDEX id_goat_tx_hash ON goat_tx_record (`tx_hash`); \ No newline at end of file diff --git a/crates/store/migrations/20250814115805_create_instance_table.sql b/crates/store/migrations/20250814115805_create_instance_table.sql deleted file mode 100644 index c563492b2..000000000 --- a/crates/store/migrations/20250814115805_create_instance_table.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `instance`; -CREATE TABLE instance -( - `instance_id` TEXT NOT NULL DEFAULT '', - `network` TEXT NOT NULL DEFAULT 'test', - `from_addr` TEXT NOT NULL DEFAULT '', - `to_addr` TEXT NOT NULL DEFAULT '', - `amount` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `fees` TEXT NOT NULL DEFAULT '[0, 0, 0]', - `status` TEXT NOT NULL DEFAULT '', - `input_utxos` TEXT NOT NULL DEFAULT '', - `goat_tx_hash` TEXT NOT NULL DEFAULT '', - `goat_tx_height` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `user_xonly_pubkey` TEXT NOT NULL DEFAULT '[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]', - `user_change_addr` TEXT NOT NULL DEFAULT '', - `user_refund_addr` TEXT NOT NULL DEFAULT '', - `btc_txid` TEXT, - `pegin_confirm_txid` TEXT, - `pegin_cancel_txid` TEXT, - `committees_answers` TEXT NOT NULL DEFAULT '{}', - `pegin_data_tx_hash` TEXT NOT NULL DEFAULT '', - `btc_height` BIGINT UNSIGNED NOT NULL DEFAULT 0, - `parameters` TEXT, - `post_pegin_txhash` TEXT, - `status_updated_at` BIGINT NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`instance_id`) -); \ No newline at end of file diff --git a/crates/store/migrations/20250904154339_create_graph_raw_data_table.sql b/crates/store/migrations/20250904154339_create_graph_raw_data_table.sql deleted file mode 100644 index ce5433568..000000000 --- a/crates/store/migrations/20250904154339_create_graph_raw_data_table.sql +++ /dev/null @@ -1,9 +0,0 @@ -DROP TABLE IF EXISTS `graph_raw_data`; -CREATE TABLE graph_raw_data -( - `graph_id` TEXT NOT NULL, - `raw_data` TEXT NOT NULL DEFAULT '', - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`graph_id`) -); diff --git a/crates/store/migrations/20251010084043_create_pegin_instance_process_data_table.sql b/crates/store/migrations/20251010084043_create_pegin_instance_process_data_table.sql deleted file mode 100644 index b9de1c130..000000000 --- a/crates/store/migrations/20251010084043_create_pegin_instance_process_data_table.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Add migration script here --- Add migration script here -DROP TABLE IF EXISTS `pegin_instance_process_data`; -CREATE TABLE pegin_instance_process_data -( - `instance_id` TEXT NOT NULL, - `process_data` TEXT NOT NULL DEFAULT '', - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`instance_id`) -); diff --git a/crates/store/migrations/20251010154017_create_pegin_graph_process_data_table.sql b/crates/store/migrations/20251010154017_create_pegin_graph_process_data_table.sql deleted file mode 100644 index 06ae83579..000000000 --- a/crates/store/migrations/20251010154017_create_pegin_graph_process_data_table.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `pegin_graph_process_data`; -CREATE TABLE pegin_graph_process_data -( - `graph_id` TEXT NOT NULL, - `instance_id` TEXT NOT NULL, - `process_data` TEXT NOT NULL DEFAULT '', - `is_endorsed` BOOLEAN NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`graph_id`) -); diff --git a/crates/store/migrations/20251024161053_create_commit_info_table.sql b/crates/store/migrations/20251024161053_create_commit_info_table.sql deleted file mode 100644 index d038518d3..000000000 --- a/crates/store/migrations/20251024161053_create_commit_info_table.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `commit_info`; -CREATE TABLE commit_info -( - `txid` TEXT NOT NULL, - `threshold` BIGINT NOT NULL DEFAULT 0, - `publisher_public_keys` TEXT NOT NULL, - `commit_proof_id` BIGINT NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`txid`) -); \ No newline at end of file diff --git a/crates/store/migrations/20251112025410_create_graph_btc_tx_vout_monitor_table.sql b/crates/store/migrations/20251112025410_create_graph_btc_tx_vout_monitor_table.sql deleted file mode 100644 index 1372f517b..000000000 --- a/crates/store/migrations/20251112025410_create_graph_btc_tx_vout_monitor_table.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Add migration script here --- Add migration script here -DROP TABLE IF EXISTS `graph_btc_tx_vout_monitor`; -CREATE TABLE `graph_btc_tx_vout_monitor` -( - `graph_id` TEXT NOT NULL, - `tx_name` TEXT NOT NULL, - `txid` TEXT NOT NULL, - `height` BIGINT NOT NULL DEFAULT 0, - `vout_len` BIGINT NOT NULL DEFAULT 0, - `monitor_data` TEXT NOT NULL DEFAULT '', - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`graph_id`, `txid`) -); diff --git a/crates/store/migrations/20251213124200_create_watch_contract_table.sql.sql b/crates/store/migrations/20251213124200_create_watch_contract_table.sql.sql deleted file mode 100644 index 1a7f02530..000000000 --- a/crates/store/migrations/20251213124200_create_watch_contract_table.sql.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `watch_contract`; -CREATE TABLE watch_contract -( - `contract_addr` TEXT NOT NULL DEFAULT '', - `gap` BIGINT NOT NULL DEFAULT 0, - `from_height` BIGINT NOT NULL DEFAULT 0, - `the_graph_url` TEXT NOT NULL DEFAULT '', - `status` TEXT NOT NULL DEFAULT 'UnSync', - `extra` TEXT, - `updated_at` BIGINT NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`contract_addr`) -); \ No newline at end of file diff --git a/crates/store/migrations/20251215023559_create_operator_proof_table.sql b/crates/store/migrations/20251215023559_create_operator_proof_table.sql deleted file mode 100644 index ae2c2d6c8..000000000 --- a/crates/store/migrations/20251215023559_create_operator_proof_table.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `operator_proof`; -CREATE TABLE operator_proof -( - `id` INTEGER PRIMARY KEY, - `instance_id` TEXT NOT NULL, - `graph_id` TEXT NOT NULL, - `execution_layer_block_number` BIGINT NOT NULL DEFAULT 0, - `path_to_proof` TEXT, - `proof_size` BIGINT NOT NULL DEFAULT 0, - `public_value_hex` TEXT, - `cycles` BIGINT NOT NULL DEFAULT 0, - `proof_state` BIGINT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), - `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, - `proving_time` BIGINT NOT NULL DEFAULT 0, - `zkm_version` TEXT NOT NULL DEFAULT '', - `operator_committed_blockhash` TEXT NOT NULL, - `extra` TEXT, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0 -); \ No newline at end of file diff --git a/crates/store/migrations/20251215023612_create_watchtower_proof_table.sql b/crates/store/migrations/20251215023612_create_watchtower_proof_table.sql deleted file mode 100644 index 02229977a..000000000 --- a/crates/store/migrations/20251215023612_create_watchtower_proof_table.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `watchtower_proof`; -CREATE TABLE watchtower_proof -( - `id` INTEGER PRIMARY KEY, - `instance_id` TEXT NOT NULL, - `graph_id` TEXT NOT NULL, - `public_key` TEXT NOT NULL, - `challenge_txid` TEXT NOT NULL, - `challenge_init_txid` TEXT NOT NULL, - `execution_layer_block_number` BIGINT NOT NULL DEFAULT 0, - `path_to_proof` TEXT, - `proof_size` BIGINT NOT NULL DEFAULT 0, - `public_value_hex` TEXT, - `cycles` BIGINT NOT NULL DEFAULT 0, - `proof_state` BIGINT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), - `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, - `proving_time` BIGINT NOT NULL DEFAULT 0, - `zkm_version` TEXT NOT NULL DEFAULT '', - `node_index` INTEGER NOT NULL DEFAULT 0, - `included` BOOLEAN NOT NULL DEFAULT 0, - `extra` TEXT, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0 -); \ No newline at end of file diff --git a/crates/store/migrations/20251215024440_create_long_running_task_table.sql b/crates/store/migrations/20251215024440_create_long_running_task_table.sql deleted file mode 100644 index 2fb652ea9..000000000 --- a/crates/store/migrations/20251215024440_create_long_running_task_table.sql +++ /dev/null @@ -1,44 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `long_running_task_proof`; -CREATE TABLE long_running_task_proof -( - `id` INTEGER PRIMARY KEY, - `block_start` BIGINT NOT NULL DEFAULT 0, - `block_end` BIGINT NOT NULL DEFAULT 0, - `chain_name` TEXT NOT NULL CHECK ( chain_name in ('header-chain', 'state-chain', 'commit-chain') ), - `path_to_proof` TEXT, - `cycles` BIGINT NOT NULL DEFAULT 0, - `proof_state` INT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), - `proof_size` BIGINT NOT NULL DEFAULT 0, - `public_value_hex` TEXT, - `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, - `proving_time` BIGINT NOT NULL DEFAULT 0, - `zkm_version` TEXT NOT NULL DEFAULT '', - `extra` TEXT, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0 -); - -INSERT INTO long_running_task_proof ( - block_start, - block_end, - chain_name, - path_to_proof, - cycles, - proof_state, - total_time_to_proof, - proving_time, - zkm_version, - extra -) VALUES ( - 0, - 116000, - 'header-chain', - '../circuits/data/header-chain/0-116000.bin', - 3147770424, - 2, - 637403, - 637403, - 'v1.2.3', - '' -); diff --git a/crates/store/migrations/20260108112458_create_node_table.sql b/crates/store/migrations/20260108112458_create_node_table.sql deleted file mode 100644 index 19ea88d8c..000000000 --- a/crates/store/migrations/20260108112458_create_node_table.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `node`; -CREATE TABLE node -( - `peer_id` TEXT NOT NULL DEFAULT '', - `actor` TEXT NOT NULL DEFAULT '', - `goat_addr` TEXT NOT NULL DEFAULT '', - `btc_pub_key` TEXT NOT NULL DEFAULT '', - socket_addr TEXT NOT NULL DEFAULT '', - `node_name` TEXT NOT NULL DEFAULT '', - `service_fee_rate` REAL NOT NULL DEFAULT 0.0, - `available_peg_btc` - TEXT NOT NULL DEFAULT '0', - `reward` - TEXT NOT NULL DEFAULT '0', - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`peer_id`) -); \ No newline at end of file diff --git a/crates/store/migrations/20260109021138_create_bridge_out_global_stats_table.sql b/crates/store/migrations/20260109021138_create_bridge_out_global_stats_table.sql deleted file mode 100644 index d038b068a..000000000 --- a/crates/store/migrations/20260109021138_create_bridge_out_global_stats_table.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Add migration script here -DROP TABLE IF EXISTS `bridge_out_global_stats`; -CREATE TABLE bridge_out_global_stats -( - `id` INTEGER PRIMARY KEY, - `initial_txn` BIGINT NOT NULL DEFAULT 0, - `initial_amount` TEXT NOT NULL DEFAULT '0', - `claim_txn` BIGINT NOT NULL DEFAULT 0, - `claim_amount` TEXT NOT NULL DEFAULT '0', - `refund_txn` BIGINT NOT NULL DEFAULT 0, - `refund_amount` TEXT NOT NULL DEFAULT '0', - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0 -); \ No newline at end of file diff --git a/crates/store/migrations/20260206160000_create_sequencer_set_hash_monitor_tables.sql b/crates/store/migrations/20260206160000_create_sequencer_set_hash_monitor_tables.sql deleted file mode 100644 index aa4ebc144..000000000 --- a/crates/store/migrations/20260206160000_create_sequencer_set_hash_monitor_tables.sql +++ /dev/null @@ -1,22 +0,0 @@ -CREATE TABLE IF NOT EXISTS sequencer_set_hash_changes -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - cosmos_block_height BIGINT NOT NULL UNIQUE, - goat_block_height BIGINT NOT NULL, - validators_hash TEXT NOT NULL, - created_at BIGINT NOT NULL DEFAULT 0, - updated_at BIGINT NOT NULL DEFAULT 0 -); - -CREATE INDEX IF NOT EXISTS idx_sequencer_set_hash_changes_goat_block_height - ON sequencer_set_hash_changes (goat_block_height); - -CREATE TABLE IF NOT EXISTS sequencer_set_scan_state -( - id INTEGER PRIMARY KEY CHECK (id = 1), - next_cosmos_block_height BIGINT NOT NULL, - latest_goat_block_height BIGINT NOT NULL, - latest_validators_hash TEXT NOT NULL, - created_at BIGINT NOT NULL DEFAULT 0, - updated_at BIGINT NOT NULL DEFAULT 0 -); diff --git a/crates/store/migrations/20260211100000_drop_graph_ipfs_base_url.sql b/crates/store/migrations/20260211100000_drop_graph_ipfs_base_url.sql deleted file mode 100644 index 31a3a60ed..000000000 --- a/crates/store/migrations/20260211100000_drop_graph_ipfs_base_url.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Drop deprecated column graph_ipfs_base_url from graph table -ALTER TABLE graph DROP COLUMN `graph_ipfs_base_url`; diff --git a/crates/store/migrations/20260406120000_drop_graph_zkm_version.sql b/crates/store/migrations/20260406120000_drop_graph_zkm_version.sql deleted file mode 100644 index 3938b9ba9..000000000 --- a/crates/store/migrations/20260406120000_drop_graph_zkm_version.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Drop graph-level zkm_version because version semantics now live on proofs only -ALTER TABLE graph DROP COLUMN `zkm_version`; diff --git a/crates/store/migrations/20260504120000_create_wrapper_proof_table.sql b/crates/store/migrations/20260504120000_create_wrapper_proof_table.sql deleted file mode 100644 index 6cd2adce8..000000000 --- a/crates/store/migrations/20260504120000_create_wrapper_proof_table.sql +++ /dev/null @@ -1,30 +0,0 @@ -CREATE TABLE IF NOT EXISTS wrapper_proof -( - `id` INTEGER PRIMARY KEY, - `operator_proof_id` BIGINT NOT NULL, - `instance_id` TEXT NOT NULL, - `graph_id` TEXT NOT NULL, - `execution_layer_block_number` BIGINT NOT NULL DEFAULT 0, - `operator_path_to_proof` TEXT NOT NULL DEFAULT '', - `path_to_proof` TEXT, - `public_value_hex` TEXT, - `operator_vk_hash` TEXT NOT NULL DEFAULT '', - `genesis_sequencer_commit_txid` TEXT NOT NULL DEFAULT '', - `operator_public_value_hex` TEXT, - `proof_size` BIGINT NOT NULL DEFAULT 0, - `cycles` BIGINT NOT NULL DEFAULT 0, - `proof_state` BIGINT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), - `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, - `proving_time` BIGINT NOT NULL DEFAULT 0, - `zkm_version` TEXT NOT NULL DEFAULT '', - `extra` TEXT, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - UNIQUE (`operator_proof_id`) -); - -CREATE INDEX IF NOT EXISTS idx_wrapper_proof_instance_graph - ON wrapper_proof (`instance_id`, `graph_id`); - -CREATE INDEX IF NOT EXISTS idx_wrapper_proof_state - ON wrapper_proof (`proof_state`); diff --git a/crates/store/migrations/20260524000000_create_pending_graph_init_table.sql b/crates/store/migrations/20260524000000_create_pending_graph_init_table.sql deleted file mode 100644 index f03cb8c07..000000000 --- a/crates/store/migrations/20260524000000_create_pending_graph_init_table.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE IF NOT EXISTS pending_graph_init -( - `instance_id` TEXT NOT NULL, - `operator_pubkey` TEXT NOT NULL, - `graph_id` TEXT NOT NULL UNIQUE, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`instance_id`, `operator_pubkey`) -); diff --git a/crates/store/migrations/20260611000000_add_gc_v2_graph_txids.sql b/crates/store/migrations/20260611000000_add_gc_v2_graph_txids.sql deleted file mode 100644 index 5f468bd61..000000000 --- a/crates/store/migrations/20260611000000_add_gc_v2_graph_txids.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE graph - ADD COLUMN `watchtower_challenge_timeout_txids` TEXT NOT NULL DEFAULT '[]'; - -ALTER TABLE graph - ADD COLUMN `operator_challenge_nack_txids` TEXT NOT NULL DEFAULT '[]'; - -ALTER TABLE graph - ADD COLUMN `operator_commit_timeout_txid` TEXT; diff --git a/crates/store/migrations/20260622000000_drop_wrapper_proof_table.sql b/crates/store/migrations/20260622000000_drop_wrapper_proof_table.sql deleted file mode 100644 index 4a5426310..000000000 --- a/crates/store/migrations/20260622000000_drop_wrapper_proof_table.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS wrapper_proof; diff --git a/crates/store/migrations/20260720000000_add_graph_definition_hash.sql b/crates/store/migrations/20260720000000_add_graph_definition_hash.sql deleted file mode 100644 index 4240e96ed..000000000 --- a/crates/store/migrations/20260720000000_add_graph_definition_hash.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE graph - ADD COLUMN `definition_hash` TEXT NOT NULL DEFAULT ''; diff --git a/crates/store/migrations/20260720000001_create_graph_compensation_marker.sql b/crates/store/migrations/20260720000001_create_graph_compensation_marker.sql deleted file mode 100644 index 6f0a73147..000000000 --- a/crates/store/migrations/20260720000001_create_graph_compensation_marker.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE graph_compensation_marker ( - graph_id BLOB NOT NULL, - message_id TEXT NOT NULL, - created_at INTEGER NOT NULL, - PRIMARY KEY (graph_id, message_id) -); diff --git a/crates/store/migrations/20260807130000_create_message_debug_table.sql b/crates/store/migrations/20260807130000_create_message_debug_table.sql deleted file mode 100644 index 7c16af203..000000000 --- a/crates/store/migrations/20260807130000_create_message_debug_table.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE IF NOT EXISTS message_debug_reason ( - message_id TEXT NOT NULL, - reason_code TEXT NOT NULL, - reason_detail TEXT NOT NULL, - first_seen_at BIGINT NOT NULL, - last_seen_at BIGINT NOT NULL, - occurrences BIGINT NOT NULL DEFAULT 1, - PRIMARY KEY (message_id, reason_code, reason_detail) -); diff --git a/crates/store/migrations/20260812120000_create_p2p_inbox_table.sql b/crates/store/migrations/20260812120000_create_p2p_inbox_table.sql deleted file mode 100644 index 6cdceaff4..000000000 --- a/crates/store/migrations/20260812120000_create_p2p_inbox_table.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE IF NOT EXISTS p2p_inbox ( - message_id TEXT NOT NULL PRIMARY KEY, - business_id TEXT, - actor TEXT NOT NULL, - from_peer TEXT NOT NULL, - msg_type TEXT NOT NULL, - content BLOB NOT NULL, - content_size BIGINT NOT NULL, - state TEXT NOT NULL DEFAULT 'Pending', - attempt_count BIGINT NOT NULL DEFAULT 0, - next_retry_at BIGINT NOT NULL DEFAULT 0, - lease_until BIGINT NOT NULL DEFAULT 0, - last_error TEXT, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_p2p_inbox_ready - ON p2p_inbox (state, next_retry_at, created_at); diff --git a/crates/store/migrations/20260812121000_create_p2p_outbox_table.sql b/crates/store/migrations/20260812121000_create_p2p_outbox_table.sql deleted file mode 100644 index 7dd97ed4e..000000000 --- a/crates/store/migrations/20260812121000_create_p2p_outbox_table.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE IF NOT EXISTS p2p_outbox ( - message_id TEXT NOT NULL PRIMARY KEY, - msg_type TEXT NOT NULL, - content BLOB NOT NULL, - state TEXT NOT NULL DEFAULT 'Pending', - attempt_count BIGINT NOT NULL DEFAULT 0, - next_retry_at BIGINT NOT NULL DEFAULT 0, - lease_until BIGINT NOT NULL DEFAULT 0, - last_error TEXT, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_p2p_outbox_ready - ON p2p_outbox (state, next_retry_at, created_at); diff --git a/crates/store/migrations/20260812122000_add_p2p_inbox_lease_token.sql b/crates/store/migrations/20260812122000_add_p2p_inbox_lease_token.sql deleted file mode 100644 index 57232d41a..000000000 --- a/crates/store/migrations/20260812122000_add_p2p_inbox_lease_token.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE p2p_inbox - ADD COLUMN lease_token TEXT NOT NULL DEFAULT ''; diff --git a/crates/store/migrations/20260827120000_add_p2p_outbox_retry_window.sql b/crates/store/migrations/20260827120000_add_p2p_outbox_retry_window.sql deleted file mode 100644 index 7f027539f..000000000 --- a/crates/store/migrations/20260827120000_add_p2p_outbox_retry_window.sql +++ /dev/null @@ -1,6 +0,0 @@ -ALTER TABLE p2p_outbox ADD COLUMN retry_until BIGINT NOT NULL DEFAULT 0; -ALTER TABLE p2p_outbox ADD COLUMN retry_interval_secs BIGINT NOT NULL DEFAULT 0; -ALTER TABLE p2p_outbox ADD COLUMN ack_peer_id TEXT NOT NULL DEFAULT ''; - -CREATE INDEX IF NOT EXISTS idx_p2p_outbox_retry_window - ON p2p_outbox (state, retry_until, next_retry_at); diff --git a/crates/store/migrations/20260827130000_create_swap_escrow_table.sql b/crates/store/migrations/20260827130000_create_swap_escrow_table.sql deleted file mode 100644 index 812d611c9..000000000 --- a/crates/store/migrations/20260827130000_create_swap_escrow_table.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Swap-based bridge-out escrows, keyed by the on-chain escrow hash. --- Fully independent of the BitVM `instance`/`graph` flow. -DROP TABLE IF EXISTS `swap_escrow`; -CREATE TABLE swap_escrow -( - `escrow_hash` TEXT NOT NULL, -- 0x-prefixed 32-byte hex, lowercase - `network` TEXT NOT NULL DEFAULT '', - `status` TEXT NOT NULL DEFAULT '', -- SwapEscrowStatus - `offerer_addr` TEXT NOT NULL DEFAULT '', -- GOAT account funding the escrow - `claimer_addr` TEXT NOT NULL DEFAULT '', -- GOAT account entitled to claim - `btc_addr` TEXT NOT NULL DEFAULT '', -- BTC address receiving the payout - `token` TEXT NOT NULL DEFAULT '', -- escrow token contract address - `amount` TEXT NOT NULL DEFAULT '0', -- escrow amount, U256 decimal string - `refund_deadline` BIGINT NOT NULL DEFAULT 0, -- unix secs after which refund is possible - `escrow_data` TEXT, -- hex abi-encoded EscrowData from Initialize tx - `init_tx_hash` TEXT NOT NULL DEFAULT '', -- swap Initialize GOAT tx - `init_tx_height` BIGINT NOT NULL DEFAULT 0, - `claim_tx_hash` TEXT NOT NULL DEFAULT '', -- swap Claim GOAT tx - `claim_btc_txid` TEXT, -- BTC payout tx committed by the claim - `refund_tx_hash` TEXT NOT NULL DEFAULT '', -- swap Refund GOAT tx - `status_updated_at` BIGINT NOT NULL DEFAULT 0, - `created_at` BIGINT NOT NULL DEFAULT 0, - `updated_at` BIGINT NOT NULL DEFAULT 0, - PRIMARY KEY (`escrow_hash`) -); diff --git a/crates/store/migrations/20260914000000_add_queue_poison_guards.sql b/crates/store/migrations/20260914000000_add_queue_poison_guards.sql deleted file mode 100644 index 964e1f850..000000000 --- a/crates/store/migrations/20260914000000_add_queue_poison_guards.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Poison-message guards for the two durable work queues. --- --- Both queues previously had no way to tell "the handler returned Err and asked --- for a retry" apart from "the attempt never finished because the process died". --- Only the latter indicates a message that reproducibly takes the node down, so --- it needs its own counter and a much smaller ceiling: a transient RPC or SQLite --- outage must not push legitimate messages toward quarantine. --- --- `abandon_count` is incremented when an expired `Processing` lease is reclaimed, --- so the next worker durably records that the previous dispatch never finished. - -ALTER TABLE p2p_inbox ADD COLUMN abandon_count BIGINT NOT NULL DEFAULT 0; - -ALTER TABLE message ADD COLUMN attempt_count BIGINT NOT NULL DEFAULT 0; -ALTER TABLE message ADD COLUMN abandon_count BIGINT NOT NULL DEFAULT 0; -ALTER TABLE message ADD COLUMN last_error TEXT; - -CREATE INDEX IF NOT EXISTS idx_message_claimable ON message (state, lock_time_until, created_at); diff --git a/crates/store/migrations/20260918000000_add_p2p_admission.sql b/crates/store/migrations/20260918000000_add_p2p_admission.sql deleted file mode 100644 index 3ea18e53b..000000000 --- a/crates/store/migrations/20260918000000_add_p2p_admission.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Admission control for externally received gossip. --- --- Inbound P2P messages are persisted before any business validation, so the --- node must bound what an arbitrary peer can make it store, and must be able to --- tell an authenticated sender apart from an anonymous one. - --- p2p_inbox: how the sender was classified when a row was admitted, and a digest --- used to collapse an identical payload the same sender re-publishes with a fresh --- gossip id. `admission_class` is `Committee`, `Registered`, or `Unregistered`. -ALTER TABLE p2p_inbox ADD COLUMN admission_class TEXT NOT NULL DEFAULT 'Unregistered'; -ALTER TABLE p2p_inbox ADD COLUMN content_hash BLOB; - --- Covering index for the per-sender quota query. `content` is a large blob and --- `content_size` follows it, so reading usage from the table would walk every --- overflow page of every queued row. `state` leads so the partial working set is --- scanned directly. -CREATE INDEX IF NOT EXISTS idx_p2p_inbox_admission - ON p2p_inbox (state, admission_class, from_peer, content_size); - -CREATE INDEX IF NOT EXISTS idx_p2p_inbox_content_hash - ON p2p_inbox (from_peer, content_hash); - --- node: the sender's proof that the on-chain-registered key `btc_pub_key` --- authorised this `peer_id`. gossipsub authenticates the peer id; this Schnorr --- signature (by the node master key over the peer id) is what lets a receiver --- trust the mapping and grant the registered sender class. `binding_issued_at` --- orders re-bindings so one key maps to a single peer id at a time. -ALTER TABLE node ADD COLUMN binding_sig TEXT NOT NULL DEFAULT ''; -ALTER TABLE node ADD COLUMN binding_issued_at BIGINT NOT NULL DEFAULT 0; diff --git a/crates/store/migrations/20260918000001_add_p2p_registered_peer.sql b/crates/store/migrations/20260918000001_add_p2p_registered_peer.sql deleted file mode 100644 index 8b0864b58..000000000 --- a/crates/store/migrations/20260918000001_add_p2p_registered_peer.sql +++ /dev/null @@ -1,35 +0,0 @@ --- Peers this node confirmed on chain as registered senders: committee members --- (peer id registry) and operators (staked master key bound to the peer id). --- --- The admission registry is an in-memory cache filled by chain lookups, and --- first-time lookups are budgeted so a flood of fresh identities cannot turn --- into a flood of RPC calls. Without this table a restart would send every real --- member back through that budget, competing with the flood. --- --- `pubkey` is the hex x-only master key the confirmation was made for (empty for --- committee rows). A registration is only as good as the key that was checked: --- a peer that re-binds to another key must not inherit it, and a key that moves --- to another peer id takes its row along. Rows are written only after a positive --- chain answer, removed on a negative one, and an operator key holds at most one --- row, so the table is bounded by the on-chain registries, not by who connects. -CREATE TABLE IF NOT EXISTS p2p_registered_peer -( - peer_id TEXT NOT NULL, - kind TEXT NOT NULL, - pubkey TEXT NOT NULL DEFAULT '', - verified_at BIGINT NOT NULL, - PRIMARY KEY (peer_id, kind) -); - -CREATE INDEX IF NOT EXISTS idx_p2p_registered_peer_pubkey - ON p2p_registered_peer (kind, pubkey); - --- Highest gossipsub sequence number admitted per registered author. Replay --- protection keeps a window per author in memory; this mark is what survives a --- restart, so messages captured before it cannot be replayed into a node that --- has just come back up. One row per registered author. -CREATE TABLE IF NOT EXISTS p2p_replay_mark -( - peer_id TEXT NOT NULL PRIMARY KEY, - highest BIGINT NOT NULL -); diff --git a/crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql b/crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql deleted file mode 100644 index 2480d0c43..000000000 --- a/crates/store/migrations/20260918000002_add_p2p_outbox_publish_count.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Successful publishes of an outbox row. Signing-round rows are re-published for --- as long as their round is open, and back off by this count. `attempt_count` --- cannot serve: it also counts claims whose publish failed, so a node that came --- up without peers would reach the slowest tier before delivering anything. -ALTER TABLE p2p_outbox ADD COLUMN publish_count BIGINT NOT NULL DEFAULT 0; diff --git a/crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql b/crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql deleted file mode 100644 index 45aa50d82..000000000 --- a/crates/store/migrations/20260927000000_add_p2p_setup_delivery_metadata.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE p2p_outbox ADD COLUMN delivery_id TEXT NOT NULL DEFAULT ''; - -CREATE INDEX IF NOT EXISTS idx_p2p_outbox_delivery_id - ON p2p_outbox (delivery_id, state); diff --git a/crates/store/migrations/20261015000000_schema.sql b/crates/store/migrations/20261015000000_schema.sql new file mode 100644 index 000000000..bab0941f4 --- /dev/null +++ b/crates/store/migrations/20261015000000_schema.sql @@ -0,0 +1,404 @@ +-- Baseline schema. Nodes are deployed on a fresh database, so the earlier +-- incremental migrations are folded into this one file. + +-- ------------------------------------------------------------ peg-in / graphs + +CREATE TABLE instance +( + `instance_id` TEXT NOT NULL DEFAULT '', + `network` TEXT NOT NULL DEFAULT 'test', + `from_addr` TEXT NOT NULL DEFAULT '', + `to_addr` TEXT NOT NULL DEFAULT '', + `amount` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `fees` TEXT NOT NULL DEFAULT '[0, 0, 0]', + `status` TEXT NOT NULL DEFAULT '', + `input_utxos` TEXT NOT NULL DEFAULT '', + `goat_tx_hash` TEXT NOT NULL DEFAULT '', + `goat_tx_height` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `user_xonly_pubkey` TEXT NOT NULL DEFAULT '[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]', + `user_change_addr` TEXT NOT NULL DEFAULT '', + `user_refund_addr` TEXT NOT NULL DEFAULT '', + `btc_txid` TEXT, + `pegin_confirm_txid` TEXT, + `pegin_cancel_txid` TEXT, + `committees_answers` TEXT NOT NULL DEFAULT '{}', + `pegin_data_tx_hash` TEXT NOT NULL DEFAULT '', + `btc_height` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `parameters` TEXT, + `post_pegin_txhash` TEXT, + `status_updated_at` BIGINT NOT NULL DEFAULT 0, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + -- The committee's instance key was deleted after the peg-in settled. + `key_deleted` INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (`instance_id`) +); + +CREATE TABLE graph +( + `graph_id` TEXT NOT NULL DEFAULT '', + `instance_id` TEXT NOT NULL, + `kickoff_index` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `from_addr` TEXT NOT NULL DEFAULT '', + `to_addr` TEXT NOT NULL DEFAULT '', + `amount` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `challenge_amount` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `status` TEXT NOT NULL DEFAULT '', + `sub_status` TEXT NOT NULL DEFAULT '', + `operator_pubkey` TEXT NOT NULL DEFAULT '', + `cur_prekickoff_txid` TEXT, + `next_prekickoff` TEXT, + `force_skip_kickoff_txid` TEXT, + `quick_challenge_txid` TEXT, + `challenge_incomplete_kickoff_txid` TEXT, + `pegin_txid` TEXT, + `kickoff_txid` TEXT, + `take1_txid` TEXT, + `challenge_txid` TEXT, + `take2_txid` TEXT, + `watchtower_challenge_init_txid` TEXT, + `operator_assert_txid` TEXT, + `verifier_assert_txids` TEXT NOT NULL DEFAULT '[]', + `disprove_txids` TEXT NOT NULL DEFAULT '[]', + `init_withdraw_tx_hash` TEXT, + `bridge_out_start_at` BIGINT NOT NULL DEFAULT 0, + `status_updated_at` BIGINT NOT NULL DEFAULT 0, + `proceed_withdraw_height` BIGINT NOT NULL DEFAULT 0, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + `watchtower_challenge_timeout_txids` TEXT NOT NULL DEFAULT '[]', + `operator_challenge_nack_txids` TEXT NOT NULL DEFAULT '[]', + `operator_commit_timeout_txid` TEXT, + -- Hash of the stored definition; a replay with another hash is refused. + `definition_hash` TEXT NOT NULL DEFAULT '', + PRIMARY KEY (`graph_id`) +); +-- One graph per operator slot. +CREATE UNIQUE INDEX idx_graph_operator_slot + ON graph (operator_pubkey, kickoff_index); + +CREATE TABLE graph_raw_data +( + `graph_id` TEXT NOT NULL, + `raw_data` TEXT NOT NULL DEFAULT '', + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`graph_id`) +); + +-- An operator's graph setup in progress: one per (instance, operator). +CREATE TABLE pending_graph_init +( + `instance_id` TEXT NOT NULL, + `operator_pubkey` TEXT NOT NULL, + `graph_id` TEXT NOT NULL UNIQUE, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`instance_id`, `operator_pubkey`) +); + +CREATE TABLE pegin_instance_process_data +( + `instance_id` TEXT NOT NULL, + `process_data` TEXT NOT NULL DEFAULT '', + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`instance_id`) +); + +CREATE TABLE pegin_graph_process_data +( + `graph_id` TEXT NOT NULL, + `instance_id` TEXT NOT NULL, + `process_data` TEXT NOT NULL DEFAULT '', + `is_endorsed` BOOLEAN NOT NULL DEFAULT 0, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`graph_id`) +); + +-- Confirmed graph transactions whose outputs the chain scan watches. +CREATE TABLE `graph_btc_tx_vout_monitor` +( + `graph_id` TEXT NOT NULL, + `tx_name` TEXT NOT NULL, + `txid` TEXT NOT NULL, + `height` BIGINT NOT NULL DEFAULT 0, + `vout_len` BIGINT NOT NULL DEFAULT 0, + `monitor_data` TEXT NOT NULL DEFAULT '', + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`graph_id`, `txid`) +); + +-- Each signing nonce is bound to one transcript before it signs. +CREATE TABLE signing_authorization ( + nonce_id TEXT NOT NULL PRIMARY KEY, + -- graph | instance + scope_kind TEXT NOT NULL, + scope_id TEXT NOT NULL, + transcript_digest TEXT NOT NULL, + created_at BIGINT NOT NULL +); + +-- ------------------------------------------------------------------- Goat + +CREATE TABLE goat_tx_record +( + `instance_id` TEXT NOT NULL, + `graph_id` TEXT NOT NULL, + `tx_type` TEXT NOT NULL DEFAULT 'Normal', + `tx_hash` TEXT NOT NULL DEFAULT '', + `height` BIGINT NOT NULL DEFAULT 0, + `is_local` BOOL NOT NULL DEFAULT 1, + `extra` TEXT, + `created_at` BIGINT NOT NULL DEFAULT 0, + `processing_status` TEXT NOT NULL DEFAULT 'Skipped', + PRIMARY KEY (`instance_id`, `graph_id`, `tx_type`) +); +CREATE INDEX id_goat_tx_hash ON goat_tx_record (`tx_hash`); + +CREATE TABLE watch_contract +( + `contract_addr` TEXT NOT NULL DEFAULT '', + `gap` BIGINT NOT NULL DEFAULT 0, + `from_height` BIGINT NOT NULL DEFAULT 0, + `the_graph_url` TEXT NOT NULL DEFAULT '', + `status` TEXT NOT NULL DEFAULT 'UnSync', + `extra` TEXT, + `updated_at` BIGINT NOT NULL DEFAULT 0, + `created_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`contract_addr`) +); + +CREATE TABLE commit_info +( + `txid` TEXT NOT NULL, + `threshold` BIGINT NOT NULL DEFAULT 0, + `publisher_public_keys` TEXT NOT NULL, + `commit_proof_id` BIGINT NOT NULL DEFAULT 0, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`txid`) +); + +CREATE TABLE bridge_out_global_stats +( + `id` INTEGER PRIMARY KEY, + `initial_txn` BIGINT NOT NULL DEFAULT 0, + `initial_amount` TEXT NOT NULL DEFAULT '0', + `claim_txn` BIGINT NOT NULL DEFAULT 0, + `claim_amount` TEXT NOT NULL DEFAULT '0', + `refund_txn` BIGINT NOT NULL DEFAULT 0, + `refund_amount` TEXT NOT NULL DEFAULT '0', + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE swap_escrow +( + `escrow_hash` TEXT NOT NULL, -- 0x-prefixed 32-byte hex, lowercase + `network` TEXT NOT NULL DEFAULT '', + `status` TEXT NOT NULL DEFAULT '', -- SwapEscrowStatus + `offerer_addr` TEXT NOT NULL DEFAULT '', -- GOAT account funding the escrow + `claimer_addr` TEXT NOT NULL DEFAULT '', -- GOAT account entitled to claim + `btc_addr` TEXT NOT NULL DEFAULT '', -- BTC address receiving the payout + `token` TEXT NOT NULL DEFAULT '', -- escrow token contract address + `amount` TEXT NOT NULL DEFAULT '0', -- escrow amount, U256 decimal string + `refund_deadline` BIGINT NOT NULL DEFAULT 0, -- unix secs after which refund is possible + `escrow_data` TEXT, -- hex abi-encoded EscrowData from Initialize tx + `init_tx_hash` TEXT NOT NULL DEFAULT '', -- swap Initialize GOAT tx + `init_tx_height` BIGINT NOT NULL DEFAULT 0, + `claim_tx_hash` TEXT NOT NULL DEFAULT '', -- swap Claim GOAT tx + `claim_btc_txid` TEXT, -- BTC payout tx committed by the claim + `refund_tx_hash` TEXT NOT NULL DEFAULT '', -- swap Refund GOAT tx + `status_updated_at` BIGINT NOT NULL DEFAULT 0, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`escrow_hash`) +); + +CREATE TABLE sequencer_set_hash_changes +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + cosmos_block_height BIGINT NOT NULL UNIQUE, + goat_block_height BIGINT NOT NULL, + validators_hash TEXT NOT NULL, + created_at BIGINT NOT NULL DEFAULT 0, + updated_at BIGINT NOT NULL DEFAULT 0 +); +CREATE INDEX idx_sequencer_set_hash_changes_goat_block_height + ON sequencer_set_hash_changes (goat_block_height); + +CREATE TABLE sequencer_set_scan_state +( + id INTEGER PRIMARY KEY CHECK (id = 1), + next_cosmos_block_height BIGINT NOT NULL, + latest_goat_block_height BIGINT NOT NULL, + latest_validators_hash TEXT NOT NULL, + created_at BIGINT NOT NULL DEFAULT 0, + updated_at BIGINT NOT NULL DEFAULT 0 +); + +-- -------------------------------------------------------------------- P2P + +CREATE TABLE node +( + `peer_id` TEXT NOT NULL DEFAULT '', + `actor` TEXT NOT NULL DEFAULT '', + `goat_addr` TEXT NOT NULL DEFAULT '', + `btc_pub_key` TEXT NOT NULL DEFAULT '', + socket_addr TEXT NOT NULL DEFAULT '', + `node_name` TEXT NOT NULL DEFAULT '', + `service_fee_rate` REAL NOT NULL DEFAULT 0.0, + `available_peg_btc` + TEXT NOT NULL DEFAULT '0', + `reward` + TEXT NOT NULL DEFAULT '0', + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0, + -- The peer's signature binding `btc_pub_key` to `peer_id`, and when it was issued. + binding_sig TEXT NOT NULL DEFAULT '', + binding_issued_at BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (`peer_id`) +); + +-- Peers whose on-chain registration this node confirmed (committee, operator). +CREATE TABLE p2p_registered_peer +( + peer_id TEXT NOT NULL, + kind TEXT NOT NULL, + pubkey TEXT NOT NULL DEFAULT '', + verified_at BIGINT NOT NULL, + PRIMARY KEY (peer_id, kind) +); +CREATE INDEX idx_p2p_registered_peer_pubkey + ON p2p_registered_peer (kind, pubkey); + +-- ----------------------------------------------------------- execution engine + +CREATE TABLE engine_inbox ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + -- Producer-supplied dedup key: content id, chain event id, height key, + -- or job result id. Arrival of a known id never creates a second row. + event_id TEXT NOT NULL UNIQUE, + aggregate_kind TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + -- Stable hash of (aggregate_kind, aggregate_id); workers own shard_key % K. + shard_key BIGINT NOT NULL, + kind TEXT NOT NULL, + source TEXT NOT NULL, + author TEXT NOT NULL DEFAULT '', + -- Cleared once the event is Applied; kept while Pending, Waiting or Failed. + payload BLOB, + payload_size BIGINT NOT NULL DEFAULT 0, + -- Pending | Applied | Failed | Poisoned + state TEXT NOT NULL DEFAULT 'Pending', + attempts BIGINT NOT NULL DEFAULT 0, + next_at BIGINT NOT NULL DEFAULT 0, + -- Set while the event is Pending because a dependency is missing. + waiting_ref TEXT, + waiting_detail TEXT, + -- Heavy messages: who to ACK once the material is durably applied or parked. + ack_peer TEXT, + ack_delivery_id TEXT, + last_error TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + applied_at BIGINT, + -- The sender class the author had when a network event was admitted; + -- empty for local, chain and job events. Meters the inbox quotas. + admission_class TEXT NOT NULL DEFAULT '' +); +CREATE INDEX idx_engine_inbox_runnable + ON engine_inbox (state, next_at, shard_key, seq); +CREATE INDEX idx_engine_inbox_waiting + ON engine_inbox (state, waiting_ref); +CREATE INDEX idx_engine_inbox_admission + ON engine_inbox (source, state, admission_class, author, payload_size); + +CREATE TABLE engine_aggregate ( + aggregate_kind TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT '', + phase TEXT NOT NULL DEFAULT '', + -- Opaque state owned by the aggregate handler (JSON). + state TEXT, + version BIGINT NOT NULL DEFAULT 0, + last_event_id TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY (aggregate_kind, aggregate_id) +); + +-- A job action or event kind that panicked on an aggregate: held back until +-- `blocked_until`, released by a clean run or a manual clear. +CREATE TABLE engine_aggregate_poison ( + aggregate_kind TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + action TEXT NOT NULL, + reason TEXT NOT NULL, + poison_count INTEGER NOT NULL, + poisoned_at BIGINT NOT NULL, + blocked_until BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY (aggregate_kind, aggregate_id, action) +); + +CREATE TABLE engine_outbox ( + outbox_id TEXT NOT NULL PRIMARY KEY, + aggregate_kind TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + -- The phase whose terminal predicate closes a small message. + phase_ref TEXT NOT NULL, + kind TEXT NOT NULL, + topic TEXT NOT NULL, + payload BLOB NOT NULL, + -- Heavy messages: JSON object of outstanding recipient peer id -> delivery id. + -- NULL for small messages, which close on receiver-side evidence instead. + ack_set TEXT, + -- Open | Closed + state TEXT NOT NULL DEFAULT 'Open', + close_reason TEXT, + publish_count BIGINT NOT NULL DEFAULT 0, + next_at BIGINT NOT NULL DEFAULT 0, + lease_until BIGINT NOT NULL DEFAULT 0, + last_error TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + closed_at BIGINT +); +CREATE INDEX idx_engine_outbox_due + ON engine_outbox (state, next_at); +CREATE INDEX idx_engine_outbox_phase + ON engine_outbox (aggregate_kind, aggregate_id, phase_ref, state); + +CREATE TABLE engine_job ( + -- Stable id H(aggregate ‖ phase ‖ action ‖ params hash); re-creation is a no-op. + job_id TEXT NOT NULL PRIMARY KEY, + aggregate_kind TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + phase TEXT NOT NULL, + action TEXT NOT NULL, + -- cpu | pegout | network + pool TEXT NOT NULL, + params BLOB, + -- Pending | Running | Done | Failed | Cancelled | Poisoned + state TEXT NOT NULL DEFAULT 'Pending', + -- Incremented on every claim; a completion with a stale generation is fenced. + generation BIGINT NOT NULL DEFAULT 0, + lease_until BIGINT NOT NULL DEFAULT 0, + attempts BIGINT NOT NULL DEFAULT 0, + next_at BIGINT NOT NULL DEFAULT 0, + deadline_at BIGINT, + cancel_requested INTEGER NOT NULL DEFAULT 0, + result BLOB, + last_error TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + finished_at BIGINT +); +CREATE INDEX idx_engine_job_due + ON engine_job (state, next_at, lease_until); +CREATE INDEX idx_engine_job_aggregate + ON engine_job (aggregate_kind, aggregate_id, phase, state); diff --git a/crates/store/migrations/20261020000000_proof_builder_tables.sql b/crates/store/migrations/20261020000000_proof_builder_tables.sql new file mode 100644 index 000000000..df9e7c5a0 --- /dev/null +++ b/crates/store/migrations/20261020000000_proof_builder_tables.sql @@ -0,0 +1,72 @@ +-- Tables of proof-builder-rpc (crates/store/src/localdb/proof.rs). The node does not use them; they +-- share the store migrations because proof-builder-rpc opens its database through store::create_local_db. +-- Same definitions as the dev migrations 20251215023559 / 20251215023612 / 20251215024440, including the +-- header-chain 0..116000 seed proof. + +CREATE TABLE operator_proof +( + `id` INTEGER PRIMARY KEY, + `instance_id` TEXT NOT NULL, + `graph_id` TEXT NOT NULL, + `execution_layer_block_number` BIGINT NOT NULL DEFAULT 0, + `path_to_proof` TEXT, + `proof_size` BIGINT NOT NULL DEFAULT 0, + `public_value_hex` TEXT, + `cycles` BIGINT NOT NULL DEFAULT 0, + `proof_state` BIGINT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), + `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, + `proving_time` BIGINT NOT NULL DEFAULT 0, + `zkm_version` TEXT NOT NULL DEFAULT '', + `operator_committed_blockhash` TEXT NOT NULL, + `extra` TEXT, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE watchtower_proof +( + `id` INTEGER PRIMARY KEY, + `instance_id` TEXT NOT NULL, + `graph_id` TEXT NOT NULL, + `public_key` TEXT NOT NULL, + `challenge_txid` TEXT NOT NULL, + `challenge_init_txid` TEXT NOT NULL, + `execution_layer_block_number` BIGINT NOT NULL DEFAULT 0, + `path_to_proof` TEXT, + `proof_size` BIGINT NOT NULL DEFAULT 0, + `public_value_hex` TEXT, + `cycles` BIGINT NOT NULL DEFAULT 0, + `proof_state` BIGINT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), + `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, + `proving_time` BIGINT NOT NULL DEFAULT 0, + `zkm_version` TEXT NOT NULL DEFAULT '', + `node_index` INTEGER NOT NULL DEFAULT 0, + `included` BOOLEAN NOT NULL DEFAULT 0, + `extra` TEXT, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE long_running_task_proof +( + `id` INTEGER PRIMARY KEY, + `block_start` BIGINT NOT NULL DEFAULT 0, + `block_end` BIGINT NOT NULL DEFAULT 0, + `chain_name` TEXT NOT NULL CHECK ( chain_name in ('header-chain', 'state-chain', 'commit-chain') ), + `path_to_proof` TEXT, + `cycles` BIGINT NOT NULL DEFAULT 0, + `proof_state` INT NOT NULL DEFAULT 0 CHECK (proof_state IN (0, 1, 2, 3)), + `proof_size` BIGINT NOT NULL DEFAULT 0, + `public_value_hex` TEXT, + `total_time_to_proof` BIGINT NOT NULL DEFAULT 0, + `proving_time` BIGINT NOT NULL DEFAULT 0, + `zkm_version` TEXT NOT NULL DEFAULT '', + `extra` TEXT, + `created_at` BIGINT NOT NULL DEFAULT 0, + `updated_at` BIGINT NOT NULL DEFAULT 0 +); + +INSERT INTO long_running_task_proof (block_start, block_end, chain_name, path_to_proof, cycles, proof_state, + total_time_to_proof, proving_time, zkm_version, extra) +VALUES (0, 116000, 'header-chain', '../circuits/data/header-chain/0-116000.bin', 3147770424, 2, 637403, 637403, + 'v1.2.3', ''); diff --git a/crates/store/src/engine/mod.rs b/crates/store/src/engine/mod.rs new file mode 100644 index 000000000..89987ba92 --- /dev/null +++ b/crates/store/src/engine/mod.rs @@ -0,0 +1,1732 @@ +//! Storage for engine events, aggregate state, outbox messages and leased jobs. +//! +//! These methods perform database operations. Scheduling, retry and delivery +//! policies are implemented by the execution engine. + +use crate::localdb::StorageProcessor; +use serde::{Deserialize, Serialize}; +use sqlx::Row; +use sqlx::sqlite::SqliteRow; +use std::collections::BTreeMap; +use strum::{Display, EnumString}; + +/// Lifecycle of an inbox event. `Pending` covers both runnable and waiting +/// rows; a waiting row carries a `waiting_ref`. +#[derive(Clone, Copy, Debug, Display, EnumString, PartialEq, Eq, Serialize, Deserialize)] +pub enum EngineInboxState { + Pending, + Applied, + Failed, + Poisoned, +} + +#[derive(Clone, Copy, Debug, Display, EnumString, PartialEq, Eq, Serialize, Deserialize)] +pub enum EngineOutboxState { + Open, + Closed, +} + +#[derive(Clone, Copy, Debug, Display, EnumString, PartialEq, Eq, Serialize, Deserialize)] +pub enum EngineJobState { + Pending, + Running, + Done, + Failed, + Cancelled, + Poisoned, +} + +/// Stable shard key for an aggregate: FNV-1a over `kind:id`, folded to a +/// non-negative i64 so `shard_key % K` works in SQL for any worker count. +pub fn engine_shard_key(aggregate_kind: &str, aggregate_id: &str) -> i64 { + const OFFSET: u64 = 0xcbf2_9ce4_8422_2325; + const PRIME: u64 = 0x0000_0100_0000_01b3; + let mut hash = OFFSET; + for byte in aggregate_kind.bytes().chain([b':']).chain(aggregate_id.bytes()) { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(PRIME); + } + (hash >> 1) as i64 +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EngineAckTarget { + pub peer: String, + pub delivery_id: String, +} + +#[derive(Clone, Debug, Default)] +pub struct NewEngineInboxEvent { + pub event_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub kind: String, + pub source: String, + pub author: String, + /// How admission classified the author of a network event (empty for + /// local, chain and job events); the storage quota is kept per class. + pub admission_class: String, + pub payload: Vec, + pub ack: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EngineInboxInsert { + Inserted { + seq: i64, + }, + /// The event id was already known. `ack_again` is true when the material + /// is already durably applied or parked, so a heavy message may be ACKed + /// again; `rearmed` is true when a `Failed` row was put back to `Pending`. + Duplicate { + state: EngineInboxState, + ack_again: bool, + rearmed: bool, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EngineInboxEvent { + pub seq: i64, + pub event_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub shard_key: i64, + pub kind: String, + pub source: String, + pub author: String, + pub admission_class: String, + pub payload: Option>, + pub payload_size: i64, + pub state: String, + pub attempts: i64, + pub next_at: i64, + pub waiting_ref: Option, + pub waiting_detail: Option, + pub ack_peer: Option, + pub ack_delivery_id: Option, + pub last_error: Option, + pub created_at: i64, + pub updated_at: i64, + pub applied_at: Option, +} + +impl EngineInboxEvent { + pub fn ack_target(&self) -> Option { + match (&self.ack_peer, &self.ack_delivery_id) { + (Some(peer), Some(delivery_id)) => { + Some(EngineAckTarget { peer: peer.clone(), delivery_id: delivery_id.clone() }) + } + _ => None, + } + } +} + +const ENGINE_INBOX_COLUMNS: &str = "seq, event_id, aggregate_kind, aggregate_id, shard_key, kind, \ + source, author, admission_class, payload, payload_size, state, attempts, next_at, \ + waiting_ref, waiting_detail, ack_peer, ack_delivery_id, last_error, created_at, updated_at, \ + applied_at"; + +fn engine_inbox_event_from_row(row: &SqliteRow) -> Result { + Ok(EngineInboxEvent { + seq: row.try_get("seq")?, + event_id: row.try_get("event_id")?, + aggregate_kind: row.try_get("aggregate_kind")?, + aggregate_id: row.try_get("aggregate_id")?, + shard_key: row.try_get("shard_key")?, + kind: row.try_get("kind")?, + source: row.try_get("source")?, + author: row.try_get("author")?, + admission_class: row.try_get("admission_class")?, + payload: row.try_get("payload")?, + payload_size: row.try_get("payload_size")?, + state: row.try_get("state")?, + attempts: row.try_get("attempts")?, + next_at: row.try_get("next_at")?, + waiting_ref: row.try_get("waiting_ref")?, + waiting_detail: row.try_get("waiting_detail")?, + ack_peer: row.try_get("ack_peer")?, + ack_delivery_id: row.try_get("ack_delivery_id")?, + last_error: row.try_get("last_error")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + applied_at: row.try_get("applied_at")?, + }) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EngineAggregate { + pub aggregate_kind: String, + pub aggregate_id: String, + pub status: String, + pub phase: String, + pub state: Option, + pub version: i64, + pub last_event_id: Option, + pub created_at: i64, + pub updated_at: i64, +} + +const ENGINE_AGGREGATE_COLUMNS: &str = "aggregate_kind, aggregate_id, status, phase, state, \ + version, last_event_id, created_at, updated_at"; + +fn engine_aggregate_from_row(row: &SqliteRow) -> Result { + Ok(EngineAggregate { + aggregate_kind: row.try_get("aggregate_kind")?, + aggregate_id: row.try_get("aggregate_id")?, + status: row.try_get("status")?, + phase: row.try_get("phase")?, + state: row.try_get("state")?, + version: row.try_get("version")?, + last_event_id: row.try_get("last_event_id")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + }) +} + +/// What an apply changes on the aggregate row. `None` leaves a field alone; +/// `Some(None)` clears an optional field. +#[derive(Clone, Debug, Default)] +pub struct EngineAggregateUpdate { + pub status: Option, + pub phase: Option, + pub state: Option>, + pub last_event_id: Option, +} + +/// An action (job action or event kind) of an aggregate that panicked. It is +/// held back until `blocked_until`; the backoff grows with `poison_count`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnginePoison { + pub aggregate_kind: String, + pub aggregate_id: String, + pub action: String, + pub reason: String, + pub poison_count: i64, + pub poisoned_at: i64, + pub blocked_until: i64, +} + +/// A parked inbox event: what it waits on and since when. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EngineWaitingEvent { + pub event_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub kind: String, + pub waiting_ref: String, + pub waiting_detail: Option, + pub next_at: i64, + pub created_at: i64, + pub updated_at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EngineJobInsert { + Inserted, + /// A job with this id exists already; it is run again only if it is still open. + Existing(EngineJobState), +} + +impl EngineJobInsert { + pub fn is_inserted(&self) -> bool { + matches!(self, EngineJobInsert::Inserted) + } +} + +#[derive(Clone, Debug)] +pub struct NewEngineOutboxMessage { + pub outbox_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub phase_ref: String, + pub kind: String, + pub topic: String, + pub payload: Vec, + /// Outstanding recipients (peer id -> delivery id) for heavy messages. + pub ack_set: Option>, + /// First publish time; usually `now`. + pub first_at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EngineOutboxMessage { + pub outbox_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub phase_ref: String, + pub kind: String, + pub topic: String, + pub payload: Vec, + pub ack_set: Option>, + pub state: String, + pub close_reason: Option, + pub publish_count: i64, + pub next_at: i64, + pub lease_until: i64, + pub last_error: Option, + pub created_at: i64, + pub updated_at: i64, + pub closed_at: Option, +} + +const ENGINE_OUTBOX_COLUMNS: &str = "outbox_id, aggregate_kind, aggregate_id, phase_ref, kind, \ + topic, payload, ack_set, state, close_reason, publish_count, next_at, lease_until, last_error, \ + created_at, updated_at, closed_at"; + +fn engine_outbox_message_from_row(row: &SqliteRow) -> Result { + let ack_set: Option = row.try_get("ack_set")?; + let ack_set = match ack_set { + Some(json) => Some(serde_json::from_str(&json).map_err(|error| { + sqlx::Error::ColumnDecode { index: "ack_set".to_string(), source: Box::new(error) } + })?), + None => None, + }; + Ok(EngineOutboxMessage { + outbox_id: row.try_get("outbox_id")?, + aggregate_kind: row.try_get("aggregate_kind")?, + aggregate_id: row.try_get("aggregate_id")?, + phase_ref: row.try_get("phase_ref")?, + kind: row.try_get("kind")?, + topic: row.try_get("topic")?, + payload: row.try_get("payload")?, + ack_set, + state: row.try_get("state")?, + close_reason: row.try_get("close_reason")?, + publish_count: row.try_get("publish_count")?, + next_at: row.try_get("next_at")?, + lease_until: row.try_get("lease_until")?, + last_error: row.try_get("last_error")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + closed_at: row.try_get("closed_at")?, + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EngineOutboxCloseTarget { + Row(String), + Phase { aggregate_kind: String, aggregate_id: String, phase_ref: String }, + Aggregate { aggregate_kind: String, aggregate_id: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EngineOutboxAck { + /// The peer was removed from the row's ack set; `closed` when it was the last. + Matched { + outbox_id: String, + closed: bool, + }, + NoMatch, +} + +#[derive(Clone, Debug)] +pub struct NewEngineJob { + pub job_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub phase: String, + pub action: String, + pub pool: String, + pub params: Vec, + pub deadline_at: Option, + pub first_at: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EngineJob { + pub job_id: String, + pub aggregate_kind: String, + pub aggregate_id: String, + pub phase: String, + pub action: String, + pub pool: String, + pub params: Option>, + pub state: String, + pub generation: i64, + pub lease_until: i64, + pub attempts: i64, + pub next_at: i64, + pub deadline_at: Option, + pub cancel_requested: bool, + pub result: Option>, + pub last_error: Option, + pub created_at: i64, + pub updated_at: i64, + pub finished_at: Option, +} + +const ENGINE_JOB_COLUMNS: &str = "job_id, aggregate_kind, aggregate_id, phase, action, pool, \ + params, state, generation, lease_until, attempts, next_at, deadline_at, cancel_requested, \ + result, last_error, created_at, updated_at, finished_at"; + +fn engine_job_from_row(row: &SqliteRow) -> Result { + Ok(EngineJob { + job_id: row.try_get("job_id")?, + aggregate_kind: row.try_get("aggregate_kind")?, + aggregate_id: row.try_get("aggregate_id")?, + phase: row.try_get("phase")?, + action: row.try_get("action")?, + pool: row.try_get("pool")?, + params: row.try_get("params")?, + state: row.try_get("state")?, + generation: row.try_get("generation")?, + lease_until: row.try_get("lease_until")?, + attempts: row.try_get("attempts")?, + next_at: row.try_get("next_at")?, + deadline_at: row.try_get("deadline_at")?, + cancel_requested: row.try_get::("cancel_requested")? != 0, + result: row.try_get("result")?, + last_error: row.try_get("last_error")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + finished_at: row.try_get("finished_at")?, + }) +} + +/// How a claimed job ended, as reported by the executor. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EngineJobFinish { + Done(Vec), + /// Transient failure: back to `Pending` at `next_at`. + Retry { + error: String, + next_at: i64, + }, + Failed(String), + Poisoned(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EngineJobFinishOutcome { + Applied { + state: EngineJobState, + }, + /// The generation no longer belongs to the caller (lease lost and re-claimed, + /// or the job was cancelled). Nothing was written. + Stale { + current_generation: i64, + state: String, + }, + Missing, +} + +/// Queued inbox usage for one admission class, with the share attributable to +/// the author being admitted. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct InboxClassUsage { + pub admission_class: String, + pub rows: i64, + pub bytes: i64, + pub peer_rows: i64, + pub peer_bytes: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EngineStateCount { + pub table: String, + pub state: String, + pub count: i64, +} + +fn ack_set_json(ack_set: &BTreeMap) -> anyhow::Result { + Ok(serde_json::to_string(ack_set)?) +} + +impl<'a> StorageProcessor<'a> { + // ------------------------------------------------------------------ inbox + + /// Record an event once. A duplicate id is never inserted twice: `Failed` + /// rows are re-armed, everything else is left as is. + pub async fn insert_engine_inbox_event( + &mut self, + event: &NewEngineInboxEvent, + now: i64, + ) -> anyhow::Result { + let shard_key = engine_shard_key(&event.aggregate_kind, &event.aggregate_id); + let (ack_peer, ack_delivery_id) = match &event.ack { + Some(ack) => (Some(ack.peer.as_str()), Some(ack.delivery_id.as_str())), + None => (None, None), + }; + let inserted = sqlx::query( + "INSERT OR IGNORE INTO engine_inbox \ + (event_id, aggregate_kind, aggregate_id, shard_key, kind, source, author, \ + admission_class, payload, payload_size, state, attempts, next_at, ack_peer, \ + ack_delivery_id, created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'Pending', 0, ?, ?, ?, ?, ?) \ + RETURNING seq", + ) + .bind(&event.event_id) + .bind(&event.aggregate_kind) + .bind(&event.aggregate_id) + .bind(shard_key) + .bind(&event.kind) + .bind(&event.source) + .bind(&event.author) + .bind(&event.admission_class) + .bind(&event.payload) + .bind(event.payload.len() as i64) + .bind(now) + .bind(ack_peer) + .bind(ack_delivery_id) + .bind(now) + .bind(now) + .fetch_optional(self.conn()) + .await?; + if let Some(row) = inserted { + return Ok(EngineInboxInsert::Inserted { seq: row.try_get("seq")? }); + } + let existing = + sqlx::query("SELECT state, waiting_ref FROM engine_inbox WHERE event_id = ?") + .bind(&event.event_id) + .fetch_one(self.conn()) + .await?; + let state: String = existing.try_get("state")?; + let waiting_ref: Option = existing.try_get("waiting_ref")?; + let state = state + .parse::() + .map_err(|_| anyhow::anyhow!("unknown engine inbox state {state:?}"))?; + let mut rearmed = false; + if state == EngineInboxState::Failed { + rearmed = sqlx::query( + "UPDATE engine_inbox SET state = 'Pending', next_at = ?, waiting_ref = NULL, \ + waiting_detail = NULL, last_error = NULL, updated_at = ? \ + WHERE event_id = ? AND state = 'Failed'", + ) + .bind(now) + .bind(now) + .bind(&event.event_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0; + } + let ack_again = match state { + EngineInboxState::Applied => true, + EngineInboxState::Pending => waiting_ref.is_some(), + EngineInboxState::Failed | EngineInboxState::Poisoned => false, + }; + Ok(EngineInboxInsert::Duplicate { state, ack_again, rearmed }) + } + + pub async fn find_engine_inbox_event( + &mut self, + event_id: &str, + ) -> anyhow::Result> { + let row = sqlx::query(&format!( + "SELECT {ENGINE_INBOX_COLUMNS} FROM engine_inbox WHERE event_id = ?" + )) + .bind(event_id) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| engine_inbox_event_from_row(&row)).transpose()?) + } + + /// Oldest runnable event owned by `shard` out of `shard_count`. No lease is + /// taken: a shard has exactly one worker, and an apply either commits as a + /// whole or leaves the row untouched. + pub async fn next_runnable_engine_inbox_event( + &mut self, + shard_count: i64, + shard: i64, + now: i64, + ) -> anyhow::Result> { + let row = sqlx::query(&format!( + "SELECT {ENGINE_INBOX_COLUMNS} FROM engine_inbox e \ + WHERE e.state = 'Pending' AND e.next_at <= ? AND (e.shard_key % ?) = ? \ + AND NOT EXISTS (SELECT 1 FROM engine_aggregate_poison p \ + WHERE p.aggregate_kind = e.aggregate_kind AND p.aggregate_id = e.aggregate_id \ + AND p.action = e.kind AND p.blocked_until > ?) \ + ORDER BY e.seq LIMIT 1" + )) + .bind(now) + .bind(shard_count) + .bind(shard) + .bind(now) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| engine_inbox_event_from_row(&row)).transpose()?) + } + + pub async fn mark_engine_inbox_applied( + &mut self, + event_id: &str, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_inbox SET state = 'Applied', payload = NULL, waiting_ref = NULL, \ + waiting_detail = NULL, last_error = NULL, applied_at = ?, updated_at = ? \ + WHERE event_id = ? AND state = 'Pending'", + ) + .bind(now) + .bind(now) + .bind(event_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// Park the event until `next_at` (or an earlier wake on `waiting_ref`). + /// The payload stays so the apply can be repeated. + pub async fn mark_engine_inbox_waiting( + &mut self, + event_id: &str, + waiting_ref: &str, + waiting_detail: &str, + next_at: i64, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_inbox SET waiting_ref = ?, waiting_detail = ?, next_at = ?, \ + updated_at = ? \ + WHERE event_id = ? AND state = 'Pending'", + ) + .bind(waiting_ref) + .bind(waiting_detail) + .bind(next_at) + .bind(now) + .bind(event_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// Transient failure: stay `Pending`, retry at `next_at`. + pub async fn mark_engine_inbox_retry( + &mut self, + event_id: &str, + error: &str, + next_at: i64, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_inbox SET attempts = attempts + 1, next_at = ?, last_error = ?, \ + waiting_ref = NULL, waiting_detail = NULL, updated_at = ? \ + WHERE event_id = ? AND state = 'Pending'", + ) + .bind(next_at) + .bind(error) + .bind(now) + .bind(event_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// Permanent business rejection. The payload is kept for inspection and a + /// duplicate arrival re-arms the row. + pub async fn mark_engine_inbox_failed( + &mut self, + event_id: &str, + reason: &str, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_inbox SET state = 'Failed', attempts = attempts + 1, last_error = ?, \ + waiting_ref = NULL, waiting_detail = NULL, updated_at = ? \ + WHERE event_id = ? AND state = 'Pending'", + ) + .bind(reason) + .bind(now) + .bind(event_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + pub async fn mark_engine_inbox_poisoned( + &mut self, + event_id: &str, + reason: &str, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_inbox SET state = 'Poisoned', attempts = attempts + 1, last_error = ?, \ + waiting_ref = NULL, waiting_detail = NULL, updated_at = ? \ + WHERE event_id = ? AND state = 'Pending'", + ) + .bind(reason) + .bind(now) + .bind(event_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// Make every event waiting on one of `refs` runnable now. Reads first so + /// an idle call takes no write lock. + pub async fn wake_engine_inbox_waiting( + &mut self, + refs: &[String], + now: i64, + ) -> anyhow::Result { + let mut woken = 0; + for waiting_ref in refs { + let pending: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM engine_inbox \ + WHERE state = 'Pending' AND waiting_ref = ? AND next_at > ?", + ) + .bind(waiting_ref) + .bind(now) + .fetch_one(self.conn()) + .await?; + if pending == 0 { + continue; + } + woken += sqlx::query( + "UPDATE engine_inbox SET next_at = ?, updated_at = ? \ + WHERE state = 'Pending' AND waiting_ref = ? AND next_at > ?", + ) + .bind(now) + .bind(now) + .bind(waiting_ref) + .bind(now) + .execute(self.conn()) + .await? + .rows_affected(); + } + Ok(woken) + } + + /// Periodic re-check: every waiting event becomes runnable now. + pub async fn rearm_engine_inbox_waiting(&mut self, now: i64) -> anyhow::Result { + let pending: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM engine_inbox \ + WHERE state = 'Pending' AND waiting_ref IS NOT NULL AND next_at > ?", + ) + .bind(now) + .fetch_one(self.conn()) + .await?; + if pending == 0 { + return Ok(0); + } + Ok(sqlx::query( + "UPDATE engine_inbox SET next_at = ?, updated_at = ? \ + WHERE state = 'Pending' AND waiting_ref IS NOT NULL AND next_at > ?", + ) + .bind(now) + .bind(now) + .bind(now) + .execute(self.conn()) + .await? + .rows_affected()) + } + + /// Startup: re-check every pending event regardless of its backoff. + pub async fn rearm_engine_inbox_pending(&mut self, now: i64) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_inbox SET next_at = ?, updated_at = ? \ + WHERE state = 'Pending' AND next_at > ?", + ) + .bind(now) + .bind(now) + .bind(now) + .execute(self.conn()) + .await? + .rows_affected()) + } + + pub async fn list_engine_inbox_events_for_aggregate( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + limit: i64, + ) -> anyhow::Result> { + let rows = sqlx::query(&format!( + "SELECT {ENGINE_INBOX_COLUMNS} FROM engine_inbox \ + WHERE aggregate_kind = ? AND aggregate_id = ? ORDER BY seq DESC LIMIT ?" + )) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(limit) + .fetch_all(self.conn()) + .await?; + rows.iter().map(|row| Ok(engine_inbox_event_from_row(row)?)).collect() + } + + /// Remove terminal inbox rows older than `before` and closed outbox / finished + /// job rows older than `before`. + pub async fn purge_engine_terminal_rows(&mut self, before: i64) -> anyhow::Result { + let inbox = sqlx::query( + "DELETE FROM engine_inbox \ + WHERE state IN ('Applied', 'Failed', 'Poisoned') AND updated_at < ?", + ) + .bind(before) + .execute(self.conn()) + .await? + .rows_affected(); + let outbox = + sqlx::query("DELETE FROM engine_outbox WHERE state = 'Closed' AND updated_at < ?") + .bind(before) + .execute(self.conn()) + .await? + .rows_affected(); + let jobs = sqlx::query( + "DELETE FROM engine_job \ + WHERE state IN ('Done', 'Failed', 'Cancelled', 'Poisoned') AND updated_at < ?", + ) + .bind(before) + .execute(self.conn()) + .await? + .rows_affected(); + Ok(inbox + outbox + jobs) + } + + /// Drop the rejected (`Failed`) network events last updated before + /// `before`: the admission quota should stop charging for them well before + /// the terminal retention. A sender that wants one back sends it again. + pub async fn purge_rejected_network_inbox_rows(&mut self, before: i64) -> anyhow::Result { + Ok(sqlx::query( + "DELETE FROM engine_inbox \ + WHERE source = 'network' AND state = 'Failed' AND updated_at < ?", + ) + .bind(before) + .execute(self.conn()) + .await? + .rows_affected()) + } + + // -------------------------------------------------------------- aggregate + + pub async fn find_engine_aggregate( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + ) -> anyhow::Result> { + let row = sqlx::query(&format!( + "SELECT {ENGINE_AGGREGATE_COLUMNS} FROM engine_aggregate \ + WHERE aggregate_kind = ? AND aggregate_id = ?" + )) + .bind(aggregate_kind) + .bind(aggregate_id) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| engine_aggregate_from_row(&row)).transpose()?) + } + + /// Create the row if needed and apply `update`; bumps `version`. + pub async fn upsert_engine_aggregate( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + update: &EngineAggregateUpdate, + now: i64, + ) -> anyhow::Result { + sqlx::query( + "INSERT OR IGNORE INTO engine_aggregate \ + (aggregate_kind, aggregate_id, status, phase, version, created_at, updated_at) \ + VALUES (?, ?, '', '', 0, ?, ?)", + ) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(now) + .bind(now) + .execute(self.conn()) + .await?; + let (set_state, state) = match &update.state { + Some(state) => (true, state.clone()), + None => (false, None), + }; + sqlx::query( + "UPDATE engine_aggregate SET version = version + 1, updated_at = ?, \ + status = COALESCE(?, status), phase = COALESCE(?, phase), \ + state = CASE WHEN ? THEN ? ELSE state END, \ + last_event_id = COALESCE(?, last_event_id) \ + WHERE aggregate_kind = ? AND aggregate_id = ?", + ) + .bind(now) + .bind(&update.status) + .bind(&update.phase) + .bind(set_state) + .bind(state) + .bind(&update.last_event_id) + .bind(aggregate_kind) + .bind(aggregate_id) + .execute(self.conn()) + .await?; + self.find_engine_aggregate(aggregate_kind, aggregate_id).await?.ok_or_else(|| { + anyhow::anyhow!("engine aggregate {aggregate_kind}:{aggregate_id} vanished") + }) + } + + /// Record a panic of `action` on the aggregate: the action is held back for + /// `backoff_secs(count)` seconds, where `count` is the number of consecutive + /// poisons of this action. The row survives until the action runs cleanly + /// or the aggregate is cleared by hand. + pub async fn mark_engine_aggregate_poisoned( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + action: &str, + reason: &str, + now: i64, + backoff_secs: &(dyn Fn(i64) -> i64 + Sync), + ) -> anyhow::Result { + let row = sqlx::query( + "INSERT INTO engine_aggregate_poison \ + (aggregate_kind, aggregate_id, action, reason, poison_count, poisoned_at, \ + blocked_until, updated_at) \ + VALUES (?, ?, ?, ?, 1, ?, ?, ?) \ + ON CONFLICT(aggregate_kind, aggregate_id, action) DO UPDATE SET \ + reason = excluded.reason, \ + poison_count = engine_aggregate_poison.poison_count + 1, \ + poisoned_at = excluded.poisoned_at, updated_at = excluded.updated_at \ + RETURNING poison_count", + ) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(action) + .bind(reason) + .bind(now) + .bind(now) + .bind(now) + .fetch_one(self.conn()) + .await?; + let poison_count: i64 = row.try_get("poison_count")?; + let blocked_until = now.saturating_add(backoff_secs(poison_count).max(0)); + sqlx::query( + "UPDATE engine_aggregate_poison SET blocked_until = ? \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND action = ?", + ) + .bind(blocked_until) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(action) + .execute(self.conn()) + .await?; + Ok(EnginePoison { + aggregate_kind: aggregate_kind.to_string(), + aggregate_id: aggregate_id.to_string(), + action: action.to_string(), + reason: reason.to_string(), + poison_count, + poisoned_at: now, + blocked_until, + }) + } + + /// `action` ran cleanly: it is no longer poisoned and its count starts over. + pub async fn release_engine_poison( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + action: &str, + ) -> anyhow::Result { + Ok(sqlx::query( + "DELETE FROM engine_aggregate_poison \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND action = ?", + ) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(action) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// A manual clear of one action or of every action of the aggregate. + pub async fn clear_engine_aggregate_poison( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + action: Option<&str>, + ) -> anyhow::Result { + Ok(sqlx::query( + "DELETE FROM engine_aggregate_poison \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND (? IS NULL OR action = ?)", + ) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(action) + .bind(action) + .execute(self.conn()) + .await? + .rows_affected()) + } + + pub async fn list_engine_poisons( + &mut self, + aggregate: Option<(&str, &str)>, + limit: i64, + ) -> anyhow::Result> { + let (aggregate_kind, aggregate_id) = match aggregate { + Some((kind, id)) => (Some(kind), Some(id)), + None => (None, None), + }; + let rows = sqlx::query( + "SELECT aggregate_kind, aggregate_id, action, reason, poison_count, poisoned_at, \ + blocked_until \ + FROM engine_aggregate_poison \ + WHERE (? IS NULL OR aggregate_kind = ?) AND (? IS NULL OR aggregate_id = ?) \ + ORDER BY poisoned_at, aggregate_kind, aggregate_id, action LIMIT ?", + ) + .bind(aggregate_kind) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(aggregate_id) + .bind(limit) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(EnginePoison { + aggregate_kind: row.try_get("aggregate_kind")?, + aggregate_id: row.try_get("aggregate_id")?, + action: row.try_get("action")?, + reason: row.try_get("reason")?, + poison_count: row.try_get("poison_count")?, + poisoned_at: row.try_get("poisoned_at")?, + blocked_until: row.try_get("blocked_until")?, + }) + }) + .collect() + } + + /// Parked events, oldest park first. + pub async fn list_engine_inbox_waiting( + &mut self, + limit: i64, + ) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT event_id, aggregate_kind, aggregate_id, kind, waiting_ref, waiting_detail, \ + next_at, created_at, updated_at \ + FROM engine_inbox WHERE state = 'Pending' AND waiting_ref IS NOT NULL \ + ORDER BY updated_at, seq LIMIT ?", + ) + .bind(limit) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(EngineWaitingEvent { + event_id: row.try_get("event_id")?, + aggregate_kind: row.try_get("aggregate_kind")?, + aggregate_id: row.try_get("aggregate_id")?, + kind: row.try_get("kind")?, + waiting_ref: row.try_get("waiting_ref")?, + waiting_detail: row.try_get("waiting_detail")?, + next_at: row.try_get("next_at")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + }) + }) + .collect() + } + + // ----------------------------------------------------------------- outbox + + /// Insert once; a known `outbox_id` (open or closed) is left untouched so a + /// closed row is never reopened. + pub async fn insert_engine_outbox_message( + &mut self, + message: &NewEngineOutboxMessage, + now: i64, + ) -> anyhow::Result { + let ack_set = match &message.ack_set { + Some(ack_set) => Some(ack_set_json(ack_set)?), + None => None, + }; + Ok(sqlx::query( + "INSERT OR IGNORE INTO engine_outbox \ + (outbox_id, aggregate_kind, aggregate_id, phase_ref, kind, topic, payload, ack_set, \ + state, publish_count, next_at, lease_until, created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'Open', 0, ?, 0, ?, ?)", + ) + .bind(&message.outbox_id) + .bind(&message.aggregate_kind) + .bind(&message.aggregate_id) + .bind(&message.phase_ref) + .bind(&message.kind) + .bind(&message.topic) + .bind(&message.payload) + .bind(ack_set) + .bind(message.first_at) + .bind(now) + .bind(now) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + pub async fn find_engine_outbox_message( + &mut self, + outbox_id: &str, + ) -> anyhow::Result> { + let row = sqlx::query(&format!( + "SELECT {ENGINE_OUTBOX_COLUMNS} FROM engine_outbox WHERE outbox_id = ?" + )) + .bind(outbox_id) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| engine_outbox_message_from_row(&row)).transpose()?) + } + + /// Claim up to `limit` open rows that are due and not leased. Candidates + /// are read first so an idle publisher never takes the write lock. + pub async fn claim_due_engine_outbox_messages( + &mut self, + now: i64, + lease_until: i64, + limit: i64, + ) -> anyhow::Result> { + let candidates = sqlx::query( + "SELECT outbox_id FROM engine_outbox \ + WHERE state = 'Open' AND next_at <= ? AND lease_until <= ? \ + ORDER BY next_at, created_at LIMIT ?", + ) + .bind(now) + .bind(now) + .bind(limit) + .fetch_all(self.conn()) + .await?; + let mut claimed = Vec::with_capacity(candidates.len()); + for candidate in candidates { + let outbox_id: String = candidate.try_get("outbox_id")?; + let row = sqlx::query(&format!( + "UPDATE engine_outbox SET lease_until = ?, updated_at = ? \ + WHERE outbox_id = ? AND state = 'Open' AND next_at <= ? AND lease_until <= ? \ + RETURNING {ENGINE_OUTBOX_COLUMNS}" + )) + .bind(lease_until) + .bind(now) + .bind(&outbox_id) + .bind(now) + .bind(now) + .fetch_optional(self.conn()) + .await?; + if let Some(row) = row { + claimed.push(engine_outbox_message_from_row(&row)?); + } + } + Ok(claimed) + } + + pub async fn record_engine_outbox_publish( + &mut self, + outbox_id: &str, + next_at: i64, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_outbox SET publish_count = publish_count + 1, next_at = ?, \ + lease_until = 0, last_error = NULL, updated_at = ? \ + WHERE outbox_id = ? AND state = 'Open'", + ) + .bind(next_at) + .bind(now) + .bind(outbox_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + pub async fn record_engine_outbox_publish_failure( + &mut self, + outbox_id: &str, + error: &str, + next_at: i64, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_outbox SET next_at = ?, lease_until = 0, last_error = ?, updated_at = ? \ + WHERE outbox_id = ? AND state = 'Open'", + ) + .bind(next_at) + .bind(error) + .bind(now) + .bind(outbox_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + pub async fn close_engine_outbox( + &mut self, + target: &EngineOutboxCloseTarget, + reason: &str, + now: i64, + ) -> anyhow::Result { + // A closed row keeps its identity and ACK bookkeeping, not the payload. + let set = "SET state = 'Closed', payload = X'', close_reason = ?, closed_at = ?, \ + lease_until = 0, updated_at = ?"; + let result = match target { + EngineOutboxCloseTarget::Row(outbox_id) => { + sqlx::query(&format!( + "UPDATE engine_outbox {set} WHERE outbox_id = ? AND state = 'Open'" + )) + .bind(reason) + .bind(now) + .bind(now) + .bind(outbox_id) + .execute(self.conn()) + .await? + } + EngineOutboxCloseTarget::Phase { aggregate_kind, aggregate_id, phase_ref } => { + sqlx::query(&format!( + "UPDATE engine_outbox {set} \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND phase_ref = ? \ + AND state = 'Open'" + )) + .bind(reason) + .bind(now) + .bind(now) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(phase_ref) + .execute(self.conn()) + .await? + } + EngineOutboxCloseTarget::Aggregate { aggregate_kind, aggregate_id } => { + sqlx::query(&format!( + "UPDATE engine_outbox {set} \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND state = 'Open'" + )) + .bind(reason) + .bind(now) + .bind(now) + .bind(aggregate_kind) + .bind(aggregate_id) + .execute(self.conn()) + .await? + } + }; + Ok(result.rows_affected()) + } + + /// Apply an ACK from `peer` for `delivery_id`. Only a peer listed in the + /// row's outstanding set with exactly that delivery id counts. + /// + /// This is a read-modify-write of `ack_set`: run it inside an immediate + /// transaction so concurrent ACKs for one row serialize. On a plain + /// connection two ACKs can each write back the set they read and put an + /// already confirmed peer back, and the row then never closes. + pub async fn acknowledge_engine_outbox( + &mut self, + peer: &str, + delivery_id: &str, + now: i64, + ) -> anyhow::Result { + let candidates = sqlx::query(&format!( + "SELECT {ENGINE_OUTBOX_COLUMNS} FROM engine_outbox \ + WHERE state = 'Open' AND ack_set IS NOT NULL AND instr(ack_set, ?) > 0" + )) + .bind(delivery_id) + .fetch_all(self.conn()) + .await?; + for row in candidates { + let message = engine_outbox_message_from_row(&row)?; + let Some(mut ack_set) = message.ack_set else { continue }; + if ack_set.get(peer).map(String::as_str) != Some(delivery_id) { + continue; + } + ack_set.remove(peer); + let closed = ack_set.is_empty(); + let updated = if closed { + sqlx::query( + "UPDATE engine_outbox SET ack_set = '{}', state = 'Closed', payload = X'', \ + close_reason = 'acked', closed_at = ?, lease_until = 0, updated_at = ? \ + WHERE outbox_id = ? AND state = 'Open'", + ) + .bind(now) + .bind(now) + .bind(&message.outbox_id) + .execute(self.conn()) + .await? + } else { + sqlx::query( + "UPDATE engine_outbox SET ack_set = ?, updated_at = ? \ + WHERE outbox_id = ? AND state = 'Open'", + ) + .bind(ack_set_json(&ack_set)?) + .bind(now) + .bind(&message.outbox_id) + .execute(self.conn()) + .await? + }; + if updated.rows_affected() > 0 { + return Ok(EngineOutboxAck::Matched { outbox_id: message.outbox_id, closed }); + } + } + Ok(EngineOutboxAck::NoMatch) + } + + /// Startup: leases held by a previous process are void. + pub async fn reset_engine_outbox_leases(&mut self) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_outbox SET lease_until = 0 WHERE state = 'Open' AND lease_until > 0", + ) + .execute(self.conn()) + .await? + .rows_affected()) + } + + // ------------------------------------------------------------------- jobs + + /// Insert once; a known job id is left as it is, and the caller learns in + /// which state it was found. + pub async fn insert_engine_job( + &mut self, + job: &NewEngineJob, + now: i64, + ) -> anyhow::Result { + let inserted = sqlx::query( + "INSERT OR IGNORE INTO engine_job \ + (job_id, aggregate_kind, aggregate_id, phase, action, pool, params, state, \ + generation, lease_until, attempts, next_at, deadline_at, cancel_requested, \ + created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, 'Pending', 0, 0, 0, ?, ?, 0, ?, ?)", + ) + .bind(&job.job_id) + .bind(&job.aggregate_kind) + .bind(&job.aggregate_id) + .bind(&job.phase) + .bind(&job.action) + .bind(&job.pool) + .bind(&job.params) + .bind(job.first_at) + .bind(job.deadline_at) + .bind(now) + .bind(now) + .execute(self.conn()) + .await? + .rows_affected() + > 0; + if inserted { + return Ok(EngineJobInsert::Inserted); + } + let state: String = sqlx::query("SELECT state FROM engine_job WHERE job_id = ?") + .bind(&job.job_id) + .fetch_one(self.conn()) + .await? + .try_get("state")?; + let state = state + .parse::() + .map_err(|_| anyhow::anyhow!("unknown engine job state {state:?}"))?; + Ok(EngineJobInsert::Existing(state)) + } + + pub async fn find_engine_job(&mut self, job_id: &str) -> anyhow::Result> { + let row = + sqlx::query(&format!("SELECT {ENGINE_JOB_COLUMNS} FROM engine_job WHERE job_id = ?")) + .bind(job_id) + .fetch_optional(self.conn()) + .await?; + Ok(row.map(|row| engine_job_from_row(&row)).transpose()?) + } + + /// Claim up to `limit` due jobs of `pool`: pending rows whose `next_at` + /// passed, or running rows whose lease lapsed without a cancel request (a + /// cancelled one is finished by `finish_lapsed_cancelled_engine_jobs`). + /// Each claim increments the generation so a late completion from the + /// previous holder is fenced. Jobs whose action is poisoned on their + /// aggregate are skipped while the poison's backoff runs, and so is every + /// job of an aggregate that already has a job running under a live lease, + /// in any pool: the jobs of one aggregate read and write the same rows and + /// run one at a time (the apply is serialized the same way). Candidates are + /// read first so an idle executor never takes the write lock. + pub async fn claim_due_engine_jobs( + &mut self, + pool: &str, + now: i64, + lease_until: i64, + limit: i64, + ) -> anyhow::Result> { + let candidates = sqlx::query( + "SELECT j.job_id FROM engine_job j \ + WHERE j.pool = ? \ + AND ((j.state = 'Pending' AND j.next_at <= ?) \ + OR (j.state = 'Running' AND j.lease_until <= ? AND j.cancel_requested = 0)) \ + AND NOT EXISTS (SELECT 1 FROM engine_aggregate_poison p \ + WHERE p.aggregate_kind = j.aggregate_kind AND p.aggregate_id = j.aggregate_id \ + AND p.action = j.action AND p.blocked_until > ?) \ + AND NOT EXISTS (SELECT 1 FROM engine_job r \ + WHERE r.aggregate_kind = j.aggregate_kind AND r.aggregate_id = j.aggregate_id \ + AND r.job_id <> j.job_id AND r.state = 'Running' AND r.lease_until > ?) \ + ORDER BY j.next_at, j.created_at LIMIT ?", + ) + .bind(pool) + .bind(now) + .bind(now) + .bind(now) + .bind(now) + .bind(limit) + .fetch_all(self.conn()) + .await?; + let mut claimed = Vec::with_capacity(candidates.len()); + for candidate in candidates { + let job_id: String = candidate.try_get("job_id")?; + let row = sqlx::query(&format!( + "UPDATE engine_job SET state = 'Running', generation = generation + 1, \ + lease_until = ?, attempts = attempts + 1, updated_at = ? \ + WHERE job_id = ? \ + AND ((state = 'Pending' AND next_at <= ?) \ + OR (state = 'Running' AND lease_until <= ? AND cancel_requested = 0)) \ + AND NOT EXISTS (SELECT 1 FROM engine_job r \ + WHERE r.aggregate_kind = engine_job.aggregate_kind \ + AND r.aggregate_id = engine_job.aggregate_id \ + AND r.job_id <> engine_job.job_id \ + AND r.state = 'Running' AND r.lease_until > ?) \ + RETURNING {ENGINE_JOB_COLUMNS}" + )) + .bind(lease_until) + .bind(now) + .bind(&job_id) + .bind(now) + .bind(now) + .bind(now) + .fetch_optional(self.conn()) + .await?; + if let Some(row) = row { + claimed.push(engine_job_from_row(&row)?); + } + } + Ok(claimed) + } + + /// Running jobs that were asked to cancel and whose lease lapsed (a crash + /// or restart took their holder away) finish as `Cancelled` instead of being + /// run again; the rows are returned so their results can be reported. + pub async fn finish_lapsed_cancelled_engine_jobs( + &mut self, + now: i64, + ) -> anyhow::Result> { + let rows = sqlx::query(&format!( + "UPDATE engine_job SET state = 'Cancelled', last_error = 'cancelled', \ + lease_until = 0, finished_at = ?, updated_at = ? \ + WHERE state = 'Running' AND lease_until <= ? AND cancel_requested = 1 \ + RETURNING {ENGINE_JOB_COLUMNS}" + )) + .bind(now) + .bind(now) + .bind(now) + .fetch_all(self.conn()) + .await?; + rows.iter().map(|row| Ok(engine_job_from_row(row)?)).collect() + } + + /// Extend the lease; false when the caller no longer holds the job. + pub async fn renew_engine_job_lease( + &mut self, + job_id: &str, + generation: i64, + lease_until: i64, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_job SET lease_until = ?, updated_at = ? \ + WHERE job_id = ? AND generation = ? AND state = 'Running'", + ) + .bind(lease_until) + .bind(now) + .bind(job_id) + .bind(generation) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// Finish a claimed job. Only the current generation may write; a stale + /// holder gets `Stale` and must not touch the row. + pub async fn finish_engine_job( + &mut self, + job_id: &str, + generation: i64, + finish: &EngineJobFinish, + now: i64, + ) -> anyhow::Result { + let Some(current) = self.find_engine_job(job_id).await? else { + return Ok(EngineJobFinishOutcome::Missing); + }; + if current.generation != generation || current.state != EngineJobState::Running.to_string() + { + return Ok(EngineJobFinishOutcome::Stale { + current_generation: current.generation, + state: current.state, + }); + } + let (state, result, error, next_at): (EngineJobState, Option<&[u8]>, Option<&str>, i64) = + match finish { + EngineJobFinish::Done(result) if current.cancel_requested => { + (EngineJobState::Cancelled, Some(result.as_slice()), None, 0) + } + EngineJobFinish::Done(result) => { + (EngineJobState::Done, Some(result.as_slice()), None, 0) + } + EngineJobFinish::Retry { .. } if current.cancel_requested => { + (EngineJobState::Cancelled, None, Some("cancelled"), 0) + } + EngineJobFinish::Retry { error, next_at } => { + (EngineJobState::Pending, None, Some(error.as_str()), *next_at) + } + EngineJobFinish::Failed(error) => { + (EngineJobState::Failed, None, Some(error.as_str()), 0) + } + EngineJobFinish::Poisoned(error) => { + (EngineJobState::Poisoned, None, Some(error.as_str()), 0) + } + }; + let finished_at = match state { + EngineJobState::Pending | EngineJobState::Running => None, + _ => Some(now), + }; + let updated = sqlx::query( + "UPDATE engine_job SET state = ?, result = ?, last_error = ?, next_at = ?, \ + lease_until = 0, finished_at = ?, updated_at = ? \ + WHERE job_id = ? AND generation = ? AND state = 'Running'", + ) + .bind(state.to_string()) + .bind(result) + .bind(error) + .bind(next_at) + .bind(finished_at) + .bind(now) + .bind(job_id) + .bind(generation) + .execute(self.conn()) + .await?; + if updated.rows_affected() == 0 { + return Ok(EngineJobFinishOutcome::Stale { + current_generation: current.generation, + state: current.state, + }); + } + Ok(EngineJobFinishOutcome::Applied { state }) + } + + /// Cancel a job. A pending job is cancelled immediately; a running one is + /// flagged and finishes as `Cancelled` when its holder reports. + pub async fn cancel_engine_job(&mut self, job_id: &str, now: i64) -> anyhow::Result { + let cancelled = sqlx::query( + "UPDATE engine_job SET state = 'Cancelled', finished_at = ?, updated_at = ? \ + WHERE job_id = ? AND state = 'Pending'", + ) + .bind(now) + .bind(now) + .bind(job_id) + .execute(self.conn()) + .await? + .rows_affected(); + if cancelled > 0 { + return Ok(true); + } + Ok(sqlx::query( + "UPDATE engine_job SET cancel_requested = 1, updated_at = ? \ + WHERE job_id = ? AND state = 'Running'", + ) + .bind(now) + .bind(job_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + /// Cancel every open job of an aggregate (its instance failed or the graph + /// became unusable). + pub async fn cancel_engine_jobs_for_aggregate( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + now: i64, + ) -> anyhow::Result { + let pending = sqlx::query( + "UPDATE engine_job SET state = 'Cancelled', finished_at = ?, updated_at = ? \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND state = 'Pending'", + ) + .bind(now) + .bind(now) + .bind(aggregate_kind) + .bind(aggregate_id) + .execute(self.conn()) + .await? + .rows_affected(); + let running = sqlx::query( + "UPDATE engine_job SET cancel_requested = 1, updated_at = ? \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND state = 'Running'", + ) + .bind(now) + .bind(aggregate_kind) + .bind(aggregate_id) + .execute(self.conn()) + .await? + .rows_affected(); + Ok(pending + running) + } + + /// Drop a finished job's result and params blobs once the aggregate + /// consumed the result: a terminal job is never run again, and its params + /// may hold a whole network message. The row itself stays for + /// observability until it is purged. + pub async fn clear_engine_job_result( + &mut self, + job_id: &str, + now: i64, + ) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_job SET result = NULL, params = NULL, updated_at = ? \ + WHERE job_id = ? AND state IN ('Done', 'Failed', 'Cancelled', 'Poisoned')", + ) + .bind(now) + .bind(job_id) + .execute(self.conn()) + .await? + .rows_affected() + > 0) + } + + pub async fn cancel_engine_jobs_for_phase( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + phase: &str, + now: i64, + ) -> anyhow::Result { + let rows = sqlx::query( + "SELECT job_id FROM engine_job \ + WHERE aggregate_kind = ? AND aggregate_id = ? AND phase = ? \ + AND state IN ('Pending', 'Running')", + ) + .bind(aggregate_kind) + .bind(aggregate_id) + .bind(phase) + .fetch_all(self.conn()) + .await?; + let mut cancelled = 0; + for row in rows { + let job_id: String = row.try_get("job_id")?; + if self.cancel_engine_job(&job_id, now).await? { + cancelled += 1; + } + } + Ok(cancelled) + } + + pub async fn list_engine_jobs_for_aggregate( + &mut self, + aggregate_kind: &str, + aggregate_id: &str, + ) -> anyhow::Result> { + let rows = sqlx::query(&format!( + "SELECT {ENGINE_JOB_COLUMNS} FROM engine_job \ + WHERE aggregate_kind = ? AND aggregate_id = ? ORDER BY created_at" + )) + .bind(aggregate_kind) + .bind(aggregate_id) + .fetch_all(self.conn()) + .await?; + rows.iter().map(|row| Ok(engine_job_from_row(row)?)).collect() + } + + /// Startup: a job left `Running` by a previous process is claimable at once. + pub async fn reset_engine_job_leases(&mut self) -> anyhow::Result { + Ok(sqlx::query( + "UPDATE engine_job SET lease_until = 0 WHERE state = 'Running' AND lease_until > 0", + ) + .execute(self.conn()) + .await? + .rows_affected()) + } + + // ------------------------------------------------------------------ quota + + /// Queued network events (`Pending`, waiting, `Failed` or `Poisoned`: the + /// rows that still hold a payload) per admission class. + pub async fn engine_inbox_class_totals(&mut self) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT admission_class, COUNT(*) AS total_rows, \ + COALESCE(SUM(payload_size), 0) AS total_bytes \ + FROM engine_inbox \ + WHERE source = 'network' AND state IN ('Pending', 'Failed', 'Poisoned') \ + GROUP BY admission_class", + ) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(InboxClassUsage { + admission_class: row.try_get("admission_class")?, + rows: row.try_get("total_rows")?, + bytes: row.try_get("total_bytes")?, + ..Default::default() + }) + }) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// One author's queued network events per class; `rows` and `bytes` + /// stay zero. + pub async fn engine_inbox_peer_usage( + &mut self, + author: &str, + ) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT admission_class, COUNT(*) AS peer_rows, \ + COALESCE(SUM(payload_size), 0) AS peer_bytes \ + FROM engine_inbox \ + WHERE source = 'network' AND state IN ('Pending', 'Failed', 'Poisoned') AND author = ? \ + GROUP BY admission_class", + ) + .bind(author) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(InboxClassUsage { + admission_class: row.try_get("admission_class")?, + peer_rows: row.try_get("peer_rows")?, + peer_bytes: row.try_get("peer_bytes")?, + ..Default::default() + }) + }) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + // ---------------------------------------------------------------- metrics + + pub async fn engine_state_counts(&mut self) -> anyhow::Result> { + let rows = sqlx::query( + "SELECT 'inbox' AS tbl, state, COUNT(*) AS count FROM engine_inbox GROUP BY state \ + UNION ALL \ + SELECT 'inbox_waiting' AS tbl, 'Waiting' AS state, COUNT(*) AS count FROM engine_inbox \ + WHERE state = 'Pending' AND waiting_ref IS NOT NULL \ + UNION ALL \ + SELECT 'outbox' AS tbl, state, COUNT(*) AS count FROM engine_outbox GROUP BY state \ + UNION ALL \ + SELECT 'job' AS tbl, state, COUNT(*) AS count FROM engine_job GROUP BY state \ + UNION ALL \ + SELECT 'aggregate' AS tbl, 'Poisoned' AS state, \ + COUNT(DISTINCT aggregate_kind || ':' || aggregate_id) AS count \ + FROM engine_aggregate_poison \ + UNION ALL \ + SELECT 'aggregate' AS tbl, 'Waiting' AS state, \ + COUNT(DISTINCT aggregate_kind || ':' || aggregate_id) AS count \ + FROM engine_inbox WHERE state = 'Pending' AND waiting_ref IS NOT NULL", + ) + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| { + Ok(EngineStateCount { + table: row.try_get("tbl")?, + state: row.try_get("state")?, + count: row.try_get("count")?, + }) + }) + .collect() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/store/src/engine/tests.rs b/crates/store/src/engine/tests.rs new file mode 100644 index 000000000..56f3735f4 --- /dev/null +++ b/crates/store/src/engine/tests.rs @@ -0,0 +1,582 @@ +use super::*; +use crate::localdb::{LocalDB, create_local_db}; + +async fn setup_db() -> LocalDB { + create_local_db("sqlite::memory:").await +} + +fn event(id: &str, aggregate_id: &str) -> NewEngineInboxEvent { + NewEngineInboxEvent { + event_id: id.to_string(), + aggregate_kind: "test".to_string(), + aggregate_id: aggregate_id.to_string(), + kind: "ping".to_string(), + source: "local".to_string(), + author: String::new(), + admission_class: String::new(), + payload: b"payload".to_vec(), + ack: Some(EngineAckTarget { peer: "peer-a".to_string(), delivery_id: "d1".to_string() }), + } +} + +fn network_event(id: &str, author: &str, class: &str, bytes: usize) -> NewEngineInboxEvent { + NewEngineInboxEvent { + event_id: id.to_string(), + aggregate_kind: "graph".to_string(), + aggregate_id: "g".to_string(), + kind: "CreateGraph".to_string(), + source: "network".to_string(), + author: author.to_string(), + admission_class: class.to_string(), + payload: vec![0; bytes], + ack: None, + } +} + +/// The quota sees what an author currently makes the node store: rows +/// that still hold a payload, by class, with the author's own share. +#[tokio::test] +async fn inbox_usage_counts_queued_network_rows_per_class_and_author() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for (id, author, class, bytes) in [ + ("a1", "alice", "Unregistered", 10), + ("a2", "alice", "Registered", 20), + ("b1", "bob", "Unregistered", 40), + ("c1", "carol", "Committee", 80), + ] { + s.insert_engine_inbox_event(&network_event(id, author, class, bytes), 1).await.unwrap(); + } + // A local event and an applied network event hold no payload. + s.insert_engine_inbox_event(&event("local", "x"), 1).await.unwrap(); + s.insert_engine_inbox_event(&network_event("done", "bob", "Unregistered", 160), 1) + .await + .unwrap(); + assert!(s.mark_engine_inbox_applied("done", 2).await.unwrap()); + // Waiting and failed rows keep theirs. + assert!(s.mark_engine_inbox_waiting("b1", "dep", "x", 100, 2).await.unwrap()); + assert!(s.mark_engine_inbox_failed("c1", "bad", 2).await.unwrap()); + + let mut totals = s.engine_inbox_class_totals().await.unwrap(); + totals.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); + let total = |class: &str| -> (i64, i64) { + totals + .iter() + .find(|usage| usage.admission_class == class) + .map(|usage| (usage.rows, usage.bytes)) + .unwrap_or((0, 0)) + }; + assert_eq!(total("Unregistered"), (2, 50)); + assert_eq!(total("Registered"), (1, 20)); + assert_eq!(total("Committee"), (1, 80)); + assert!(totals.iter().all(|usage| usage.peer_rows == 0 && usage.peer_bytes == 0)); + + let alice = s.engine_inbox_peer_usage("alice").await.unwrap(); + let share = |class: &str| -> (i64, i64) { + alice + .iter() + .find(|usage| usage.admission_class == class) + .map(|usage| (usage.peer_rows, usage.peer_bytes)) + .unwrap_or((0, 0)) + }; + assert_eq!(share("Unregistered"), (1, 10)); + assert_eq!(share("Registered"), (1, 20)); + assert!(s.engine_inbox_peer_usage("nobody").await.unwrap().is_empty()); +} + +#[test] +fn shard_key_is_stable_and_non_negative() { + assert_eq!(engine_shard_key("graph", "abc"), engine_shard_key("graph", "abc")); + assert_ne!(engine_shard_key("graph", "abc"), engine_shard_key("instance", "abc")); + assert!(engine_shard_key("graph", "abc") >= 0); + assert!(engine_shard_key("", "") >= 0); +} + +#[tokio::test] +async fn inbox_dedup_rearms_failed_and_acks_applied_or_waiting() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let e = event("e1", "a"); + assert!(matches!( + s.insert_engine_inbox_event(&e, 10).await.unwrap(), + EngineInboxInsert::Inserted { .. } + )); + // Pending and not yet applied: ignored, no ACK. + assert_eq!( + s.insert_engine_inbox_event(&e, 11).await.unwrap(), + EngineInboxInsert::Duplicate { + state: EngineInboxState::Pending, + ack_again: false, + rearmed: false + } + ); + // Waiting: parked with material kept -> ACK again. + assert!(s.mark_engine_inbox_waiting("e1", "dep:x", "need x", 1_000, 12).await.unwrap()); + assert_eq!( + s.insert_engine_inbox_event(&e, 13).await.unwrap(), + EngineInboxInsert::Duplicate { + state: EngineInboxState::Pending, + ack_again: true, + rearmed: false + } + ); + let parked = s.find_engine_inbox_event("e1").await.unwrap().unwrap(); + assert_eq!(parked.payload.as_deref(), Some(b"payload".as_slice())); + assert_eq!(parked.next_at, 1_000); + // Failed: re-armed as Pending now, no ACK. + assert!(s.mark_engine_inbox_failed("e1", "bad", 14).await.unwrap()); + assert_eq!( + s.insert_engine_inbox_event(&e, 15).await.unwrap(), + EngineInboxInsert::Duplicate { + state: EngineInboxState::Failed, + ack_again: false, + rearmed: true + } + ); + let rearmed = s.find_engine_inbox_event("e1").await.unwrap().unwrap(); + assert_eq!(rearmed.state, "Pending"); + assert_eq!(rearmed.next_at, 15); + assert!(rearmed.last_error.is_none()); + // Applied: payload cleared, ACK again. + assert!(s.mark_engine_inbox_applied("e1", 16).await.unwrap()); + assert_eq!( + s.insert_engine_inbox_event(&e, 17).await.unwrap(), + EngineInboxInsert::Duplicate { + state: EngineInboxState::Applied, + ack_again: true, + rearmed: false + } + ); + let applied = s.find_engine_inbox_event("e1").await.unwrap().unwrap(); + assert!(applied.payload.is_none()); + assert_eq!(applied.applied_at, Some(16)); + assert_eq!(applied.ack_target().unwrap().peer, "peer-a"); + // Poisoned rows are terminal and never re-armed. + s.insert_engine_inbox_event(&event("e2", "a"), 20).await.unwrap(); + assert!(s.mark_engine_inbox_poisoned("e2", "panic", 21).await.unwrap()); + assert_eq!( + s.insert_engine_inbox_event(&event("e2", "a"), 22).await.unwrap(), + EngineInboxInsert::Duplicate { + state: EngineInboxState::Poisoned, + ack_again: false, + rearmed: false + } + ); +} + +#[tokio::test] +async fn inbox_runnable_selection_respects_shard_order_and_wakeups() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + s.insert_engine_inbox_event(&event("e1", "agg-1"), 10).await.unwrap(); + s.insert_engine_inbox_event(&event("e2", "agg-1"), 11).await.unwrap(); + s.insert_engine_inbox_event(&event("e3", "agg-2"), 12).await.unwrap(); + let shard = engine_shard_key("test", "agg-1") % 7; + let other = engine_shard_key("test", "agg-2") % 7; + // Only this shard's rows, oldest first. + let next = s.next_runnable_engine_inbox_event(7, shard, 100).await.unwrap().unwrap(); + assert_eq!(next.event_id, "e1"); + assert!(s.mark_engine_inbox_applied("e1", 100).await.unwrap()); + let next = s.next_runnable_engine_inbox_event(7, shard, 100).await.unwrap().unwrap(); + assert_eq!(next.event_id, "e2"); + // A waiting row is not runnable until its ref is woken or the time passes. + assert!(s.mark_engine_inbox_waiting("e2", "graph:g0", "need g0", 10_000, 100).await.unwrap()); + assert!( + s.next_runnable_engine_inbox_event(7, shard, 100).await.unwrap().is_none() + || shard == other + ); + assert_eq!(s.wake_engine_inbox_waiting(&["graph:other".to_string()], 101).await.unwrap(), 0); + assert_eq!(s.wake_engine_inbox_waiting(&["graph:g0".to_string()], 101).await.unwrap(), 1); + let woken = s.find_engine_inbox_event("e2").await.unwrap().unwrap(); + assert_eq!(woken.next_at, 101); + assert_eq!(woken.waiting_ref.as_deref(), Some("graph:g0")); + // Periodic re-arm and startup re-arm. + assert!(s.mark_engine_inbox_waiting("e2", "graph:g0", "need g0", 10_000, 102).await.unwrap()); + assert!(s.mark_engine_inbox_retry("e3", "busy", 5_000, 102).await.unwrap()); + assert_eq!(s.rearm_engine_inbox_waiting(103).await.unwrap(), 1); + assert_eq!(s.rearm_engine_inbox_pending(104).await.unwrap(), 1); + let retried = s.find_engine_inbox_event("e3").await.unwrap().unwrap(); + assert_eq!(retried.attempts, 1); + assert_eq!(retried.next_at, 104); +} + +#[tokio::test] +async fn aggregate_upsert_and_poison_per_action() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let update = EngineAggregateUpdate { + status: Some("Presigned".to_string()), + phase: Some("Confirm".to_string()), + state: Some(Some("{\"n\":1}".to_string())), + last_event_id: Some("e1".to_string()), + }; + let aggregate = s.upsert_engine_aggregate("instance", "i1", &update, 10).await.unwrap(); + assert_eq!(aggregate.status, "Presigned"); + assert_eq!(aggregate.phase, "Confirm"); + assert_eq!(aggregate.state.as_deref(), Some("{\"n\":1}")); + assert_eq!(aggregate.last_event_id.as_deref(), Some("e1")); + assert_eq!(aggregate.version, 1); + // Untouched fields keep their value; `Some(None)` clears the state. + let again = EngineAggregateUpdate { phase: Some("Post".to_string()), ..Default::default() }; + let aggregate = s.upsert_engine_aggregate("instance", "i1", &again, 20).await.unwrap(); + assert_eq!((aggregate.status.as_str(), aggregate.phase.as_str()), ("Presigned", "Post")); + assert_eq!(aggregate.state.as_deref(), Some("{\"n\":1}")); + assert_eq!(aggregate.version, 2); + let clear = EngineAggregateUpdate { state: Some(None), ..Default::default() }; + let aggregate = s.upsert_engine_aggregate("instance", "i1", &clear, 30).await.unwrap(); + assert!(aggregate.state.is_none()); + assert_eq!(aggregate.status, "Presigned"); + // Poison is per action: the first poison blocks for backoff(1), the + // second for backoff(2); a clean run of the action releases it. + let backoff = |count: i64| if count <= 1 { 0 } else { 600 }; + let first = s + .mark_engine_aggregate_poisoned("instance", "i1", "sign", "panic", 40, &backoff) + .await + .unwrap(); + assert_eq!((first.poison_count, first.blocked_until), (1, 40)); + let second = s + .mark_engine_aggregate_poisoned("instance", "i1", "sign", "panic again", 41, &backoff) + .await + .unwrap(); + assert_eq!((second.poison_count, second.blocked_until), (2, 641)); + assert_eq!(second.reason, "panic again"); + s.mark_engine_aggregate_poisoned("instance", "i1", "vote", "panic", 42, &backoff) + .await + .unwrap(); + let listed = s.list_engine_poisons(Some(("instance", "i1")), 10).await.unwrap(); + assert_eq!( + listed.iter().map(|p| (p.action.as_str(), p.poison_count)).collect::>(), + vec![("sign", 2), ("vote", 1)] + ); + assert!(s.release_engine_poison("instance", "i1", "vote").await.unwrap()); + assert!(!s.release_engine_poison("instance", "i1", "vote").await.unwrap()); + assert_eq!(s.list_engine_poisons(None, 10).await.unwrap().len(), 1); + // A manual clear removes every action of the aggregate. + s.mark_engine_aggregate_poisoned("instance", "i2", "x", "panic", 60, &backoff).await.unwrap(); + assert_eq!(s.clear_engine_aggregate_poison("instance", "i1", None).await.unwrap(), 1); + assert_eq!(s.clear_engine_aggregate_poison("instance", "i1", None).await.unwrap(), 0); + assert_eq!(s.clear_engine_aggregate_poison("instance", "i2", Some("y")).await.unwrap(), 0); + assert_eq!(s.clear_engine_aggregate_poison("instance", "i2", Some("x")).await.unwrap(), 1); +} + +#[tokio::test] +async fn poisoned_event_kinds_wait_out_their_backoff() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + s.insert_engine_inbox_event(&event("e1", "a"), 10).await.unwrap(); + let backoff = |_: i64| 100; + s.mark_engine_aggregate_poisoned("test", "a", "ping", "panic", 10, &backoff).await.unwrap(); + // The kind is held back while the backoff runs, other kinds and aggregates are not. + assert!(s.next_runnable_engine_inbox_event(1, 0, 50).await.unwrap().is_none()); + let mut other = event("e2", "a"); + other.kind = "pong".to_string(); + s.insert_engine_inbox_event(&other, 11).await.unwrap(); + let next = s.next_runnable_engine_inbox_event(1, 0, 50).await.unwrap().unwrap(); + assert_eq!(next.event_id, "e2"); + s.mark_engine_inbox_applied("e2", 51).await.unwrap(); + assert_eq!( + s.next_runnable_engine_inbox_event(1, 0, 110).await.unwrap().unwrap().event_id, + "e1" + ); + // Parking does not count as an attempt. + assert!(s.mark_engine_inbox_waiting("e1", "x", "y", 200, 111).await.unwrap()); + assert_eq!(s.find_engine_inbox_event("e1").await.unwrap().unwrap().attempts, 0); + let waiting = s.list_engine_inbox_waiting(10).await.unwrap(); + assert_eq!(waiting.len(), 1); + assert_eq!((waiting[0].event_id.as_str(), waiting[0].waiting_ref.as_str()), ("e1", "x")); +} + +fn outbox(id: &str, ack_set: Option>) -> NewEngineOutboxMessage { + NewEngineOutboxMessage { + outbox_id: id.to_string(), + aggregate_kind: "graph".to_string(), + aggregate_id: "g1".to_string(), + phase_ref: "setup".to_string(), + kind: "CreateGraph".to_string(), + topic: "all".to_string(), + payload: vec![1, 2, 3], + ack_set, + first_at: 10, + } +} + +#[tokio::test] +async fn outbox_claim_publish_ack_and_close() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let acks = BTreeMap::from([ + ("peer-a".to_string(), "deliv-a".to_string()), + ("peer-b".to_string(), "deliv-b".to_string()), + ]); + assert!(s.insert_engine_outbox_message(&outbox("o1", Some(acks)), 10).await.unwrap()); + assert!(!s.insert_engine_outbox_message(&outbox("o1", None), 11).await.unwrap()); + assert!(s.insert_engine_outbox_message(&outbox("o2", None), 10).await.unwrap()); + // Claim: both due, leased until 100. + let claimed = s.claim_due_engine_outbox_messages(10, 100, 10).await.unwrap(); + assert_eq!(claimed.len(), 2); + assert!(s.claim_due_engine_outbox_messages(11, 100, 10).await.unwrap().is_empty()); + assert!(s.record_engine_outbox_publish("o1", 40, 12).await.unwrap()); + assert!(s.record_engine_outbox_publish_failure("o2", "no peers", 20, 12).await.unwrap()); + let o1 = s.find_engine_outbox_message("o1").await.unwrap().unwrap(); + assert_eq!((o1.publish_count, o1.next_at, o1.lease_until), (1, 40, 0)); + let o2 = s.find_engine_outbox_message("o2").await.unwrap().unwrap(); + assert_eq!((o2.publish_count, o2.next_at, o2.last_error.as_deref()), (0, 20, Some("no peers"))); + // ACK: wrong peer for a known delivery id does not count. + assert_eq!( + s.acknowledge_engine_outbox("peer-b", "deliv-a", 13).await.unwrap(), + EngineOutboxAck::NoMatch + ); + assert_eq!( + s.acknowledge_engine_outbox("peer-a", "deliv-a", 13).await.unwrap(), + EngineOutboxAck::Matched { outbox_id: "o1".to_string(), closed: false } + ); + assert_eq!( + s.acknowledge_engine_outbox("peer-a", "deliv-a", 14).await.unwrap(), + EngineOutboxAck::NoMatch + ); + assert_eq!( + s.acknowledge_engine_outbox("peer-b", "deliv-b", 15).await.unwrap(), + EngineOutboxAck::Matched { outbox_id: "o1".to_string(), closed: true } + ); + let o1 = s.find_engine_outbox_message("o1").await.unwrap().unwrap(); + assert_eq!((o1.state.as_str(), o1.close_reason.as_deref()), ("Closed", Some("acked"))); + // Closed rows are not re-opened by a repeated insert, and not re-published. + assert!(!s.insert_engine_outbox_message(&outbox("o1", None), 16).await.unwrap()); + assert!(!s.record_engine_outbox_publish("o1", 50, 16).await.unwrap()); + // Phase close covers the small message. + let target = EngineOutboxCloseTarget::Phase { + aggregate_kind: "graph".to_string(), + aggregate_id: "g1".to_string(), + phase_ref: "setup".to_string(), + }; + assert_eq!(s.close_engine_outbox(&target, "evidence", 17).await.unwrap(), 1); + // Aggregate close and row close. + assert!(s.insert_engine_outbox_message(&outbox("o3", None), 18).await.unwrap()); + assert!(s.insert_engine_outbox_message(&outbox("o4", None), 18).await.unwrap()); + assert_eq!( + s.close_engine_outbox(&EngineOutboxCloseTarget::Row("o3".to_string()), "x", 19) + .await + .unwrap(), + 1 + ); + let target = EngineOutboxCloseTarget::Aggregate { + aggregate_kind: "graph".to_string(), + aggregate_id: "g1".to_string(), + }; + assert_eq!(s.close_engine_outbox(&target, "instance_failed", 20).await.unwrap(), 1); + // Startup lease reset. + assert!(s.insert_engine_outbox_message(&outbox("o5", None), 21).await.unwrap()); + assert_eq!(s.claim_due_engine_outbox_messages(21, 500, 10).await.unwrap().len(), 1); + assert_eq!(s.reset_engine_outbox_leases().await.unwrap(), 1); + assert_eq!(s.claim_due_engine_outbox_messages(22, 500, 10).await.unwrap().len(), 1); +} + +fn job(id: &str, action: &str) -> NewEngineJob { + NewEngineJob { + job_id: id.to_string(), + aggregate_kind: "graph".to_string(), + aggregate_id: "g1".to_string(), + phase: "kickoff".to_string(), + action: action.to_string(), + pool: "cpu".to_string(), + params: vec![9], + deadline_at: None, + first_at: 10, + } +} + +#[tokio::test] +async fn jobs_of_one_aggregate_run_one_at_a_time() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let other = NewEngineJob { + job_id: "other".to_string(), + aggregate_id: "g2".to_string(), + pool: "network".to_string(), + ..job("other", "ingest") + }; + let same_pool = job("j2", "ingest"); + let network = NewEngineJob { pool: "network".to_string(), ..job("j3", "fetch") }; + for row in [&job("j1", "sign"), &same_pool, &network, &other] { + assert!(s.insert_engine_job(row, 10).await.unwrap().is_inserted()); + } + // The oldest job of g1 is claimed; its siblings wait in every pool, + // another aggregate's job does not. + let cpu = s.claim_due_engine_jobs("cpu", 10, 100, 10).await.unwrap(); + assert_eq!(cpu.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j1"]); + let net = s.claim_due_engine_jobs("network", 11, 100, 10).await.unwrap(); + assert_eq!(net.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["other"]); + s.finish_engine_job("other", 1, &EngineJobFinish::Done(vec![]), 11).await.unwrap(); + // Done: the next sibling runs. + s.finish_engine_job("j1", 1, &EngineJobFinish::Done(vec![]), 12).await.unwrap(); + let cpu = s.claim_due_engine_jobs("cpu", 13, 100, 10).await.unwrap(); + assert_eq!(cpu.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j2"]); + assert!(s.claim_due_engine_jobs("network", 14, 100, 10).await.unwrap().is_empty()); + // A lapsed lease no longer blocks the siblings (the stale holder is fenced). + let net = s.claim_due_engine_jobs("network", 101, 200, 10).await.unwrap(); + assert_eq!(net.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j3"]); +} + +#[tokio::test] +async fn job_lease_generation_fencing_and_cancel() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + assert!(s.insert_engine_job(&job("j1", "broadcast"), 10).await.unwrap().is_inserted()); + assert_eq!( + s.insert_engine_job(&job("j1", "broadcast"), 11).await.unwrap(), + EngineJobInsert::Existing(EngineJobState::Pending) + ); + // First claim: generation 1, leased until 100. + let first = s.claim_due_engine_jobs("cpu", 10, 100, 10).await.unwrap(); + assert_eq!(first.len(), 1); + assert_eq!( + (first[0].generation, first[0].attempts, first[0].state.as_str()), + (1, 1, "Running") + ); + assert!(s.claim_due_engine_jobs("cpu", 50, 100, 10).await.unwrap().is_empty()); + assert!(s.claim_due_engine_jobs("network", 200, 300, 10).await.unwrap().is_empty()); + // Lease lapses: a second holder claims with generation 2. + let second = s.claim_due_engine_jobs("cpu", 200, 300, 10).await.unwrap(); + assert_eq!(second.len(), 1); + assert_eq!(second[0].generation, 2); + // The late first holder cannot renew or finish. + assert!(!s.renew_engine_job_lease("j1", 1, 400, 201).await.unwrap()); + assert!(s.renew_engine_job_lease("j1", 2, 400, 201).await.unwrap()); + assert_eq!( + s.finish_engine_job("j1", 1, &EngineJobFinish::Done(vec![1]), 202).await.unwrap(), + EngineJobFinishOutcome::Stale { current_generation: 2, state: "Running".to_string() } + ); + let job_row = s.find_engine_job("j1").await.unwrap().unwrap(); + assert!(job_row.result.is_none()); + assert_eq!(job_row.state, "Running"); + // The current holder's transient failure re-queues, then it finishes. + assert_eq!( + s.finish_engine_job( + "j1", + 2, + &EngineJobFinish::Retry { error: "rpc".to_string(), next_at: 500 }, + 203 + ) + .await + .unwrap(), + EngineJobFinishOutcome::Applied { state: EngineJobState::Pending } + ); + assert!(s.claim_due_engine_jobs("cpu", 499, 600, 10).await.unwrap().is_empty()); + let third = s.claim_due_engine_jobs("cpu", 500, 600, 10).await.unwrap(); + assert_eq!(third[0].generation, 3); + assert_eq!( + s.finish_engine_job("j1", 3, &EngineJobFinish::Done(vec![7]), 501).await.unwrap(), + EngineJobFinishOutcome::Applied { state: EngineJobState::Done } + ); + let done = s.find_engine_job("j1").await.unwrap().unwrap(); + assert_eq!( + (done.state.as_str(), done.result.as_deref(), done.finished_at), + ("Done", Some([7].as_slice()), Some(501)) + ); + assert_eq!( + s.finish_engine_job("j1", 3, &EngineJobFinish::Done(vec![8]), 502).await.unwrap(), + EngineJobFinishOutcome::Stale { current_generation: 3, state: "Done".to_string() } + ); + assert_eq!( + s.finish_engine_job("missing", 1, &EngineJobFinish::Done(vec![]), 503).await.unwrap(), + EngineJobFinishOutcome::Missing + ); + // Cancel: pending is cancelled at once, running finishes as Cancelled. + assert!(s.insert_engine_job(&job("j2", "broadcast"), 600).await.unwrap().is_inserted()); + assert!(s.cancel_engine_job("j2", 601).await.unwrap()); + assert_eq!(s.find_engine_job("j2").await.unwrap().unwrap().state, "Cancelled"); + assert!(s.insert_engine_job(&job("j3", "broadcast"), 600).await.unwrap().is_inserted()); + let running = s.claim_due_engine_jobs("cpu", 600, 700, 10).await.unwrap(); + assert_eq!(running[0].job_id, "j3"); + assert!(s.cancel_engine_job("j3", 602).await.unwrap()); + assert_eq!( + s.finish_engine_job("j3", 1, &EngineJobFinish::Done(vec![1]), 603).await.unwrap(), + EngineJobFinishOutcome::Applied { state: EngineJobState::Cancelled } + ); + // Phase cancel and poison isolation. + assert!(s.insert_engine_job(&job("j4", "sign"), 700).await.unwrap().is_inserted()); + assert!(s.insert_engine_job(&job("j5", "broadcast"), 700).await.unwrap().is_inserted()); + let backoff = |_: i64| 100; + s.mark_engine_aggregate_poisoned("graph", "g1", "sign", "panic", 701, &backoff).await.unwrap(); + let claimable = s.claim_due_engine_jobs("cpu", 701, 800, 10).await.unwrap(); + assert_eq!(claimable.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j5"]); + assert_eq!(s.clear_engine_aggregate_poison("graph", "g1", None).await.unwrap(), 1); + assert_eq!(s.cancel_engine_jobs_for_phase("graph", "g1", "kickoff", 703).await.unwrap(), 2); + assert_eq!(s.find_engine_job("j4").await.unwrap().unwrap().state, "Cancelled"); + assert!(s.find_engine_job("j5").await.unwrap().unwrap().cancel_requested); + assert!(s.insert_engine_job(&job("j6", "broadcast"), 800).await.unwrap().is_inserted()); + // j5 is still running (leased until 800): its sibling j6 waits. + assert!(s.claim_due_engine_jobs("cpu", 799, 900, 10).await.unwrap().is_empty()); + // Startup lease reset: the cancelled running job is not run again but + // finished as Cancelled (the executor sweeps before it claims), then j6 runs. + assert_eq!(s.reset_engine_job_leases().await.unwrap(), 1); + let swept = s.finish_lapsed_cancelled_engine_jobs(801).await.unwrap(); + assert_eq!(swept.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j5"]); + assert_eq!(swept[0].state, "Cancelled"); + assert!(s.finish_lapsed_cancelled_engine_jobs(802).await.unwrap().is_empty()); + let claimed = s.claim_due_engine_jobs("cpu", 802, 900, 10).await.unwrap(); + assert_eq!(claimed.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j6"]); + // A poisoned action is claimable again once its backoff passed. + assert!(s.insert_engine_job(&job("j7", "sign"), 900).await.unwrap().is_inserted()); + s.finish_engine_job("j6", 1, &EngineJobFinish::Done(vec![]), 900).await.unwrap(); + s.mark_engine_aggregate_poisoned("graph", "g1", "sign", "panic", 900, &backoff).await.unwrap(); + assert!(s.claim_due_engine_jobs("cpu", 950, 1000, 10).await.unwrap().is_empty()); + let claimed = s.claim_due_engine_jobs("cpu", 1000, 1100, 10).await.unwrap(); + assert_eq!(claimed.iter().map(|j| j.job_id.as_str()).collect::>(), vec!["j7"]); +} + +#[tokio::test] +async fn state_counts_and_purge() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + s.insert_engine_inbox_event(&event("e1", "a"), 10).await.unwrap(); + s.insert_engine_inbox_event(&event("e2", "a"), 10).await.unwrap(); + s.mark_engine_inbox_applied("e1", 11).await.unwrap(); + s.mark_engine_inbox_waiting("e2", "x", "y", 100, 11).await.unwrap(); + s.insert_engine_outbox_message(&outbox("o1", None), 10).await.unwrap(); + s.insert_engine_job(&job("j1", "a"), 10).await.unwrap(); + let counts = s.engine_state_counts().await.unwrap(); + let get = |table: &str, state: &str| { + counts.iter().find(|c| c.table == table && c.state == state).map(|c| c.count).unwrap_or(0) + }; + assert_eq!(get("inbox", "Applied"), 1); + assert_eq!(get("inbox", "Pending"), 1); + assert_eq!(get("inbox_waiting", "Waiting"), 1); + assert_eq!(get("outbox", "Open"), 1); + assert_eq!(get("job", "Pending"), 1); + // Purge removes only terminal rows older than the cutoff. + assert_eq!(s.purge_engine_terminal_rows(11).await.unwrap(), 0); + assert_eq!(s.purge_engine_terminal_rows(12).await.unwrap(), 1); + assert!(s.find_engine_inbox_event("e1").await.unwrap().is_none()); + assert!(s.find_engine_inbox_event("e2").await.unwrap().is_some()); + assert_eq!(s.list_engine_inbox_events_for_aggregate("test", "a", 10).await.unwrap().len(), 1); +} + +/// Rejected network events stop counting against the quota long before +/// the terminal retention; everything else keeps its row. +#[tokio::test] +async fn rejected_network_rows_are_purged_early() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + for id in ["rejected", "parked", "fresh-rejected", "pending"] { + s.insert_engine_inbox_event(&network_event(id, "alice", "Registered", 8), 100) + .await + .unwrap(); + } + // A rejected local event is never the quota's business. + s.insert_engine_inbox_event(&event("local", "x"), 100).await.unwrap(); + s.mark_engine_inbox_failed("local", "no", 150).await.unwrap(); + s.mark_engine_inbox_failed("rejected", "no", 150).await.unwrap(); + s.mark_engine_inbox_failed("fresh-rejected", "no", 950).await.unwrap(); + s.mark_engine_inbox_waiting("parked", "slot", "d", 1_000, 150).await.unwrap(); + assert_eq!(s.purge_rejected_network_inbox_rows(900).await.unwrap(), 1); + for (id, kept) in [ + ("rejected", false), + ("parked", true), + ("fresh-rejected", true), + ("pending", true), + ("local", true), + ] { + assert_eq!(s.find_engine_inbox_event(id).await.unwrap().is_some(), kept, "{id}"); + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ceb6d9fc8..9386b572b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -1,6 +1,16 @@ +pub mod engine; pub mod localdb; mod schema; +pub mod signing; mod utils; -pub use localdb::create_local_db; +pub use engine::*; +pub use localdb::{DEFAULT_BUSY_TIMEOUT, create_local_db, create_local_db_with_busy_timeout}; pub use schema::*; +pub use signing::SigningAuthorization; + +/// Whether a `sqlx` error sits anywhere in `error`'s chain: the database did +/// not answer, which callers retry rather than treat as a business failure. +pub fn is_storage_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| cause.downcast_ref::().is_some()) +} diff --git a/crates/store/src/localdb.rs b/crates/store/src/localdb.rs deleted file mode 100644 index b6dca997f..000000000 --- a/crates/store/src/localdb.rs +++ /dev/null @@ -1,7021 +0,0 @@ -use crate::utils::{QueryBuilder, QueryParam, create_place_holders}; -use crate::{ - BridgeOutGlobalStats, EventWatchMetricsSnapshot, GoatTxRecord, Graph, GraphBtcTxVoutMonitor, - GraphRawData, GraphStatus, GraphStatusSource, GraphStatusTransitionOutcome, Instance, - LongRunningTaskProof, Message, MessageDebugOverview, MessageDebugReason, MetricsStateCount, - Node, NodeAlertMetricsSnapshot, NodesOverview, OnDemandProofCompletion, OnDemandProofTable, - OperatorProof, P2pInboxAdmissionClass, P2pInboxClassUsage, P2pInboxMessage, P2pOutboxMessage, - PeginGraphProcessData, PeginInstanceProcessData, PendingGraphInit, SequencerSetHashChange, - SequencerSetScanState, SerializableTxid, SwapEscrow, SwapEscrowStatus, WatchContract, - WatchtowerProof, -}; - -use indexmap::IndexMap; -use sqlx::migrate::Migrator; -use sqlx::pool::PoolConnection; -use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqliteRow}; -use sqlx::types::Uuid; -use sqlx::{Row, Sqlite, SqliteConnection, SqlitePool, Transaction, migrate::MigrateDatabase}; -use std::str::FromStr; -use std::time::{SystemTime, UNIX_EPOCH}; - -fn get_current_timestamp_secs() -> i64 { - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 -} - -/// Columns every `message` SELECT must fetch. -const MESSAGE_COLUMNS: &str = "message_id, business_id, from_peer, actor, msg_type, content, \ - message_version, state, weight, lock_time_until, attempt_count, abandon_count, last_error, \ - created_at"; - -/// Columns every `p2p_inbox` SELECT must fetch. -const P2P_INBOX_COLUMNS: &str = "message_id, business_id, actor, from_peer, msg_type, content, \ - content_size, state, attempt_count, abandon_count, next_retry_at, lease_until, lease_token, \ - last_error, admission_class, content_hash, created_at, updated_at"; - -fn message_from_row(row: &SqliteRow) -> Result { - Ok(Message { - message_id: row.try_get("message_id")?, - business_id: row.try_get("business_id")?, - actor: row.try_get("actor")?, - from_peer: row.try_get("from_peer")?, - msg_type: row.try_get("msg_type")?, - content: row.try_get("content")?, - state: row.try_get("state")?, - message_version: row.try_get("message_version")?, - weight: row.try_get("weight")?, - lock_time_until: row.try_get("lock_time_until")?, - attempt_count: row.try_get("attempt_count")?, - abandon_count: row.try_get("abandon_count")?, - last_error: row.try_get("last_error")?, - created_at: row.try_get("created_at")?, - }) -} - -fn p2p_inbox_message_from_row(row: &SqliteRow) -> Result { - Ok(P2pInboxMessage { - message_id: row.try_get("message_id")?, - business_id: row.try_get("business_id")?, - actor: row.try_get("actor")?, - from_peer: row.try_get("from_peer")?, - msg_type: row.try_get("msg_type")?, - content: row.try_get("content")?, - content_size: row.try_get("content_size")?, - state: row.try_get("state")?, - attempt_count: row.try_get("attempt_count")?, - abandon_count: row.try_get("abandon_count")?, - next_retry_at: row.try_get("next_retry_at")?, - lease_until: row.try_get("lease_until")?, - lease_token: row.try_get("lease_token")?, - last_error: row.try_get("last_error")?, - admission_class: row.try_get("admission_class")?, - content_hash: row.try_get("content_hash")?, - created_at: row.try_get("created_at")?, - updated_at: row.try_get("updated_at")?, - }) -} - -fn p2p_outbox_message_from_row(row: &SqliteRow) -> Result { - Ok(P2pOutboxMessage { - message_id: row.try_get("message_id")?, - msg_type: row.try_get("msg_type")?, - content: row.try_get("content")?, - state: row.try_get("state")?, - attempt_count: row.try_get("attempt_count")?, - next_retry_at: row.try_get("next_retry_at")?, - lease_until: row.try_get("lease_until")?, - last_error: row.try_get("last_error")?, - retry_until: row.try_get("retry_until")?, - retry_interval_secs: row.try_get("retry_interval_secs")?, - ack_peer_id: row.try_get("ack_peer_id")?, - publish_count: row.try_get("publish_count")?, - created_at: row.try_get("created_at")?, - }) -} - -#[derive(Clone, Debug)] -pub struct LocalDB { - pub path: String, - pub is_mem: bool, - pub conn: SqlitePool, -} - -#[derive(Debug)] -pub enum ConnectionHolder<'a> { - Pooled(PoolConnection), - Direct(SqliteConnection), - Transaction(Transaction<'a, Sqlite>), -} - -#[derive(Debug)] -pub struct StorageProcessor<'a> { - pub conn: ConnectionHolder<'a>, - pub in_transaction: bool, -} - -#[derive(Clone, Debug, Default)] -pub struct MessageQueueStats { - pub pending_ready: i64, - pub pending_locked: i64, - pub failed: i64, - pub oldest_pending_at: Option, -} - -static MIGRATOR: Migrator = sqlx::migrate!("./migrations"); -impl LocalDB { - pub async fn new(path: &str, is_mem: bool) -> LocalDB { - if !Sqlite::database_exists(path).await.unwrap_or(false) { - tracing::info!("Creating database {}", path); - match Sqlite::create_database(path).await { - Ok(_) => tracing::info!( - event = "database_lifecycle", - outcome = "created", - "local database created" - ), - Err(error) => panic!("error: {error}"), - } - } else { - tracing::info!("Database already exists"); - } - - let mut options = SqliteConnectOptions::from_str(path).unwrap().create_if_missing(true); - if !is_mem { - // File-backed nodes run event watchers, P2P handlers, and maintenance tasks - // concurrently. WAL allows their readers to proceed while a short write commits. - options = options.journal_mode(SqliteJournalMode::Wal); - } - let conn = SqlitePool::connect_with(options).await.unwrap(); - Self { path: path.to_string(), is_mem, conn } - } - - pub async fn migrate(&self) { - match MIGRATOR.run(&self.conn).await { - Ok(_) => tracing::info!("Migration success"), - Err(error) => { - panic!("error: {error:?}"); - } - } - } - - pub async fn acquire<'a>(&self) -> anyhow::Result> { - Ok(StorageProcessor { - conn: ConnectionHolder::Pooled(self.conn.acquire().await?), - in_transaction: false, - }) - } - pub async fn start_transaction<'a>(&self) -> anyhow::Result> { - Ok(StorageProcessor { - conn: ConnectionHolder::Transaction(self.conn.begin().await?), - in_transaction: true, - }) - } - - /// Start a short write transaction before reading state that will be - /// immediately reconciled. This prevents a stale snapshot from dropping - /// a concurrent update between the read and the conditional write. - pub async fn start_immediate_transaction<'a>(&self) -> anyhow::Result> { - Ok(StorageProcessor { - conn: ConnectionHolder::Transaction(self.conn.begin_with("BEGIN IMMEDIATE").await?), - in_transaction: true, - }) - } -} - -#[derive(Clone, Debug, Default)] -pub struct InstanceQuery { - pub from_addr: Option, - pub statuses: Vec, - pub earliest_updated: Option, - pub pegin_request_height_threshold: Option, - pub order: Option, - pub raw_conditions: Vec, - pub offset: Option, - pub limit: Option, -} - -impl InstanceQuery { - pub fn with_from_addr(mut self, from_addr: String) -> Self { - self.from_addr = Some(from_addr); - self - } - - pub fn with_status(mut self, status: String) -> Self { - self.statuses.push(status); - self - } - - pub fn with_statuses(mut self, statuses: Vec) -> Self { - self.statuses = statuses; - self - } - - pub fn with_order(mut self, order: String) -> Self { - self.order = Some(order); - self - } - - pub fn with_earliest_updated(mut self, earliest_updated: i64) -> Self { - self.earliest_updated = Some(earliest_updated); - self - } - - pub fn with_pegin_request_height_threshold(mut self, threshold: i64) -> Self { - self.pegin_request_height_threshold = Some(threshold); - self - } - - pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { - self.offset = Some(offset); - self.limit = Some(limit); - self - } - - pub fn with_offset(mut self, offset: u32) -> Self { - self.offset = Some(offset); - self - } - - pub fn with_limit(mut self, limit: u32) -> Self { - self.limit = Some(limit); - self - } - - pub fn with_raw_condition(mut self, raw_condition: String) -> Self { - self.raw_conditions.push(raw_condition); - self - } - - pub fn with_raw_conditions(mut self, raw_conditions: Vec) -> Self { - self.raw_conditions = raw_conditions; - self - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::new(base_sql); - if let Some(from_addr) = &self.from_addr { - query_builder.and_where("from_addr = ?", Some(QueryParam::Text(from_addr.clone()))); - } - - if !self.statuses.is_empty() { - query_builder.and_where_in("status", &self.statuses, false); - } - if let Some(earliest_updated) = self.earliest_updated { - query_builder.and_where("updated_at >= ?", Some(QueryParam::Int(earliest_updated))); - } - if let Some(pegin_request_height_threshold) = self.pegin_request_height_threshold { - query_builder.and_where( - "goat_tx_height < ?", - Some(QueryParam::Int(pegin_request_height_threshold)), - ); - } - for raw_condition in &self.raw_conditions { - query_builder.add_raw_condition(raw_condition); - } - - if let Some(order) = &self.order { - query_builder.apply_order(order); - } - query_builder.apply_pagination(self.limit, self.offset); - query_builder - } -} - -/// Instance field update parameters -/// -/// Provides a more elegant way to specify which instance fields to update -#[derive(Debug, Clone)] -pub struct InstanceUpdate { - pub instance_id: Uuid, - pub from_addr: Option, - pub to_addr: Option, - pub btc_txid: Option, - pub status: Option, - pub pegin_confirm_txid: Option, - pub pegin_cancel_txid: Option, - pub post_pegin_txhash: Option, - pub btc_height: Option, - pub committees_answers: Option>>, - pub only_if_status_in: Option>, -} - -impl InstanceUpdate { - /// Create new update parameters - pub fn new_with_instance_id(instance_id: Uuid) -> Self { - Self { - instance_id, - from_addr: None, - to_addr: None, - btc_txid: None, - status: None, - pegin_confirm_txid: None, - pegin_cancel_txid: None, - post_pegin_txhash: None, - btc_height: None, - committees_answers: None, - only_if_status_in: None, - } - } - - /// Set from_addr - pub fn with_from_addr(mut self, from_addr: String) -> Self { - self.from_addr = Some(from_addr); - self - } - - /// Set to_addr - pub fn with_to_addr(mut self, to_addr: String) -> Self { - self.to_addr = Some(to_addr); - self - } - - /// Set btc txid - pub fn with_btc_txid(mut self, btc_txid: SerializableTxid) -> Self { - self.btc_txid = Some(btc_txid); - self - } - /// Set status - pub fn with_status(mut self, status: String) -> Self { - self.status = Some(status); - self - } - - /// Set pegin confirmation transaction ID. - pub fn with_pegin_confirm_txid(mut self, txid: SerializableTxid) -> Self { - self.pegin_confirm_txid = Some(txid); - self - } - - /// Set pegin cancellation transaction ID. - pub fn with_pegin_cancel_txid(mut self, txid: SerializableTxid) -> Self { - self.pegin_cancel_txid = Some(txid); - self - } - - /// Set post pegin information - pub fn with_post_pegin(mut self, txid: String) -> Self { - self.post_pegin_txhash = Some(txid); - self - } - - /// Set committees answers - pub fn with_committees_answers( - mut self, - committees_answers: IndexMap>, - ) -> Self { - self.committees_answers = Some(committees_answers); - self - } - - /// Set btc height - pub fn with_btc_height(mut self, btc_height: i64) -> Self { - self.btc_height = Some(btc_height); - self - } - - /// Apply this update only while the instance is still in one of the - /// expected states. The condition is folded into the UPDATE statement. - pub fn with_only_if_status_in(mut self, statuses: Vec) -> Self { - self.only_if_status_in = Some(statuses); - self - } - - /// Check if any fields need to be updated - pub fn has_updates(&self) -> bool { - self.from_addr.is_some() - || self.to_addr.is_some() - || self.btc_txid.is_some() - || self.status.is_some() - || self.pegin_confirm_txid.is_some() - || self.pegin_cancel_txid.is_some() - || self.post_pegin_txhash.is_some() - || self.btc_height.is_some() - || self.committees_answers.is_some() - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::update(base_sql); - // Set field - if let Some(ref status) = self.status { - query_builder.set_field("status", QueryParam::Text(status.clone())); - query_builder - .set_field("status_updated_at", QueryParam::Int(get_current_timestamp_secs())); - } - - if let Some(ref txid) = self.pegin_confirm_txid { - query_builder.set_field("pegin_confirm_txid", QueryParam::BTCTxid(txid.clone())); - } - - if let Some(ref txid) = self.pegin_cancel_txid { - query_builder.set_field("pegin_cancel_txid", QueryParam::BTCTxid(txid.clone())); - } - - if let Some(ref txid) = self.post_pegin_txhash { - query_builder.set_field("post_pegin_txhash", QueryParam::Text(txid.clone())); - } - - if let Some(pegin_prepare_height) = self.btc_height { - query_builder.set_field("btc_height", QueryParam::Int(pegin_prepare_height)); - } - - if let Some(ref to_addr) = self.to_addr { - query_builder.set_field("to_addr", QueryParam::Text(to_addr.clone())); - } - - if let Some(ref from_addr) = self.from_addr { - query_builder.set_field("from_addr", QueryParam::Text(from_addr.clone())); - } - - if let Some(ref btc_txid) = self.btc_txid { - query_builder.set_field("btc_txid", QueryParam::BTCTxid(btc_txid.clone())); - } - - if let Some(ref committees_answers) = self.committees_answers { - let committees_answers = serde_json::to_string(committees_answers) - .expect("IndexMap> serialization is infallible"); - query_builder.set_field("committees_answers", QueryParam::Text(committees_answers)); - } - - // Add update time - let current_time = get_current_timestamp_secs(); - query_builder.set_field("updated_at", QueryParam::Int(current_time)); - - // Add WHERE clause - query_builder.and_where( - "hex(instance_id) = ? COLLATE NOCASE ", - Some(QueryParam::Text(hex::encode(self.instance_id))), - ); - - if let Some(ref statuses) = self.only_if_status_in { - if statuses.is_empty() { - // An empty allow-list must reject the update rather than - // silently dropping the compare-and-swap guard. - query_builder.and_where("1 = 0", None); - } else { - query_builder.and_where_in("status", statuses, false); - } - } - - query_builder - } -} - -/// Filter parameters for listing swap escrows. -#[derive(Clone, Debug, Default)] -pub struct SwapEscrowQuery { - pub offerer_addr: Option, - pub statuses: Vec, - pub order: Option, - pub offset: Option, - pub limit: Option, -} - -impl SwapEscrowQuery { - pub fn with_offerer_addr(mut self, offerer_addr: String) -> Self { - self.offerer_addr = Some(offerer_addr); - self - } - - pub fn with_status(mut self, status: String) -> Self { - self.statuses.push(status); - self - } - - pub fn with_order(mut self, order: String) -> Self { - self.order = Some(order); - self - } - - pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { - self.offset = Some(offset); - self.limit = Some(limit); - self - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::new(base_sql); - if let Some(offerer_addr) = &self.offerer_addr { - query_builder - .and_where("offerer_addr = ?", Some(QueryParam::Text(offerer_addr.clone()))); - } - if !self.statuses.is_empty() { - query_builder.and_where_in("status", &self.statuses, false); - } - if let Some(order) = &self.order { - query_builder.apply_order(order); - } - query_builder.apply_pagination(self.limit, self.offset); - query_builder - } -} - -/// Swap escrow field update parameters, keyed by escrow hash. -#[derive(Debug, Clone, Default)] -pub struct SwapEscrowUpdate { - pub escrow_hash: String, - pub status: Option, - pub claimer_addr: Option, - pub btc_addr: Option, - pub token: Option, - pub amount: Option, - pub refund_deadline: Option, - pub escrow_data: Option, - pub init_tx_hash: Option, - pub init_tx_height: Option, - pub claim_tx_hash: Option, - pub claim_btc_txid: Option, - pub refund_tx_hash: Option, - pub only_if_status_in: Option>, -} - -impl SwapEscrowUpdate { - pub fn new(escrow_hash: String) -> Self { - Self { escrow_hash, ..Self::default() } - } - - pub fn with_status(mut self, status: String) -> Self { - self.status = Some(status); - self - } - - pub fn with_claimer_addr(mut self, claimer_addr: String) -> Self { - self.claimer_addr = Some(claimer_addr); - self - } - - pub fn with_btc_addr(mut self, btc_addr: String) -> Self { - self.btc_addr = Some(btc_addr); - self - } - - pub fn with_token(mut self, token: String) -> Self { - self.token = Some(token); - self - } - - pub fn with_amount(mut self, amount: String) -> Self { - self.amount = Some(amount); - self - } - - pub fn with_refund_deadline(mut self, refund_deadline: i64) -> Self { - self.refund_deadline = Some(refund_deadline); - self - } - - pub fn with_escrow_data(mut self, escrow_data: String) -> Self { - self.escrow_data = Some(escrow_data); - self - } - - pub fn with_init_tx(mut self, tx_hash: String, tx_height: i64) -> Self { - self.init_tx_hash = Some(tx_hash); - self.init_tx_height = Some(tx_height); - self - } - - pub fn with_claim_tx_hash(mut self, tx_hash: String) -> Self { - self.claim_tx_hash = Some(tx_hash); - self - } - - pub fn with_claim_btc_txid(mut self, txid: SerializableTxid) -> Self { - self.claim_btc_txid = Some(txid); - self - } - - pub fn with_refund_tx_hash(mut self, tx_hash: String) -> Self { - self.refund_tx_hash = Some(tx_hash); - self - } - - /// Apply this update only while the escrow is still in one of the - /// expected states. The condition is folded into the UPDATE statement. - pub fn with_only_if_status_in(mut self, statuses: Vec) -> Self { - self.only_if_status_in = Some(statuses); - self - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::update(base_sql); - if let Some(ref status) = self.status { - query_builder.set_field("status", QueryParam::Text(status.clone())); - query_builder - .set_field("status_updated_at", QueryParam::Int(get_current_timestamp_secs())); - } - if let Some(ref claimer_addr) = self.claimer_addr { - query_builder.set_field("claimer_addr", QueryParam::Text(claimer_addr.clone())); - } - if let Some(ref btc_addr) = self.btc_addr { - query_builder.set_field("btc_addr", QueryParam::Text(btc_addr.clone())); - } - if let Some(ref token) = self.token { - query_builder.set_field("token", QueryParam::Text(token.clone())); - } - if let Some(ref amount) = self.amount { - query_builder.set_field("amount", QueryParam::Text(amount.clone())); - } - if let Some(refund_deadline) = self.refund_deadline { - query_builder.set_field("refund_deadline", QueryParam::Int(refund_deadline)); - } - if let Some(ref escrow_data) = self.escrow_data { - query_builder.set_field("escrow_data", QueryParam::Text(escrow_data.clone())); - } - if let Some(ref init_tx_hash) = self.init_tx_hash { - query_builder.set_field("init_tx_hash", QueryParam::Text(init_tx_hash.clone())); - } - if let Some(init_tx_height) = self.init_tx_height { - query_builder.set_field("init_tx_height", QueryParam::Int(init_tx_height)); - } - if let Some(ref claim_tx_hash) = self.claim_tx_hash { - query_builder.set_field("claim_tx_hash", QueryParam::Text(claim_tx_hash.clone())); - } - if let Some(ref claim_btc_txid) = self.claim_btc_txid { - query_builder.set_field("claim_btc_txid", QueryParam::BTCTxid(claim_btc_txid.clone())); - } - if let Some(ref refund_tx_hash) = self.refund_tx_hash { - query_builder.set_field("refund_tx_hash", QueryParam::Text(refund_tx_hash.clone())); - } - - query_builder.set_field("updated_at", QueryParam::Int(get_current_timestamp_secs())); - - query_builder - .and_where("escrow_hash = ?", Some(QueryParam::Text(self.escrow_hash.clone()))); - if let Some(ref statuses) = self.only_if_status_in { - if statuses.is_empty() { - // An empty allow-list must reject the update rather than - // silently dropping the compare-and-swap guard. - query_builder.and_where("1 = 0", None); - } else { - query_builder.and_where_in("status", statuses, false); - } - } - query_builder - } -} - -#[derive(Clone, Debug, Default)] -pub struct GraphQuery { - pub statuses: Vec, - pub operator_pubkey: Option, - pub kickoff_index: Option, - pub from_addr: Option, - pub graph_id: Option, - pub pegin_txid: Option, - pub raw_conditions: Vec, - pub order: Option, - pub offset: Option, - pub limit: Option, -} - -impl GraphQuery { - pub fn with_status(mut self, status: String) -> Self { - self.statuses.push(status); - self - } - - pub fn with_statuses(mut self, statuses: Vec) -> Self { - self.statuses = statuses; - self - } - pub fn with_raw_condition(mut self, raw_condition: String) -> Self { - self.raw_conditions.push(raw_condition); - self - } - - pub fn with_raw_conditions(mut self, raw_conditions: Vec) -> Self { - self.raw_conditions = raw_conditions; - self - } - - pub fn with_operator_pubkey(mut self, operator_pubkey: String) -> Self { - self.operator_pubkey = Some(operator_pubkey); - self - } - - pub fn with_order(mut self, order: String) -> Self { - self.order = Some(order); - self - } - - pub fn with_from_addr(mut self, from_addr: String) -> Self { - self.from_addr = Some(from_addr); - self - } - - pub fn with_graph_id(mut self, graph_id: String) -> Self { - self.graph_id = Some(graph_id); - self - } - - pub fn with_pegin_txid(mut self, pegin_txid: SerializableTxid) -> Self { - self.pegin_txid = Some(pegin_txid); - self - } - - pub fn with_kickoff_index(mut self, kickoff_index: i64) -> Self { - self.kickoff_index = Some(kickoff_index); - self - } - - pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { - self.offset = Some(offset); - self.limit = Some(limit); - self - } - - pub fn with_offset(mut self, offset: u32) -> Self { - self.offset = Some(offset); - self - } - - pub fn with_limit(mut self, limit: u32) -> Self { - self.limit = Some(limit); - self - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::new(base_sql); - // Add WHERE conditions - if !self.statuses.is_empty() { - query_builder.and_where_in("status", &self.statuses, false); - } - - if let Some(from_addr) = &self.from_addr { - query_builder.and_where("from_addr = ?", Some(QueryParam::Text(from_addr.clone()))); - } - - if let Some(operator) = &self.operator_pubkey { - query_builder - .and_where("operator_pubkey = ?", Some(QueryParam::Text(operator.clone()))); - } - - if let Some(kickoff_index) = self.kickoff_index { - query_builder.and_where("kickoff_index = ?", Some(QueryParam::Int(kickoff_index))); - } - - if let Some(pegin_txid) = &self.pegin_txid { - query_builder - .and_where("pegin_txid = ?", Some(QueryParam::BTCTxid(pegin_txid.clone()))); - } - - if let Some(graph_id) = &self.graph_id { - query_builder.and_where( - "hex(graph_id) = ? COLLATE NOCASE", - Some(QueryParam::Text(graph_id.clone())), - ); - } - for raw_condition in &self.raw_conditions { - query_builder.add_raw_condition(raw_condition); - } - if let Some(order) = &self.order { - query_builder.apply_order(order) - } - // Add pagination - query_builder.apply_pagination(self.limit, self.offset); - query_builder - } -} - -/// Runtime graph fields that are not part of the signed graph definition. -/// -/// Status is intentionally absent. All graph status changes must go through -/// `StorageProcessor::transition_graph_status` so a stale event cannot replace -/// a later chain-observed state. -#[derive(Clone, Debug)] -pub struct GraphRuntimeUpdate { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub challenge_txid: Option, - pub bridge_out_start_at: Option, - pub init_withdraw_tx_hash: Option, - pub proceed_withdraw_height: Option, -} - -impl GraphRuntimeUpdate { - /// Create new update parameters - pub fn new(instance_id: Uuid, graph_id: Uuid) -> Self { - Self { - instance_id, - graph_id, - challenge_txid: None, - bridge_out_start_at: None, - init_withdraw_tx_hash: None, - proceed_withdraw_height: None, - } - } - - /// Set challenge transaction ID - pub fn with_challenge_txid(mut self, challenge_txid: SerializableTxid) -> Self { - self.challenge_txid = Some(challenge_txid); - self - } - - /// Set bridge out start time - pub fn with_bridge_out_start_at(mut self, bridge_out_start_at: i64) -> Self { - self.bridge_out_start_at = Some(bridge_out_start_at); - self - } - - /// Set init withdraw transaction ID - pub fn with_init_withdraw_tx_hash(mut self, init_withdraw_tx_hash: String) -> Self { - self.init_withdraw_tx_hash = Some(init_withdraw_tx_hash); - self - } - - /// Set proceed withdraw tx height at goat chain - pub fn with_proceed_withdraw_height(mut self, proceed_withdraw_height: i64) -> Self { - self.proceed_withdraw_height = Some(proceed_withdraw_height); - self - } - - /// Check if any fields need to be updated - pub fn has_updates(&self) -> bool { - self.challenge_txid.is_some() - || self.bridge_out_start_at.is_some() - || self.init_withdraw_tx_hash.is_some() - || self.proceed_withdraw_height.is_some() - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::update(base_sql); - // Add SET fields - if let Some(ref challenge_txid) = self.challenge_txid { - query_builder.set_field("challenge_txid", QueryParam::BTCTxid(challenge_txid.clone())); - } - if let Some(bridge_out_start_at) = self.bridge_out_start_at { - query_builder.set_field("bridge_out_start_at", QueryParam::Int(bridge_out_start_at)); - } - if let Some(proceed_withdraw_height) = self.proceed_withdraw_height { - query_builder - .set_field("proceed_withdraw_height", QueryParam::Int(proceed_withdraw_height)); - } - if let Some(ref init_withdraw_tx_hash) = self.init_withdraw_tx_hash { - if init_withdraw_tx_hash.is_empty() { - // Set NULL value - query_builder.set_field_null("init_withdraw_tx_hash"); - } else { - query_builder.set_field( - "init_withdraw_tx_hash", - QueryParam::Text(init_withdraw_tx_hash.clone()), - ); - } - } - // Add update time - let current_time = get_current_timestamp_secs(); - query_builder.set_field("updated_at", QueryParam::Int(current_time)); - - // Add WHERE clause - query_builder.and_where( - "hex(graph_id) = ? COLLATE NOCASE", - Some(QueryParam::Text(hex::encode(self.graph_id))), - ); - query_builder.and_where( - "hex(instance_id) = ? COLLATE NOCASE", - Some(QueryParam::Text(hex::encode(self.instance_id))), - ); - - query_builder - } -} - -#[derive(Clone, Debug, Default)] -pub struct NodeQuery { - pub actor: Option, - pub goat_addr: Option, - pub time_threshold: Option, - pub is_in_time_threshold: bool, - pub order: Option, - pub offset: Option, - pub limit: Option, -} -impl NodeQuery { - pub fn with_actor(mut self, actor: String) -> Self { - self.actor = Some(actor); - self - } - - pub fn with_goat_addr(mut self, goat_addr: String) -> Self { - self.goat_addr = Some(goat_addr); - self - } - - pub fn with_time_threshold(mut self, time_threshold: i64, is_in_time_threshold: bool) -> Self { - self.time_threshold = Some(time_threshold); - self.is_in_time_threshold = is_in_time_threshold; - self - } - - pub fn with_order(mut self, order: String) -> Self { - self.order = Some(order); - self - } - - pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { - self.offset = Some(offset); - self.limit = Some(limit); - self - } - - pub fn with_offset(mut self, offset: u32) -> Self { - self.offset = Some(offset); - self - } - - pub fn with_limit(mut self, limit: u32) -> Self { - self.limit = Some(limit); - self - } - - pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { - let mut query_builder = QueryBuilder::new(base_sql); - - // Add WHERE conditions - if let Some(actor) = &self.actor { - query_builder.and_where("actor = ?", Some(QueryParam::Text(actor.clone()))); - } - - if let Some(goat_addr) = &self.goat_addr { - query_builder.and_where("goat_addr = ?", Some(QueryParam::Text(goat_addr.clone()))); - } - - if let Some(time_threshold) = &self.time_threshold { - if self.is_in_time_threshold { - query_builder.and_where("updated_at > ?", Some(QueryParam::Int(*time_threshold))); - } else { - query_builder.and_where("updated_at <= ?", Some(QueryParam::Int(*time_threshold))); - } - } - - if let Some(order) = &self.order { - query_builder.apply_order(order) - } - - // Add pagination - query_builder.apply_pagination(self.limit, self.offset); - query_builder - } -} - -impl<'a> StorageProcessor<'a> { - pub fn conn(&mut self) -> &mut SqliteConnection { - match &mut self.conn { - ConnectionHolder::Pooled(conn) => conn, - ConnectionHolder::Direct(conn) => conn, - ConnectionHolder::Transaction(conn) => conn, - } - } - - pub async fn commit(self) -> anyhow::Result<()> { - if let ConnectionHolder::Transaction(transaction) = self.conn { - transaction.commit().await?; - Ok(()) - } else { - panic!( - "StorageProcessor::commit can only be invoked after calling StorageProcessor::begin_transaction" - ); - } - } - - /// Returns grouped instance, graph, and queue state counts for Node metrics. - pub async fn node_metrics_state_counts(&mut self) -> anyhow::Result> { - let counts = sqlx::query_as::<_, MetricsStateCount>( - r#" - SELECT - 'instance' AS category, - status AS state, - COUNT(*) AS count, - MIN(created_at) AS oldest_created_at, - NULL AS last_success_at - FROM instance - GROUP BY status - UNION ALL - SELECT - 'swap_escrow' AS category, - status AS state, - COUNT(*) AS count, - MIN(created_at) AS oldest_created_at, - NULL AS last_success_at - FROM swap_escrow - GROUP BY status - UNION ALL - SELECT - 'graph' AS category, - status AS state, - COUNT(*) AS count, - MIN(created_at) AS oldest_created_at, - NULL AS last_success_at - FROM graph - GROUP BY status - UNION ALL - SELECT - 'message' AS category, - state, - COUNT(*) AS count, - MIN(created_at) AS oldest_created_at, - NULL AS last_success_at - FROM message - GROUP BY state - UNION ALL - SELECT - 'p2p_inbox' AS category, - state, - COUNT(*) AS count, - MIN(created_at) AS oldest_created_at, - NULL AS last_success_at - FROM p2p_inbox - GROUP BY state - ORDER BY category, state - "#, - ) - .fetch_all(self.conn()) - .await?; - Ok(counts) - } - - /// Returns local aggregate values for flow-stall and operator-liquidity alerts. - pub async fn node_alert_metrics_snapshot( - &mut self, - local_peer_id: &str, - ) -> anyhow::Result { - let snapshot = sqlx::query_as::<_, NodeAlertMetricsSnapshot>( - r#" - SELECT - ( - SELECT MIN(status_updated_at) - FROM instance - WHERE status NOT IN ( - 'RelayerL2Minted', 'PresignedFailed', 'RelayerL2MintedFailed', - 'Timeout', 'UserCanceled', 'NoEnoughCommitteesAnswered', 'UserDiscarded', - 'Failed', 'Success', 'Canceled' - ) - ) AS pegin_oldest_active_status_updated_at, - ( - SELECT MIN(status_updated_at) - FROM instance - WHERE status = 'UserInited' - ) AS pegin_oldest_committee_wait_status_updated_at, - ( - SELECT MIN(status_updated_at) - FROM swap_escrow - WHERE status NOT IN ('Claim', 'Timeout', 'Refund') - ) AS pegout_oldest_active_status_updated_at, - ( - SELECT available_peg_btc - FROM node - WHERE peer_id = ? AND actor = 'Operator' - LIMIT 1 - ) AS operator_available_pegbtc - "#, - ) - .bind(local_peer_id) - .fetch_one(self.conn()) - .await?; - Ok(snapshot) - } - - /// Returns aggregate local progress for the configured event watchers. - pub async fn event_watch_metrics_snapshot( - &mut self, - finalized_height: i64, - ) -> anyhow::Result { - Ok(sqlx::query_as::<_, EventWatchMetricsSnapshot>( - r#" - SELECT - COALESCE(MAX(MAX(? - from_height + 1, 0)), 0) AS lag_blocks, - COUNT(*) AS watcher_count, - COALESCE(SUM(CASE WHEN status IN ('UnSync', 'Syncing') THEN 1 ELSE 0 END), 0) - AS syncing_count, - COALESCE(SUM(CASE WHEN status = 'Failed' THEN 1 ELSE 0 END), 0) - AS failed_count - FROM watch_contract - WHERE from_height != 0 - "#, - ) - .bind(finalized_height) - .fetch_one(self.conn()) - .await?) - } - - /// Returns grouped proof task counts, ages, and latest success times for Proof Builder metrics. - pub async fn proof_metrics_state_counts(&mut self) -> anyhow::Result> { - let counts = sqlx::query_as::<_, MetricsStateCount>( - r#" - SELECT - chain_name AS category, - CAST(proof_state AS TEXT) AS state, - COUNT(*) AS count, - MIN(CASE WHEN proof_state IN (0, 1) THEN created_at END) AS oldest_created_at, - MAX(CASE WHEN proof_state = 2 THEN updated_at END) AS last_success_at - FROM long_running_task_proof - GROUP BY chain_name, proof_state - UNION ALL - SELECT - 'operator' AS category, - CAST(proof_state AS TEXT) AS state, - COUNT(*) AS count, - MIN(CASE WHEN proof_state IN (0, 1) THEN created_at END) AS oldest_created_at, - MAX(CASE WHEN proof_state = 2 THEN updated_at END) AS last_success_at - FROM operator_proof - GROUP BY proof_state - UNION ALL - SELECT - 'watchtower' AS category, - CAST(proof_state AS TEXT) AS state, - COUNT(*) AS count, - MIN(CASE WHEN proof_state IN (0, 1) THEN created_at END) AS oldest_created_at, - MAX(CASE WHEN proof_state = 2 THEN updated_at END) AS last_success_at - FROM watchtower_proof - GROUP BY proof_state - ORDER BY category, state - "#, - ) - .fetch_all(self.conn()) - .await?; - Ok(counts) - } - - /// Insert an instance only when its ID is not already present. - /// - /// Parameters: - /// - instance: The complete instance data to insert - /// - /// Returns: - /// - Ok(true) if the instance was inserted - /// - Ok(false) if an instance with the same ID already exists - /// - Err if the operation failed - pub async fn insert_instance_if_absent(&mut self, instance: &Instance) -> anyhow::Result { - let committees_answers_json = serde_json::to_string(&instance.committees_answers)?; - let res = sqlx::query( - "INSERT INTO instance (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, status, goat_tx_hash, goat_tx_height, - user_xonly_pubkey, user_change_addr, user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers, - pegin_data_tx_hash, btc_height, parameters, status_updated_at, post_pegin_txhash, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(instance_id) DO NOTHING", - ) - .bind(instance.instance_id) - .bind(&instance.network) - .bind(&instance.from_addr) - .bind(&instance.to_addr) - .bind(instance.amount) - .bind(instance.fees) - .bind(&instance.input_utxos) - .bind(&instance.status) - .bind(&instance.goat_tx_hash) - .bind(instance.goat_tx_height) - .bind(instance.user_xonly_pubkey) - .bind(&instance.user_change_addr) - .bind(&instance.user_refund_addr) - .bind(&instance.btc_txid) - .bind(&instance.pegin_confirm_txid) - .bind(&instance.pegin_cancel_txid) - .bind(committees_answers_json) - .bind(&instance.pegin_data_tx_hash) - .bind(instance.btc_height) - .bind(&instance.parameters) - .bind(instance.status_updated_at) - .bind(&instance.post_pegin_txhash) - .bind(instance.created_at) - .bind(instance.updated_at) - .execute(self.conn()) - .await?; - Ok(res.rows_affected() > 0) - } - - /// Create a bridge-in instance from a pegin request, or refresh one that - /// has not moved past the pegin-request stage yet. - /// - /// `PeginRequest` is a re-deliverable P2P message, so a full-row replacement - /// would let a replayed or forged request roll a live instance back to its - /// initial row and clear everything the later stages wrote. The write is - /// therefore a compare-and-swap over the current status, and it only touches - /// the columns a pegin request owns: - /// committee answers, instance parameters and the BTC-side fields are never - /// overwritten. The caller is responsible for passing canonical request - /// metadata - `goat_tx_hash`/`goat_tx_height` are refreshed from it, so that - /// a re-indexed event can still correct them. - /// - /// Returns: - /// - Ok(true) if the row was inserted or updated - /// - Ok(false) if an existing row was left untouched because its status - /// is outside `allowed_statuses` - pub async fn upsert_pegin_request_instance( - &mut self, - instance: &Instance, - allowed_statuses: &[String], - ) -> anyhow::Result { - let committees_answers_json = serde_json::to_string(&instance.committees_answers)?; - // An empty allow-list must reject the update rather than silently - // dropping the compare-and-swap guard. - let status_guard = if allowed_statuses.is_empty() { - "1 = 0".to_string() - } else { - let placeholders = allowed_statuses.iter().map(|_| "?").collect::>().join(", "); - format!("instance.status IN ({placeholders})") - }; - let sql = format!( - "INSERT INTO instance \ - (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, \ - status, goat_tx_hash, goat_tx_height, user_xonly_pubkey, user_change_addr, \ - user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers, \ - pegin_data_tx_hash, btc_height, parameters, status_updated_at, \ - post_pegin_txhash, created_at, updated_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ - ON CONFLICT(instance_id) DO UPDATE SET \ - network = excluded.network, \ - from_addr = CASE WHEN excluded.from_addr = '' \ - THEN instance.from_addr ELSE excluded.from_addr END, \ - to_addr = excluded.to_addr, \ - amount = excluded.amount, \ - fees = excluded.fees, \ - input_utxos = excluded.input_utxos, \ - status = excluded.status, \ - goat_tx_hash = excluded.goat_tx_hash, \ - goat_tx_height = excluded.goat_tx_height, \ - user_xonly_pubkey = excluded.user_xonly_pubkey, \ - user_change_addr = excluded.user_change_addr, \ - user_refund_addr = excluded.user_refund_addr, \ - status_updated_at = excluded.status_updated_at, \ - updated_at = excluded.updated_at \ - WHERE {status_guard}" - ); - let mut query = sqlx::query(&sql) - .bind(instance.instance_id) - .bind(&instance.network) - .bind(&instance.from_addr) - .bind(&instance.to_addr) - .bind(instance.amount) - .bind(instance.fees) - .bind(&instance.input_utxos) - .bind(&instance.status) - .bind(&instance.goat_tx_hash) - .bind(instance.goat_tx_height) - .bind(instance.user_xonly_pubkey) - .bind(&instance.user_change_addr) - .bind(&instance.user_refund_addr) - .bind(&instance.btc_txid) - .bind(&instance.pegin_confirm_txid) - .bind(&instance.pegin_cancel_txid) - .bind(committees_answers_json) - .bind(&instance.pegin_data_tx_hash) - .bind(instance.btc_height) - .bind(&instance.parameters) - .bind(instance.status_updated_at) - .bind(&instance.post_pegin_txhash) - .bind(instance.created_at) - .bind(instance.updated_at); - for status in allowed_statuses { - query = query.bind(status); - } - let res = query.execute(self.conn()).await?; - Ok(res.rows_affected() > 0) - } - - /// Find a single instance by its ID - /// - /// Retrieves an instance from the database using its unique instance_id. - /// - /// Parameters: - /// - instance_id: The UUID of the instance to find - /// - /// Returns: - /// - Ok(Some(instance)) if the instance was found - /// - Ok(None) if no instance with the given ID exists - /// - Err if the query failed - pub async fn find_instance(&mut self, instance_id: &Uuid) -> anyhow::Result> { - let row = sqlx::query_as::<_, Instance>("SELECT * FROM instance WHERE instance_id = ?") - .bind(instance_id) - .fetch_optional(self.conn()) - .await?; - Ok(row) - } - - /// Find multiple instances with filtering and pagination - /// - /// Retrieves instances from the database with optional filtering criteria - /// and pagination support. Uses QueryBuilder for dynamic query construction. - /// - /// Parameters: - /// - params: InstanceQuery containing all filter criteria and pagination options - /// - /// Returns: - /// - Ok((instances, total_count)) where instances is a vector of matching instances - /// and total_count is the total number of instances matching the criteria - /// - Err if the query failed - pub async fn find_instances( - &mut self, - params: InstanceQuery, - ) -> anyhow::Result<(Vec, i64)> { - let mut count_params = params.clone(); - (count_params.order, count_params.offset) = (None, None); - let instances_query_builder = params.get_query_builder("SELECT * FROM instance"); - let count_query_builder = - count_params.get_query_builder("SELECT count(*) as total_instances FROM instance"); - // Execute data query - let sql = instances_query_builder.get_sql(); - let mut data_query = sqlx::query_as::<_, Instance>(&sql); - data_query = instances_query_builder.query_as(data_query); - // Execute count query - let count_sql = count_query_builder.get_sql(); - let mut count_query = sqlx::query(&count_sql); - count_query = count_query_builder.query(count_query); - Ok(( - data_query.fetch_all(self.conn()).await?, - count_query.fetch_one(self.conn()).await?.get::("total_instances"), - )) - } - /// Insert a swap escrow only when its escrow hash is not already present. - /// - /// The chain-event watcher is the sole writer for Initialize records; - /// the escrow-hash primary key makes event replays idempotent. - pub async fn insert_swap_escrow_if_absent( - &mut self, - escrow: &SwapEscrow, - ) -> anyhow::Result { - let res = sqlx::query!( - "INSERT INTO swap_escrow \ - (escrow_hash, network, status, offerer_addr, claimer_addr, btc_addr, token, amount, \ - refund_deadline, escrow_data, init_tx_hash, init_tx_height, claim_tx_hash, \ - claim_btc_txid, refund_tx_hash, status_updated_at, created_at, updated_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ - ON CONFLICT(escrow_hash) DO NOTHING", - escrow.escrow_hash, - escrow.network, - escrow.status, - escrow.offerer_addr, - escrow.claimer_addr, - escrow.btc_addr, - escrow.token, - escrow.amount, - escrow.refund_deadline, - escrow.escrow_data, - escrow.init_tx_hash, - escrow.init_tx_height, - escrow.claim_tx_hash, - escrow.claim_btc_txid, - escrow.refund_tx_hash, - escrow.status_updated_at, - escrow.created_at, - escrow.updated_at - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected() > 0) - } - - /// Find a swap escrow by its (normalized) escrow hash. - pub async fn find_swap_escrow( - &mut self, - escrow_hash: &str, - ) -> anyhow::Result> { - let row = - sqlx::query_as::<_, SwapEscrow>("SELECT * FROM swap_escrow WHERE escrow_hash = ?") - .bind(escrow_hash) - .fetch_optional(self.conn()) - .await?; - Ok(row) - } - - /// Find swap escrows with filtering and pagination. - /// - /// Returns the matching page together with the total match count. - pub async fn find_swap_escrows( - &mut self, - params: SwapEscrowQuery, - ) -> anyhow::Result<(Vec, i64)> { - let mut count_params = params.clone(); - (count_params.order, count_params.offset) = (None, None); - let escrows_query_builder = params.get_query_builder("SELECT * FROM swap_escrow"); - let count_query_builder = - count_params.get_query_builder("SELECT count(*) as total_escrows FROM swap_escrow"); - let sql = escrows_query_builder.get_sql(); - let mut data_query = sqlx::query_as::<_, SwapEscrow>(&sql); - data_query = escrows_query_builder.query_as(data_query); - let count_sql = count_query_builder.get_sql(); - let mut count_query = sqlx::query(&count_sql); - count_query = count_query_builder.query(count_query); - Ok(( - data_query.fetch_all(self.conn()).await?, - count_query.fetch_one(self.conn()).await?.get::("total_escrows"), - )) - } - - /// Update swap escrow fields using the builder pattern. - pub async fn update_swap_escrow(&mut self, params: &SwapEscrowUpdate) -> anyhow::Result { - let query_builder = params.get_query_builder("swap_escrow"); - let update_sql = query_builder.get_sql(); - let mut query = sqlx::query(&update_sql); - query = query_builder.query(query); - let result = query.execute(self.conn()).await?; - Ok(result.rows_affected() > 0) - } - - /// Mark still-initializing swap escrows whose refund deadline has passed - /// as timed out. Returns the number of escrows transitioned. - pub async fn timeout_expired_swap_escrows(&mut self, current_time: i64) -> anyhow::Result { - let initialize_status = SwapEscrowStatus::Initialize.to_string(); - let timeout_status = SwapEscrowStatus::Timeout.to_string(); - let row = sqlx::query!( - "UPDATE swap_escrow SET status = ?, status_updated_at = ?, updated_at = ? \ - WHERE status = ? AND refund_deadline > 0 AND refund_deadline < ?", - timeout_status, - current_time, - current_time, - initialize_status, - current_time - ) - .execute(self.conn()) - .await?; - Ok(row.rows_affected()) - } - - /// Update expired instances status - /// - /// Updates the status of instances that have expired based on a time threshold. - /// This is typically used for cleanup operations to mark old instances as expired. - /// - /// Parameters: - /// - current_status: The current status to match for instances to be updated - /// - expired_status: The new status to set for expired instances - /// - time_threshold: Timestamp threshold - instances updated before this time will be marked as expired - /// - /// Returns: - /// - Ok(affected_rows) number of instances that were updated - /// - Err if the update operation failed - pub async fn update_expired_instance( - &mut self, - current_status: &str, - expired_status: &str, - time_threshold: i64, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let row = sqlx::query!( - r#"UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? WHERE status = ? AND updated_at < ?"#, - expired_status, - current_time, - current_time, - current_status, - time_threshold - ) - .execute(self.conn()) - .await?; - Ok(row.rows_affected()) - } - - /// Transition an instance only when it is still in the expected status. - pub async fn update_instance_status_if_current( - &mut self, - instance_id: &Uuid, - current_status: &str, - new_status: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let result = sqlx::query!( - "UPDATE instance SET status = ?, status_updated_at = ?, updated_at = ? \ - WHERE instance_id = ? AND status = ?", - new_status, - current_time, - current_time, - instance_id, - current_status, - ) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected() > 0) - } - - /// Update instance fields using builder pattern - /// - /// This is the most elegant update method, using the InstanceUpdate builder pattern - /// Provides type safety and clear API - pub async fn update_instance(&mut self, params: &InstanceUpdate) -> anyhow::Result { - if !params.has_updates() { - return Ok(false); - } - let query_builder = params.get_query_builder("instance"); - // Get SQL and parameters - let update_sql = query_builder.get_sql(); - // Execute query - let mut query = sqlx::query(&update_sql); - query = query_builder.query(query); - let result = query.execute(self.conn()).await?; - Ok(result.rows_affected() > 0) - } - - /// Add or update a single committee answer for an instance - /// - /// This method merges one committee answer atomically so concurrent event - /// handlers cannot overwrite each other's answers with stale full maps. - pub async fn update_instance_committee_answer( - &mut self, - instance_id: &Uuid, - committee_addr: &str, - pubkey: Vec, - ) -> anyhow::Result { - let committee_patch = serde_json::json!({ (committee_addr): pubkey }).to_string(); - let current_time = get_current_timestamp_secs(); - let result = sqlx::query( - "UPDATE instance \ - SET committees_answers = json_patch(COALESCE(committees_answers, '{}'), json(?)), \ - updated_at = ? \ - WHERE instance_id = ?", - ) - .bind(committee_patch) - .bind(current_time) - .bind(instance_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn update_instance_parameters( - &mut self, - instance_id: &Uuid, - parameters: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE instance SET parameters = ?, updated_at = ? WHERE instance_id = ?", - parameters, - current_time, - instance_id - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected() > 0) - } - - pub async fn get_instance_parameters_by_id( - &mut self, - instance_id: &Uuid, - ) -> anyhow::Result> { - let res = - sqlx::query!("SELECT parameters FROM instance WHERE instance_id = ?", instance_id) - .fetch_optional(self.conn()) - .await?; - Ok(res.and_then(|record| record.parameters)) - } - - pub async fn upsert_pending_graph_init( - &mut self, - instance_id: &Uuid, - operator_pubkey: &str, - graph_id: &Uuid, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let result = sqlx::query( - "INSERT INTO pending_graph_init - (instance_id, operator_pubkey, graph_id, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(instance_id, operator_pubkey) DO UPDATE SET - graph_id = excluded.graph_id, - updated_at = excluded.updated_at", - ) - .bind(instance_id) - .bind(operator_pubkey) - .bind(graph_id) - .bind(current_time) - .bind(current_time) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - pub async fn find_pending_graph_init_by_instance_and_operator_pubkey( - &mut self, - instance_id: &Uuid, - operator_pubkey: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as::<_, PendingGraphInit>( - "SELECT instance_id, operator_pubkey, graph_id, created_at, updated_at - FROM pending_graph_init - WHERE instance_id = ? AND operator_pubkey = ?", - ) - .bind(instance_id) - .bind(operator_pubkey) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn find_pending_graph_init_by_graph_id( - &mut self, - graph_id: &Uuid, - ) -> anyhow::Result> { - Ok(sqlx::query_as::<_, PendingGraphInit>( - "SELECT instance_id, operator_pubkey, graph_id, created_at, updated_at - FROM pending_graph_init - WHERE graph_id = ?", - ) - .bind(graph_id) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn delete_pending_graph_init( - &mut self, - instance_id: &Uuid, - operator_pubkey: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "DELETE FROM pending_graph_init WHERE instance_id = ? AND operator_pubkey = ?", - ) - .bind(instance_id) - .bind(operator_pubkey) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Insert a graph definition or verify a compatible replay. - /// - /// The canonical definition hash is an identity fence: a new graph may - /// never reuse an existing graph id and inherit its runtime projection. - /// Compatible replays may only fill missing non-identity metadata; they - /// never replace status, observed transaction ids, or withdraw metadata. - pub async fn upsert_graph_definition(&mut self, graph: &Graph) -> anyhow::Result { - if graph.definition_hash.is_empty() { - anyhow::bail!("graph {} is missing its definition hash", graph.graph_id); - } - if graph.status != GraphStatus::OperatorPresigned.to_string() - || !graph.sub_status.is_empty() - || graph.challenge_txid.is_some() - || graph.init_withdraw_tx_hash.is_some() - || graph.bridge_out_start_at != 0 - || graph.proceed_withdraw_height != 0 - { - anyhow::bail!( - "graph {} definition writes must use the OperatorPresigned baseline without runtime data", - graph.graph_id - ); - } - let verifier_assert_txids_json = serde_json::to_string(&graph.verifier_assert_txids)?; - let disprove_txids_json = serde_json::to_string(&graph.disprove_txids)?; - let watchtower_challenge_timeout_txids_json = - serde_json::to_string(&graph.watchtower_challenge_timeout_txids)?; - let operator_challenge_nack_txids_json = - serde_json::to_string(&graph.operator_challenge_nack_txids)?; - let res = sqlx::query( - "INSERT INTO graph (graph_id, instance_id, kickoff_index, from_addr, to_addr, amount, challenge_amount, - status, sub_status, operator_pubkey, definition_hash, cur_prekickoff_txid, next_prekickoff, force_skip_kickoff_txid, - quick_challenge_txid, challenge_incomplete_kickoff_txid, pegin_txid, kickoff_txid, take1_txid, - challenge_txid, take2_txid, watchtower_challenge_init_txid, operator_assert_txid, verifier_assert_txids, disprove_txids, - watchtower_challenge_timeout_txids, operator_challenge_nack_txids, operator_commit_timeout_txid, - init_withdraw_tx_hash, bridge_out_start_at, status_updated_at, proceed_withdraw_height, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(graph_id) DO UPDATE SET - from_addr = CASE - WHEN graph.from_addr = '' AND excluded.from_addr <> '' THEN excluded.from_addr - ELSE graph.from_addr - END, - to_addr = CASE - WHEN graph.to_addr = '' AND excluded.to_addr <> '' THEN excluded.to_addr - ELSE graph.to_addr - END - WHERE graph.definition_hash = excluded.definition_hash - AND graph.instance_id = excluded.instance_id", - ) - .bind(graph.graph_id) - .bind(graph.instance_id) - .bind(graph.kickoff_index) - .bind(&graph.from_addr) - .bind(&graph.to_addr) - .bind(graph.amount) - .bind(graph.challenge_amount) - .bind(GraphStatus::OperatorPresigned.to_string()) - .bind("") - .bind(&graph.operator_pubkey) - .bind(&graph.definition_hash) - .bind(graph.cur_prekickoff_txid.clone()) - .bind(graph.next_prekickoff.clone()) - .bind(graph.force_skip_kickoff_txid.clone()) - .bind(graph.quick_challenge_txid.clone()) - .bind(graph.challenge_incomplete_kickoff_txid.clone()) - .bind(graph.pegin_txid.clone()) - .bind(graph.kickoff_txid.clone()) - .bind(graph.take1_txid.clone()) - .bind(Option::::None) - .bind(graph.take2_txid.clone()) - .bind(graph.watchtower_challenge_init_txid.clone()) - .bind(graph.operator_assert_txid.clone()) - .bind(verifier_assert_txids_json) - .bind(disprove_txids_json) - .bind(watchtower_challenge_timeout_txids_json) - .bind(operator_challenge_nack_txids_json) - .bind(graph.operator_commit_timeout_txid.clone()) - .bind(Option::::None) - .bind(0_i64) - .bind(graph.status_updated_at) - .bind(0_i64) - .bind(graph.created_at) - .bind(graph.updated_at) - .execute(self.conn()) - .await?; - if res.rows_affected() == 0 { - let Some(existing) = self.find_graph(&graph.graph_id).await? else { - anyhow::bail!("graph {} disappeared while storing its definition", graph.graph_id); - }; - if existing.definition_hash != graph.definition_hash { - anyhow::bail!( - "conflicting graph definition for graph_id {}: existing={}, incoming={}", - graph.graph_id, - existing.definition_hash, - graph.definition_hash - ); - } - if existing.instance_id != graph.instance_id { - anyhow::bail!( - "graph definition instance mismatch for graph_id {}: existing={}, incoming={}", - graph.graph_id, - existing.instance_id, - graph.instance_id - ); - } - } - Ok(res.rows_affected()) - } - - pub async fn update_graph_runtime( - &mut self, - params: &GraphRuntimeUpdate, - ) -> anyhow::Result { - if !params.has_updates() { - return Ok(false); - } - let query_builder = params.get_query_builder("graph"); - let update_sql = query_builder.get_sql(); - let query = sqlx::query(&update_sql); - let query = query_builder.query(query); - let result = query.execute(self.conn()).await?; - Ok(result.rows_affected() > 0) - } - - /// Atomically advance a graph status according to its evidence source. - /// - /// The conditional UPDATE is the authority check. The follow-up read only - /// distinguishes an idempotent replay from a stale event; it never decides - /// whether a write is permitted. - pub async fn transition_graph_status( - &mut self, - instance_id: Uuid, - graph_id: Uuid, - target: GraphStatus, - source: GraphStatusSource, - sub_status: Option, - ) -> anyhow::Result { - if !target.is_protocol_status() { - anyhow::bail!("frontend-only graph status {target} cannot be persisted"); - } - - let allowed_from = target.allowed_transition_from(source); - if allowed_from.is_empty() { - // Some verified scans merely observe a baseline state (for - // example, an operator-pre-signed graph). There is no authorized - // predecessor edge to write in that case; return the current - // projection without mutating it. - return self.graph_status_transition_outcome(instance_id, graph_id, target).await; - } - - let allowed_from: Vec = allowed_from.iter().map(ToString::to_string).collect(); - let current_time = get_current_timestamp_secs(); - let mut query_builder = QueryBuilder::update("graph"); - query_builder.set_field("status", QueryParam::Text(target.to_string())); - if let Some(sub_status) = sub_status.as_ref() { - query_builder.set_field("sub_status", QueryParam::Text(sub_status.clone())); - } - query_builder.set_field("status_updated_at", QueryParam::Int(current_time)); - query_builder.set_field("updated_at", QueryParam::Int(current_time)); - query_builder.and_where( - "hex(graph_id) = ? COLLATE NOCASE", - Some(QueryParam::Text(hex::encode(graph_id))), - ); - query_builder.and_where( - "hex(instance_id) = ? COLLATE NOCASE", - Some(QueryParam::Text(hex::encode(instance_id))), - ); - query_builder.and_where_in("status", &allowed_from, false); - - let update_sql = query_builder.get_sql(); - let query = query_builder.query(sqlx::query(&update_sql)); - if query.execute(self.conn()).await?.rows_affected() > 0 { - return Ok(GraphStatusTransitionOutcome::Applied); - } - - // A zero-row conditional update has two distinct meanings: this can - // be an idempotent replay, or another writer may have moved the row to - // a state which rejects this transition. The follow-up read reports - // that distinction; it never authorizes a write. - let outcome = self.graph_status_transition_outcome(instance_id, graph_id, target).await?; - if !matches!(outcome, GraphStatusTransitionOutcome::AlreadyCurrent) { - return Ok(outcome); - } - - let Some(sub_status) = sub_status else { - return Ok(GraphStatusTransitionOutcome::AlreadyCurrent); - }; - - let current_time = get_current_timestamp_secs(); - let result = sqlx::query( - "UPDATE graph SET sub_status = ?, updated_at = ? \ - WHERE graph_id = ? AND instance_id = ? AND status = ?", - ) - .bind(sub_status) - .bind(current_time) - .bind(graph_id) - .bind(instance_id) - .bind(target.to_string()) - .execute(self.conn()) - .await?; - if result.rows_affected() > 0 { - return Ok(GraphStatusTransitionOutcome::AlreadyCurrent); - } - - // The row may have changed between the outcome read and the optional - // sub-status update. Re-read so a concurrent transition is never - // misreported as an idempotent replay or a missing graph. - self.graph_status_transition_outcome(instance_id, graph_id, target).await - } - - async fn graph_status_transition_outcome( - &mut self, - instance_id: Uuid, - graph_id: Uuid, - target: GraphStatus, - ) -> anyhow::Result { - let Some(current_graph) = self.find_graph(&graph_id).await? else { - return Ok(GraphStatusTransitionOutcome::NotFound); - }; - if current_graph.instance_id != instance_id { - return Ok(GraphStatusTransitionOutcome::NotFound); - } - let current = GraphStatus::from_str(¤t_graph.status).map_err(|_| { - anyhow::anyhow!( - "graph {graph_id} has invalid persisted status {}", - current_graph.status - ) - })?; - Ok(if current == target { - GraphStatusTransitionOutcome::AlreadyCurrent - } else { - GraphStatusTransitionOutcome::Rejected { current } - }) - } - - pub async fn find_graph(&mut self, graph_id: &Uuid) -> anyhow::Result> { - let row = sqlx::query_as::<_, Graph>( - "SELECT * - FROM graph - WHERE graph_id = ?", - ) - .bind(graph_id) - .fetch_optional(self.conn()) - .await?; - Ok(row) - } - - pub async fn find_graphs(&mut self, params: GraphQuery) -> anyhow::Result<(Vec, i64)> { - // Build base query - let mut count_params = params.clone(); - (count_params.offset, count_params.limit) = (None, None); - let query_builder = params.get_query_builder( - "SELECT graph_id, - instance_id, - kickoff_index, - from_addr, - to_addr, - amount, - challenge_amount, - status, - sub_status, - operator_pubkey, - definition_hash, - cur_prekickoff_txid, - next_prekickoff, - force_skip_kickoff_txid, - quick_challenge_txid, - challenge_incomplete_kickoff_txid, - pegin_txid, - kickoff_txid, - take1_txid, - challenge_txid, - take2_txid, - watchtower_challenge_init_txid, - operator_assert_txid, - verifier_assert_txids, - disprove_txids, - watchtower_challenge_timeout_txids, - operator_challenge_nack_txids, - operator_commit_timeout_txid, - init_withdraw_tx_hash, - bridge_out_start_at, - status_updated_at, - proceed_withdraw_height, - CASE - WHEN bridge_out_start_at > 0 - THEN bridge_out_start_at - ELSE created_at - END AS created_at, - updated_at - FROM graph", - ); - - let count_query_builder = - count_params.get_query_builder("SELECT count(graph_id) as total_graphs FROM graph"); - let sql = query_builder.get_sql(); - let mut graphs_query = sqlx::query_as::<_, Graph>(&sql); - graphs_query = query_builder.query_as(graphs_query); - let count_sql = count_query_builder.get_sql(); - let mut count_query = sqlx::query(&count_sql); - count_query = count_query_builder.query(count_query); - - Ok(( - graphs_query.fetch_all(self.conn()).await?, - count_query.fetch_one(self.conn()).await?.get::("total_graphs"), - )) - } - - pub async fn find_graphs_by_status_group_by_operator( - &mut self, - status: &str, - ) -> anyhow::Result> { - let row = sqlx::query_as::<_, Graph>( - "SELECT * - FROM graph - WHERE status = ? ORDER BY operator_pubkey, kickoff_index", - ) - .bind(status) - .fetch_all(self.conn()) - .await?; - Ok(row) - } - - pub async fn get_graphs_by_instance_id( - &mut self, - instance_id: &Uuid, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, Graph>( - "SELECT * - FROM graph - WHERE instance_id = ?", - ) - .bind(instance_id) - .fetch_all(self.conn()) - .await?; - Ok(res) - } - - pub async fn get_graph_id_by_instance_id_and_operator_pubkey( - &mut self, - instance_id: &Uuid, - operator_pubkey: &str, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, Graph>( - "SELECT * - FROM graph - WHERE instance_id = ? AND operator_pubkey = ?", - ) - .bind(instance_id) - .bind(operator_pubkey) - .fetch_optional(self.conn()) - .await?; - Ok(res.map(|graph| graph.graph_id)) - } - - pub async fn get_graph_pre_kickoff_chain_by_cur_pre_kickoff( - &mut self, - current_pre_kickoff: SerializableTxid, - ) -> anyhow::Result> { - #[derive(sqlx::FromRow)] - struct NextPrekickoffRow { - pub graph_id: Uuid, - pub instance_id: Uuid, - pub cur_prekickoff_txid: SerializableTxid, - pub next_prekickoff: SerializableTxid, - } - let res = sqlx::query_as::<_, NextPrekickoffRow>( - "SELECT graph_id, instance_id, cur_prekickoff_txid, next_prekickoff FROM graph WHERE cur_prekickoff_txid = ?", - ) - .bind(current_pre_kickoff) - .fetch_optional(self.conn()) - .await?; - Ok(res.map(|v| (v.graph_id, v.instance_id, v.cur_prekickoff_txid, v.next_prekickoff))) - } - - pub async fn get_operator_graphs(&mut self, params: GraphQuery) -> anyhow::Result> { - let graph_query_builder = params.get_query_builder("SELECT * FROM graph"); - let operator_graph_sql = graph_query_builder.get_sql(); - let mut operator_graphs_query = sqlx::query_as::<_, Graph>(&operator_graph_sql); - operator_graphs_query = graph_query_builder.query_as(operator_graphs_query); - Ok(operator_graphs_query.fetch_all(self.conn()).await?) - } - - pub async fn update_node_timestamp( - &mut self, - peer_id: &str, - timestamp: i64, - ) -> anyhow::Result<()> { - let result = - sqlx::query!(r#"UPDATE node SET updated_at = ? WHERE peer_id = ?"#, timestamp, peer_id) - .execute(self.conn()) - .await?; - - if result.rows_affected() == 0 { - // Any peer that never announced itself lands here, once per message - // burst, so this must not be louder than debug. - tracing::debug!("Node {peer_id} not found in DB, no rows updated"); - } - - Ok(()) - } - - pub async fn find_graph_neighbor_ids( - &mut self, - graph_id: Uuid, - range: i64, - ) -> anyhow::Result> { - #[derive(sqlx::FromRow)] - struct GraphIds { - pub graph_id: Uuid, - pub kickoff_index: i64, - } - if let Some(graph) = self.find_graph(&graph_id).await? { - let start = 0.max(graph.kickoff_index - range); - let end = graph.kickoff_index + range; - let res = sqlx::query_as!( - GraphIds, - "SELECT graph_id AS \"graph_id:Uuid\", kickoff_index - FROM graph - WHERE operator_pubkey = ? - AND kickoff_index >= ? - AND kickoff_index <= ?", - graph.operator_pubkey, - start, - end - ) - .fetch_all(self.conn()) - .await?; - - Ok(res.into_iter().map(|g| (g.kickoff_index, g.graph_id)).collect()) - } else { - Ok(vec![]) - } - } - /// Insert or update node without reward field - pub async fn upsert_node(&mut self, node: &Node) -> anyhow::Result { - let res = sqlx::query!( - r#" - INSERT INTO node (peer_id, node_name, actor, goat_addr, btc_pub_key, socket_addr, service_fee_rate, available_peg_btc, - binding_sig, binding_issued_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (peer_id) DO UPDATE SET actor = excluded.actor, - node_name = excluded.node_name, - goat_addr = excluded.goat_addr, - btc_pub_key = excluded.btc_pub_key, - service_fee_rate = excluded.service_fee_rate, - available_peg_btc = excluded.available_peg_btc, - socket_addr = excluded.socket_addr, - binding_sig = excluded.binding_sig, - binding_issued_at = excluded.binding_issued_at, - updated_at = excluded.updated_at - "#, - node.peer_id, - node.node_name, - node.actor, - node.goat_addr, - node.btc_pub_key, - node.socket_addr, - node.service_fee_rate, - node.available_peg_btc, - node.binding_sig, - node.binding_issued_at, - node.created_at, - node.updated_at, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - // Do not update the `updated_at` field; this field is updated based on heartbeat messages - // and is used to determine whether a node is alive. - pub async fn update_node_reward_by_peer_id( - &mut self, - peer_id: &str, - reward: &str, - ) -> anyhow::Result<()> { - sqlx::query!(r#"UPDATE node SET reward = reward + ? WHERE peer_id = ?"#, reward, peer_id) - .execute(self.conn()) - .await?; - Ok(()) - } - - pub async fn get_node_by_btc_pub_key( - &mut self, - btc_pub_key: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as!(Node, r#"SELECT * FROM node WHERE btc_pub_key = ?"#, btc_pub_key) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn find_nodes(&mut self, params: &NodeQuery) -> anyhow::Result<(Vec, i64)> { - let data_query_builder = params.get_query_builder("SELECT * FROM node"); - let mut count_params = params.clone(); - (count_params.offset, count_params.limit) = (None, None); - let count_query_builder = - count_params.get_query_builder("SELECT count(peer_id) as total_nodes FROM node"); - - let data_sql = data_query_builder.get_sql(); - let mut nodes_query = sqlx::query_as::<_, Node>(&data_sql); - nodes_query = data_query_builder.query_as(nodes_query); - - let count_sql = count_query_builder.get_sql(); - let mut count_query = sqlx::query(&count_sql); - count_query = count_query_builder.query(count_query); - - Ok(( - nodes_query.fetch_all(self.conn()).await?, - count_query.fetch_one(self.conn()).await?.get::("total_nodes"), - )) - } - - pub async fn node_overview(&mut self, time_threshold: i64) -> anyhow::Result { - let records = sqlx::query!( - r#"SELECT count(*) AS total, - actor, - SUM(CASE WHEN updated_at >= ? THEN 1 ELSE 0 END) AS online, - SUM(CASE WHEN updated_at < ? THEN 1 ELSE 0 END) AS offline - FROM node - GROUP BY actor"#, - time_threshold, - time_threshold - ) - .fetch_all(self.conn()) - .await?; - - let mut res = NodesOverview::default(); - for record in records { - res.total += record.total; - match record.actor.as_str() { - "Verifier" => { - res.offline_verifiers += record.offline; - res.online_verifiers += record.online; - } - "Operator" => { - (res.offline_operators, res.online_operators) = (record.offline, record.online); - } - "Committee" => { - (res.offline_committees, res.online_committees) = - (record.offline, record.online); - } - "Watchtower" => { - (res.offline_watchtowers, res.online_watchtowers) = - (record.offline, record.online); - } - _ => {} - }; - } - Ok(res) - } - - pub async fn node_by_id(&mut self, peer_id: &str) -> anyhow::Result> { - let res = sqlx::query_as!(Node, r#"SELECT * FROM node WHERE peer_id = ?"#, peer_id) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn get_sum_bridge_in_txn( - &mut self, - statuses: &[String], - ) -> anyhow::Result<(i64, i64)> { - #[derive(sqlx::FromRow)] - struct BridgeInRow { - pub total: i64, - pub tx_count: i64, - } - - let query_str = format!( - "SELECT SUM(amount) AS total, COUNT(*) AS tx_count - FROM instance - WHERE status IN ({})", - create_place_holders(statuses) - ); - let mut query = sqlx::query_as::<_, BridgeInRow>(&query_str); - for status in statuses { - query = query.bind(status); - } - let record = query.fetch_one(self.conn()).await?; - Ok((record.total, record.tx_count)) - } - - pub async fn get_sum_peg_out(&mut self, statuses: &[String]) -> anyhow::Result<(i64, i64)> { - #[derive(sqlx::FromRow)] - struct BridgeOutRow { - pub total: i64, - pub tx_count: i64, - } - - let query_str = format!( - "SELECT SUM(amount) AS total, COUNT(*) AS tx_count - FROM graph - WHERE status IN - ({})", - create_place_holders(statuses) - ); - let mut query = sqlx::query_as::<_, BridgeOutRow>(&query_str); - for status in statuses { - query = query.bind(status); - } - let record = query.fetch_one(self.conn()).await?; - Ok((record.total, record.tx_count)) - } - - pub async fn get_nodes_info(&mut self, time_threshold: i64) -> anyhow::Result<(i64, i64)> { - let total = sqlx::query!(r#"SELECT COUNT(peer_id) AS total FROM node"#) - .fetch_one(self.conn()) - .await? - .total; - let time_pri = - SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as i64 - time_threshold; - tracing::info!("{time_pri}"); - let alive = sqlx::query!( - r#"SELECT COUNT(peer_id) AS alive FROM node WHERE updated_at >= ?"#, - time_pri - ) - .fetch_one(self.conn()) - .await? - .alive; - Ok((total, alive)) - } - - /// Complete the exact local queue claim handed to a worker. - pub async fn complete_local_message( - &mut self, - message_id: &str, - message_version: i64, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query( - "UPDATE message \ - SET state = 'Processed', content = X'', lock_time_until = 0, \ - abandon_count = 0, last_error = NULL, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state = 'Processing'", - ) - .bind(current_time) - .bind(message_id) - .bind(message_version) - .execute(self.conn()) - .await?; - - Ok(res.rows_affected() > 0) - } - - /// Cancel queued local work for a business object, including work which has - /// already been claimed. Claim completion/defer writes require `Processing`, - /// so an in-flight worker cannot overwrite the terminal cancellation. - pub async fn cancel_messages_by_business_id( - &mut self, - business_id: &Uuid, - msg_type: Option, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = match msg_type { - Some(msg_type) => { - sqlx::query( - "UPDATE message \ - SET state = 'Cancelled', lock_time_until = 0, updated_at = ? \ - WHERE business_id = ? AND msg_type = ? \ - AND state IN ('Pending', 'Processing')", - ) - .bind(current_time) - .bind(business_id) - .bind(msg_type) - .execute(self.conn()) - .await? - } - None => { - sqlx::query( - "UPDATE message \ - SET state = 'Cancelled', lock_time_until = 0, updated_at = ? \ - WHERE business_id = ? AND state IN ('Pending', 'Processing')", - ) - .bind(current_time) - .bind(business_id) - .execute(self.conn()) - .await? - } - }; - Ok(res.rows_affected()) - } - - pub async fn set_messages_expired(&mut self, expired: i64) -> anyhow::Result<()> { - sqlx::query!( - r#"UPDATE message - SET state = 'Expired' - WHERE state IN ('Pending', 'Processing') - AND updated_at < ?"#, - expired - ) - .execute(self.conn()) - .await?; - Ok(()) - } - - pub async fn delete_old_messages(&mut self, expired: i64) -> anyhow::Result<()> { - sqlx::query!(r#"DELETE FROM message WHERE updated_at < ?"#, expired) - .execute(self.conn()) - .await?; - Ok(()) - } - - pub async fn find_message_by_business_id( - &mut self, - business_id: &Uuid, - msg_type: &str, - ) -> anyhow::Result> { - let row = sqlx::query(&format!( - "SELECT {MESSAGE_COLUMNS} FROM message WHERE business_id = ? AND msg_type = ?" - )) - .bind(business_id) - .bind(msg_type) - .fetch_optional(self.conn()) - .await?; - Ok(row.map(|row| message_from_row(&row)).transpose()?) - } - pub async fn find_messages_by_id( - &mut self, - message_id: &str, - ) -> anyhow::Result> { - let row = - sqlx::query(&format!("SELECT {MESSAGE_COLUMNS} FROM message WHERE message_id = ?")) - .bind(message_id) - .fetch_optional(self.conn()) - .await?; - Ok(row.map(|row| message_from_row(&row)).transpose()?) - } - - /// Retire local messages whose claims keep failing to report an outcome. - /// - /// A surviving `message` row makes the producer treat the work as already - /// created, so retiring on repeated handler errors would strand it until the - /// reaper; repeated abandons mean the node went down mid-dispatch on this - /// payload, which retrying cannot fix. - pub async fn quarantine_local_messages( - &mut self, - now: i64, - max_abandons: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Quarantined', \ - abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ - lock_time_until = 0, \ - last_error = ?, updated_at = ? \ - WHERE state IN ('Pending', 'Processing') \ - AND lock_time_until <= ? \ - AND abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END >= ?", - ) - .bind(format!("quarantined: exceeded max abandoned claims ({max_abandons})")) - .bind(now) - .bind(now) - .bind(max_abandons) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Rows the local dispatcher may attempt right now, oldest first. - /// - /// Nothing is written here: the dispatcher claims each row with - /// [`Self::claim_local_message`] immediately before dispatching it. Claiming - /// a whole batch up front meant that an abort or kill mid-dispatch charged - /// an unfinished attempt to every row in the batch, so healthy messages - /// followed the poison message into quarantine. - pub async fn list_claimable_local_messages( - &mut self, - now: i64, - expired: i64, - limit: i64, - max_abandons: i64, - ) -> anyhow::Result> { - let rows = sqlx::query(&format!( - "SELECT {MESSAGE_COLUMNS} FROM message \ - WHERE state IN ('Pending', 'Processing') \ - AND lock_time_until <= ? \ - AND updated_at >= ? \ - AND abandon_count < ? \ - ORDER BY created_at ASC \ - LIMIT ?" - )) - .bind(now) - .bind(expired) - .bind(max_abandons) - .bind(limit) - .fetch_all(self.conn()) - .await?; - rows.iter().map(message_from_row).collect::, _>>().map_err(Into::into) - } - - /// Claim exactly one local message for dispatch. - /// - /// Without a durable claim, a handler that panicked left the row `Pending` - /// with its lock untouched, so the very next tick re-read it and panicked - /// again with no backoff at all. Charging the claim before dispatch means - /// the record survives an abort or a kill, not just an unwinding panic. - /// - /// Re-claiming a row that is still `Processing` means the previous attempt - /// never reported an outcome, which is charged as an abandon. - /// `message_version` is deliberately left alone: callers guard their - /// completion writes with the version they were handed, and bumping it here - /// would make every one of those writes miss. Returns `None` when the row - /// is no longer claimable: cancelled, re-armed under a new version, or - /// locked since it was listed. - pub async fn claim_local_message( - &mut self, - message_id: &str, - message_version: i64, - now: i64, - lease_until: i64, - ) -> anyhow::Result> { - let row = sqlx::query(&format!( - "UPDATE message \ - SET state = 'Processing', \ - abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ - lock_time_until = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? \ - AND state IN ('Pending', 'Processing') \ - AND lock_time_until <= ? \ - RETURNING {MESSAGE_COLUMNS}" - )) - .bind(lease_until) - .bind(now) - .bind(message_id) - .bind(message_version) - .bind(now) - .fetch_optional(self.conn()) - .await?; - Ok(row.map(|row| message_from_row(&row)).transpose()?) - } - - /// List and claim up to `limit` rows in one call. - /// - /// Production dispatchers claim one row at a time; this convenience exists - /// for tests and tooling that need a whole batch held under a lease. - pub async fn claim_local_messages( - &mut self, - now: i64, - lease_until: i64, - expired: i64, - limit: i64, - max_abandons: i64, - ) -> anyhow::Result> { - let candidates = - self.list_claimable_local_messages(now, expired, limit, max_abandons).await?; - let mut claimed = Vec::with_capacity(candidates.len()); - for candidate in candidates { - if let Some(message) = self - .claim_local_message( - &candidate.message_id, - candidate.message_version, - now, - lease_until, - ) - .await? - { - claimed.push(message); - } - } - Ok(claimed) - } - - /// Record a failed dispatch attempt and reschedule it with backoff. - /// - /// `attempt_count` is observability only. Only an unfinished claim increments - /// `abandon_count` and contributes to quarantine. This is for a handler - /// error reported to the dispatcher, which is a completed attempt; a handler - /// rescheduling its own row uses [`Self::self_defer_local_message`]. - pub async fn defer_local_message( - &mut self, - message_id: &str, - message_version: i64, - lock_time_until: i64, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Pending', attempt_count = attempt_count + 1, \ - abandon_count = 0, lock_time_until = ?, last_error = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state = 'Processing'", - ) - .bind(lock_time_until) - .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(message_version) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Reschedule the row a handler is currently running, at that handler's - /// own request. - /// - /// Unlike [`Self::defer_local_message`], the consecutive-abandon counter is - /// left untouched: the handler is still running and may yet panic, and - /// resetting here let a handler that reschedules itself and then panics - /// start every round from zero, so it never reached quarantine. The - /// dispatcher resets the counter with - /// [`Self::confirm_local_message_self_defer`] once the handler returned. - pub async fn self_defer_local_message( - &mut self, - message_id: &str, - message_version: i64, - lock_time_until: i64, - reason: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Pending', attempt_count = attempt_count + 1, \ - lock_time_until = ?, last_error = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state = 'Processing'", - ) - .bind(lock_time_until) - .bind(reason.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(message_version) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Acknowledge a self-deferred row once its handler has returned. - /// - /// Only at this point is the attempt known to have reported an outcome, so - /// only here does the consecutive-abandon counter reset. Returns `false` - /// when the row is not `Pending` under this claim version, which means it - /// was not self-deferred by the caller. - pub async fn confirm_local_message_self_defer( - &mut self, - message_id: &str, - message_version: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message SET abandon_count = 0, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state = 'Pending'", - ) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(message_version) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Record a handler panic before shutting down the process. Unlike a normal - /// defer, this increments the consecutive unfinished-attempt counter and - /// pushes the next attempt out by `backoff_secs` per recorded abandon, so a - /// supervisor restart cannot replay the payload at full speed. - /// - /// A handler may reschedule its own row and then panic, leaving the row - /// `Pending` already. The version guard still identifies the claim, so the - /// abandon is charged either way and the longer of the two delays wins. - pub async fn abandon_local_message( - &mut self, - message_id: &str, - message_version: i64, - now: i64, - backoff_secs: i64, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Pending', abandon_count = abandon_count + 1, \ - lock_time_until = MAX(lock_time_until, ? + ? * (abandon_count + 1)), \ - last_error = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? \ - AND state IN ('Processing', 'Pending')", - ) - .bind(now) - .bind(backoff_secs) - .bind(error.chars().take(1024).collect::()) - .bind(now) - .bind(message_id) - .bind(message_version) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Terminally drop a local message whose payload the node can never process. - pub async fn fail_local_message( - &mut self, - message_id: &str, - message_version: i64, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Failed', content = X'', lock_time_until = 0, \ - last_error = ?, updated_at = ? \ - WHERE message_id = ? AND message_version = ? AND state = 'Processing'", - ) - .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(message_version) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Release local claims during a graceful process shutdown. The database is - /// process-local, so every `Processing` row belongs to this node process. - pub async fn release_processing_local_messages(&mut self) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Pending', lock_time_until = 0, updated_at = ? \ - WHERE state = 'Processing'", - ) - .bind(get_current_timestamp_secs()) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Startup sweep for claims left behind by a process that no longer exists. - /// - /// The database is process-local, so at startup every `Processing` row is - /// an attempt that never reported an outcome. Charging it now, rather than - /// when the lease lapses, keeps the row from sitting locked for the whole - /// lease while every producer that touches it backs off with - /// ResourceLocked. The same per-abandon backoff as a panic applies. - pub async fn reclaim_processing_local_messages( - &mut self, - now: i64, - backoff_secs: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE message \ - SET state = 'Pending', abandon_count = abandon_count + 1, \ - lock_time_until = ? + ? * (abandon_count + 1), \ - last_error = 'reclaimed at startup: previous process exited mid-dispatch', \ - updated_at = ? \ - WHERE state = 'Processing'", - ) - .bind(now) - .bind(backoff_secs) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - pub async fn get_message_queue_stats( - &mut self, - actor: &str, - now: i64, - ) -> anyhow::Result { - let row = sqlx::query( - r#"SELECT - COALESCE(SUM(CASE WHEN state = 'Pending' AND lock_time_until <= ? THEN 1 ELSE 0 END), 0) AS pending_ready, - COALESCE(SUM(CASE WHEN state = 'Pending' AND lock_time_until > ? THEN 1 ELSE 0 END), 0) AS pending_locked, - COALESCE(SUM(CASE WHEN state IN ('Failed', 'Quarantined') THEN 1 ELSE 0 END), 0) AS failed, - MIN(CASE WHEN state = 'Pending' THEN created_at END) AS oldest_pending_at - FROM message - WHERE actor = ?"#, - ) - .bind(now) - .bind(now) - .bind(actor) - .fetch_one(self.conn()) - .await?; - Ok(MessageQueueStats { - pending_ready: row.try_get("pending_ready")?, - pending_locked: row.try_get("pending_locked")?, - failed: row.try_get("failed")?, - oldest_pending_at: row.try_get("oldest_pending_at")?, - }) - } - - pub async fn find_message_debug_overviews( - &mut self, - business_id: &Uuid, - ) -> anyhow::Result> { - Ok(sqlx::query_as::<_, MessageDebugOverview>( - r#"SELECT m.message_id, - m.actor, - m.msg_type, - m.state, - m.lock_time_until, - m.created_at, - m.updated_at, - COALESCE(reason_counts.reason_count, 0) AS reason_count, - latest_reason.reason_code AS last_reason_code, - latest_reason.reason_detail AS last_reason_detail, - latest_reason.last_seen_at AS last_reason_seen_at - FROM message m - LEFT JOIN ( - SELECT message_id, COUNT(*) AS reason_count - FROM message_debug_reason - GROUP BY message_id - ) reason_counts ON reason_counts.message_id = m.message_id - LEFT JOIN message_debug_reason latest_reason - ON latest_reason.rowid = ( - SELECT rowid - FROM message_debug_reason - WHERE message_id = m.message_id - ORDER BY last_seen_at DESC, occurrences DESC, reason_code ASC - LIMIT 1 - ) - WHERE m.business_id = ? - ORDER BY m.updated_at DESC"#, - ) - .bind(business_id) - .fetch_all(self.conn()) - .await?) - } - - pub async fn find_message_debug_overview( - &mut self, - message_id: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as::<_, MessageDebugOverview>( - r#"SELECT m.message_id, - m.actor, - m.msg_type, - m.state, - m.lock_time_until, - m.created_at, - m.updated_at, - COALESCE(reason_counts.reason_count, 0) AS reason_count, - latest_reason.reason_code AS last_reason_code, - latest_reason.reason_detail AS last_reason_detail, - latest_reason.last_seen_at AS last_reason_seen_at - FROM message m - LEFT JOIN ( - SELECT message_id, COUNT(*) AS reason_count - FROM message_debug_reason - GROUP BY message_id - ) reason_counts ON reason_counts.message_id = m.message_id - LEFT JOIN message_debug_reason latest_reason - ON latest_reason.rowid = ( - SELECT rowid - FROM message_debug_reason - WHERE message_id = m.message_id - ORDER BY last_seen_at DESC, occurrences DESC, reason_code ASC - LIMIT 1 - ) - WHERE m.message_id = ?"#, - ) - .bind(message_id) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn find_message_debug_reasons( - &mut self, - message_id: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as::<_, MessageDebugReason>( - "SELECT reason_code, reason_detail, first_seen_at, last_seen_at, occurrences \ - FROM message_debug_reason WHERE message_id = ? \ - ORDER BY last_seen_at DESC, reason_code ASC", - ) - .bind(message_id) - .fetch_all(self.conn()) - .await?) - } - - pub async fn upsert_message_debug_reason( - &mut self, - message_id: &str, - reason_code: &str, - reason_detail: &str, - ) -> anyhow::Result<()> { - let now = get_current_timestamp_secs(); - sqlx::query( - "INSERT INTO message_debug_reason \ - (message_id, reason_code, reason_detail, first_seen_at, last_seen_at, occurrences) \ - VALUES (?, ?, ?, ?, ?, 1) \ - ON CONFLICT(message_id, reason_code, reason_detail) DO UPDATE SET \ - last_seen_at = excluded.last_seen_at, \ - occurrences = message_debug_reason.occurrences + 1", - ) - .bind(message_id) - .bind(reason_code) - .bind(reason_detail.chars().take(512).collect::()) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - Ok(()) - } - - /// Insert a local message, or refresh an existing one. - /// - /// Terminal states are excluded so a retired row stays retired. - /// [`Self::fail_local_message`] and [`Self::quarantine_local_messages`] - /// both mean the payload must not run again automatically; - /// without the exclusion a periodic producer re-upserting the same - /// deterministic message_id would resurrect it on every tick. - pub async fn upsert_message(&mut self, msg: Message) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query( - r#"INSERT INTO message (message_id, business_id, from_peer, actor, msg_type, content, state, message_version, lock_time_until, weight, updated_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(message_id) DO UPDATE SET business_id = excluded.business_id, - from_peer = excluded.from_peer, - actor = excluded.actor, - msg_type = excluded.msg_type, - content = excluded.content, - state = excluded.state, - message_version = message.message_version + 1, - lock_time_until = excluded.lock_time_until, - weight = excluded.weight, - last_error = NULL, - updated_at = excluded.updated_at - WHERE message.state NOT IN ('Processing', 'Cancelled', 'Failed', 'Quarantined')"#, - ) - .bind(msg.message_id) - .bind(msg.business_id) - .bind(msg.from_peer) - .bind(msg.actor) - .bind(msg.msg_type) - .bind(msg.content) - .bind(msg.state) - .bind(msg.message_version) - .bind(msg.lock_time_until) - .bind(msg.weight) - .bind(current_time) - .bind(current_time) - .execute(self.conn()) - .await?; - Ok(res.rows_affected() > 0) - } - - /// Persist an externally received P2P message before it is dispatched. - /// Replays of the same gossipsub message are deliberately ignored so a - /// terminal row cannot be repopulated with its (potentially large) content. - pub async fn insert_p2p_inbox_message( - &mut self, - message: &P2pInboxMessage, - ) -> anyhow::Result { - let now = get_current_timestamp_secs(); - let admission_class = if message.admission_class.is_empty() { - P2pInboxAdmissionClass::Unregistered.to_string() - } else { - message.admission_class.clone() - }; - let result = sqlx::query( - "INSERT INTO p2p_inbox \ - (message_id, business_id, actor, from_peer, msg_type, content, content_size, \ - state, attempt_count, next_retry_at, lease_until, lease_token, \ - admission_class, content_hash, created_at, updated_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?, 'Pending', 0, 0, 0, '', ?, ?, ?, ?) \ - ON CONFLICT(message_id) DO NOTHING", - ) - .bind(&message.message_id) - .bind(message.business_id) - .bind(&message.actor) - .bind(&message.from_peer) - .bind(&message.msg_type) - .bind(&message.content) - .bind(message.content_size) - .bind(admission_class) - .bind(&message.content_hash) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Test oracle for class totals and per-peer usage, including quarantined payloads. - #[cfg(test)] - async fn p2p_inbox_usage( - &mut self, - from_peer: &str, - ) -> anyhow::Result> { - let rows = sqlx::query( - "SELECT admission_class, \ - COUNT(*) AS total_rows, \ - COALESCE(SUM(content_size), 0) AS total_bytes, \ - COALESCE(SUM(CASE WHEN from_peer = ? THEN 1 ELSE 0 END), 0) AS peer_rows, \ - COALESCE(SUM(CASE WHEN from_peer = ? THEN content_size ELSE 0 END), 0) \ - AS peer_bytes \ - FROM p2p_inbox \ - WHERE state IN ('Pending', 'Processing', 'Quarantined') \ - GROUP BY admission_class", - ) - .bind(from_peer) - .bind(from_peer) - .fetch_all(self.conn()) - .await?; - rows.iter() - .map(|row| { - Ok(P2pInboxClassUsage { - admission_class: row.try_get("admission_class")?, - rows: row.try_get("total_rows")?, - bytes: row.try_get("total_bytes")?, - peer_rows: row.try_get("peer_rows")?, - peer_bytes: row.try_get("peer_bytes")?, - }) - }) - .collect::, sqlx::Error>>() - .map_err(Into::into) - } - - /// Per-class queued payload totals, without per-peer usage. - pub async fn p2p_inbox_class_totals(&mut self) -> anyhow::Result> { - let rows = sqlx::query( - "SELECT admission_class, \ - COUNT(*) AS total_rows, \ - COALESCE(SUM(content_size), 0) AS total_bytes \ - FROM p2p_inbox \ - WHERE state IN ('Pending', 'Processing', 'Quarantined') \ - GROUP BY admission_class", - ) - .fetch_all(self.conn()) - .await?; - rows.iter() - .map(|row| { - Ok(P2pInboxClassUsage { - admission_class: row.try_get("admission_class")?, - rows: row.try_get("total_rows")?, - bytes: row.try_get("total_bytes")?, - ..Default::default() - }) - }) - .collect::, sqlx::Error>>() - .map_err(Into::into) - } - - /// Per-peer payload usage by class; `rows` and `bytes` remain zero. - pub async fn p2p_inbox_peer_usage( - &mut self, - from_peer: &str, - ) -> anyhow::Result> { - let rows = sqlx::query( - "SELECT admission_class, \ - COUNT(*) AS peer_rows, \ - COALESCE(SUM(content_size), 0) AS peer_bytes \ - FROM p2p_inbox \ - WHERE state IN ('Pending', 'Processing', 'Quarantined') \ - AND admission_class IN ('Committee', 'Registered', 'Unregistered') \ - AND from_peer = ? \ - GROUP BY admission_class", - ) - .bind(from_peer) - .fetch_all(self.conn()) - .await?; - rows.iter() - .map(|row| { - Ok(P2pInboxClassUsage { - admission_class: row.try_get("admission_class")?, - peer_rows: row.try_get("peer_rows")?, - peer_bytes: row.try_get("peer_bytes")?, - ..Default::default() - }) - }) - .collect::, sqlx::Error>>() - .map_err(Into::into) - } - - /// Whether the sender already has this payload in a non-terminal row. - pub async fn has_queued_p2p_inbox_payload( - &mut self, - from_peer: &str, - content_hash: &[u8], - ) -> anyhow::Result { - let row = sqlx::query( - "SELECT 1 FROM p2p_inbox \ - WHERE from_peer = ? AND content_hash = ? AND state IN ('Pending', 'Processing') \ - LIMIT 1", - ) - .bind(from_peer) - .bind(content_hash) - .fetch_optional(self.conn()) - .await?; - Ok(row.is_some()) - } - - /// Expire Pending rows by `created_at`; `None` selects every admission class. - pub async fn expire_pending_p2p_inbox_messages( - &mut self, - admission_class: Option, - created_before: i64, - ) -> anyhow::Result { - let class_predicate = - if admission_class.is_some() { " AND admission_class = ?" } else { "" }; - let query = format!( - "UPDATE p2p_inbox \ - SET state = 'Failed', content = X'', lease_until = 0, next_retry_at = 0, \ - last_error = 'expired: still pending past the inbox retention window', \ - updated_at = ? \ - WHERE state = 'Pending' AND created_at < ?{class_predicate}" - ); - let mut query = sqlx::query(&query).bind(get_current_timestamp_secs()).bind(created_before); - if let Some(admission_class) = admission_class { - query = query.bind(admission_class.to_string()); - } - Ok(query.execute(self.conn()).await?.rows_affected()) - } - - /// Move inbox rows that repeatedly abandoned a claim out of the work set. - /// - /// `attempt_count` remains a pure diagnostic counter. Only `abandon_count` - /// is a poison-message signal: the claim - /// charge is committed before dispatch, so it is recorded even when the - /// process is aborted or killed rather than unwinding. - /// - /// Content is retained until the terminal-row TTL so an operator can inspect - /// and manually requeue the message. - pub async fn quarantine_p2p_inbox_messages( - &mut self, - now: i64, - max_abandons: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Quarantined', \ - abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ - lease_until = 0, next_retry_at = 0, \ - last_error = ?, updated_at = ? \ - WHERE state IN ('Pending', 'Processing') \ - AND lease_until <= ? \ - AND abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END >= ?", - ) - .bind(format!("quarantined: exceeded max abandoned claims ({max_abandons})")) - .bind(now) - .bind(now) - .bind(max_abandons) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Inbox rows the dispatcher may attempt right now, oldest first. Nothing - /// is written; see [`Self::list_claimable_local_messages`] for why rows are - /// claimed one at a time instead of as a batch. - pub async fn list_claimable_p2p_inbox_messages( - &mut self, - now: i64, - limit: i64, - max_abandons: i64, - excluded_message_ids: &[String], - ) -> anyhow::Result> { - self.list_claimable_p2p_inbox_messages_in( - now, - limit, - 0, - max_abandons, - excluded_message_ids, - "", - ) - .await - } - - /// List each class's reserved share, then lend unused capacity in C/R/U order. - pub async fn list_claimable_p2p_inbox_messages_by_class( - &mut self, - now: i64, - limit: i64, - registered_reserve: i64, - unregistered_reserve: i64, - max_abandons: i64, - excluded_message_ids: &[String], - ) -> anyhow::Result> { - const COMMITTEE: &str = " AND admission_class = 'Committee'"; - const REGISTERED: &str = " AND admission_class = 'Registered'"; - const UNREGISTERED: &str = " AND admission_class NOT IN ('Committee', 'Registered')"; - - let limit = limit.max(0); - let unregistered_reserve = unregistered_reserve.clamp(0, limit); - let registered_reserve = registered_reserve.clamp(0, limit - unregistered_reserve); - let shares = [ - (COMMITTEE, limit - registered_reserve - unregistered_reserve), - (REGISTERED, registered_reserve), - (UNREGISTERED, unregistered_reserve), - ]; - - // Guaranteed shares first ... - let mut rows: [Vec; 3] = Default::default(); - for (index, (class, share)) in shares.iter().enumerate() { - rows[index] = self - .list_claimable_p2p_inbox_messages_in( - now, - *share, - 0, - max_abandons, - excluded_message_ids, - class, - ) - .await?; - } - // ... then whatever a class left of its share is lent out, in priority - // order, to the classes that have more to claim. - for (index, (class, _)) in shares.iter().enumerate() { - let free = limit - rows.iter().map(|rows| rows.len() as i64).sum::(); - let more = self - .list_claimable_p2p_inbox_messages_in( - now, - free, - rows[index].len() as i64, - max_abandons, - excluded_message_ids, - class, - ) - .await?; - rows[index].extend(more); - } - Ok(rows.into_iter().flatten().collect()) - } - - async fn list_claimable_p2p_inbox_messages_in( - &mut self, - now: i64, - limit: i64, - offset: i64, - max_abandons: i64, - excluded_message_ids: &[String], - class_predicate: &'static str, - ) -> anyhow::Result> { - if limit <= 0 { - return Ok(Vec::new()); - } - let excluded_predicate = if excluded_message_ids.is_empty() { - String::new() - } else { - format!(" AND message_id NOT IN ({})", create_place_holders(excluded_message_ids)) - }; - let query = format!( - "SELECT {P2P_INBOX_COLUMNS} \ - FROM p2p_inbox \ - WHERE ((state = 'Pending' AND next_retry_at <= ?) \ - OR (state = 'Processing' AND lease_until <= ?)) \ - AND abandon_count < ?{excluded_predicate}{class_predicate} \ - ORDER BY created_at ASC, message_id ASC \ - LIMIT ? OFFSET ?" - ); - let mut query = sqlx::query(&query).bind(now).bind(now).bind(max_abandons); - for message_id in excluded_message_ids { - query = query.bind(message_id); - } - let rows = query.bind(limit).bind(offset).fetch_all(self.conn()).await?; - rows.iter() - .map(p2p_inbox_message_from_row) - .collect::, _>>() - .map_err(Into::into) - } - - /// Claim exactly one inbox row under a fresh lease token. - /// - /// Re-claiming a row that is still `Processing` means the previous attempt - /// never reported an outcome: the worker panicked, the process died, or it - /// hung past the lease. That is charged separately from an ordinary handler - /// error so that a transient outage cannot push healthy messages toward - /// quarantine. Returns `None` when the row is no longer claimable. - pub async fn claim_p2p_inbox_message( - &mut self, - message_id: &str, - now: i64, - lease_until: i64, - ) -> anyhow::Result> { - let lease_token = Uuid::new_v4().to_string(); - let row = sqlx::query(&format!( - "UPDATE p2p_inbox \ - SET state = 'Processing', attempt_count = attempt_count + 1, \ - abandon_count = abandon_count + CASE WHEN state = 'Processing' THEN 1 ELSE 0 END, \ - lease_until = ?, lease_token = ?, updated_at = ? \ - WHERE message_id = ? \ - AND ((state = 'Pending' AND next_retry_at <= ?) \ - OR (state = 'Processing' AND lease_until <= ?)) \ - RETURNING {P2P_INBOX_COLUMNS}" - )) - .bind(lease_until) - .bind(&lease_token) - .bind(now) - .bind(message_id) - .bind(now) - .bind(now) - .fetch_optional(self.conn()) - .await?; - Ok(row.map(|row| p2p_inbox_message_from_row(&row)).transpose()?) - } - - /// List and claim up to `limit` inbox rows in one call. Production - /// dispatchers claim one row at a time; this is for tests and tooling. - pub async fn claim_p2p_inbox_messages( - &mut self, - now: i64, - lease_until: i64, - limit: i64, - max_abandons: i64, - excluded_message_ids: &[String], - ) -> anyhow::Result> { - let candidates = self - .list_claimable_p2p_inbox_messages(now, limit, max_abandons, excluded_message_ids) - .await?; - let mut claimed = Vec::with_capacity(candidates.len()); - for candidate in candidates { - if let Some(message) = - self.claim_p2p_inbox_message(&candidate.message_id, now, lease_until).await? - { - claimed.push(message); - } - } - Ok(claimed) - } - - pub async fn complete_p2p_inbox_message( - &mut self, - message_id: &str, - lease_token: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Processed', content = X'', lease_until = 0, next_retry_at = 0, \ - updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", - ) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(lease_token) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn retry_p2p_inbox_message( - &mut self, - message_id: &str, - lease_token: &str, - next_retry_at: i64, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Pending', abandon_count = 0, lease_until = 0, \ - next_retry_at = ?, last_error = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", - ) - .bind(next_retry_at) - .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(lease_token) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Record a panic from the current lease before terminating the process. - /// The next attempt is pushed out by `backoff_secs` per recorded abandon so - /// a supervisor restart cannot replay the payload at full speed. - pub async fn abandon_p2p_inbox_message( - &mut self, - message_id: &str, - lease_token: &str, - now: i64, - backoff_secs: i64, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Pending', abandon_count = abandon_count + 1, lease_until = 0, \ - lease_token = '', next_retry_at = ? + ? * (abandon_count + 1), \ - last_error = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", - ) - .bind(now) - .bind(backoff_secs) - .bind(error.chars().take(1024).collect::()) - .bind(now) - .bind(message_id) - .bind(lease_token) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Return claimed work to the queue without charging it as a processing - /// attempt. This is used when capacity is unavailable before dispatch. - pub async fn defer_p2p_inbox_message( - &mut self, - message_id: &str, - lease_token: &str, - next_retry_at: i64, - reason: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Pending', attempt_count = MAX(attempt_count - 1, 0), lease_until = 0, \ - next_retry_at = ?, last_error = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", - ) - .bind(next_retry_at) - .bind(reason) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(lease_token) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn fail_p2p_inbox_message( - &mut self, - message_id: &str, - lease_token: &str, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Failed', content = X'', lease_until = 0, next_retry_at = 0, \ - last_error = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", - ) - .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(lease_token) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Drop terminal inbox rows once they are older than `expired_before`. - /// - /// Processed/failed payloads are already released. Quarantined payloads are - /// retained only for this bounded inspection/requeue window. Anything still - /// claimable is left alone. - pub async fn purge_terminal_p2p_inbox_messages( - &mut self, - expired_before: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "DELETE FROM p2p_inbox \ - WHERE state IN ('Processed', 'Failed', 'Quarantined') AND updated_at < ?", - ) - .bind(expired_before) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Delete the oldest terminal rows beyond `max_rows`, up to `limit` per call. - pub async fn purge_p2p_inbox_over_terminal_cap( - &mut self, - max_rows: i64, - limit: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "DELETE FROM p2p_inbox WHERE message_id IN ( \ - SELECT message_id FROM p2p_inbox \ - WHERE state IN ('Processed', 'Failed', 'Quarantined') \ - ORDER BY updated_at ASC \ - LIMIT MIN(?, MAX( \ - (SELECT COUNT(*) FROM p2p_inbox \ - WHERE state IN ('Processed', 'Failed', 'Quarantined')) - ?, 0)))", - ) - .bind(limit) - .bind(max_rows) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Clear the oldest Quarantined payloads until retained bytes are within `max_bytes`. - pub async fn trim_quarantined_p2p_inbox_payloads( - &mut self, - max_bytes: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox SET content = X'', content_size = 0, updated_at = ? \ - WHERE state = 'Quarantined' AND length(content) > 0 AND message_id IN ( \ - SELECT message_id FROM ( \ - SELECT message_id, \ - SUM(content_size) OVER ( \ - ORDER BY updated_at DESC \ - ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running \ - FROM p2p_inbox WHERE state = 'Quarantined' AND length(content) > 0 \ - ) WHERE running - content_size >= ?)", - ) - .bind(get_current_timestamp_secs()) - .bind(max_bytes) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Load up to `limit` operator bindings, confirmed operators first. - /// The caller re-verifies signatures. - pub async fn load_p2p_peer_bindings(&mut self, limit: i64) -> anyhow::Result> { - let rows = sqlx::query_as::<_, Node>( - "SELECT * FROM node \ - WHERE binding_sig != '' AND btc_pub_key != '' AND actor = 'Operator' \ - ORDER BY peer_id IN (SELECT peer_id FROM p2p_registered_peer \ - WHERE kind = 'Operator') DESC, \ - updated_at DESC \ - LIMIT ?", - ) - .bind(limit) - .fetch_all(self.conn()) - .await?; - Ok(rows) - } - - /// Persist confirmed registration; an operator key may belong to only one peer. - pub async fn upsert_p2p_registered_peer( - &mut self, - peer_id: &str, - kind: &str, - pubkey: &str, - ) -> anyhow::Result<()> { - if !pubkey.is_empty() { - sqlx::query( - "DELETE FROM p2p_registered_peer WHERE kind = ? AND pubkey = ? AND peer_id != ?", - ) - .bind(kind) - .bind(pubkey) - .bind(peer_id) - .execute(self.conn()) - .await?; - } - sqlx::query( - "INSERT INTO p2p_registered_peer (peer_id, kind, pubkey, verified_at) \ - VALUES (?, ?, ?, ?) \ - ON CONFLICT (peer_id, kind) DO UPDATE SET pubkey = excluded.pubkey, \ - verified_at = excluded.verified_at", - ) - .bind(peer_id) - .bind(kind) - .bind(pubkey) - .bind(get_current_timestamp_secs()) - .execute(self.conn()) - .await?; - Ok(()) - } - - pub async fn delete_p2p_registered_peer( - &mut self, - peer_id: &str, - kind: &str, - ) -> anyhow::Result<()> { - sqlx::query("DELETE FROM p2p_registered_peer WHERE peer_id = ? AND kind = ?") - .bind(peer_id) - .bind(kind) - .execute(self.conn()) - .await?; - Ok(()) - } - - /// `(peer_id, kind, pubkey)` of every peer confirmed in an earlier session. - pub async fn load_p2p_registered_peers( - &mut self, - ) -> anyhow::Result> { - let rows = sqlx::query("SELECT peer_id, kind, pubkey FROM p2p_registered_peer") - .fetch_all(self.conn()) - .await?; - rows.iter() - .map(|row| Ok((row.try_get("peer_id")?, row.try_get("kind")?, row.try_get("pubkey")?))) - .collect::, sqlx::Error>>() - .map_err(Into::into) - } - - /// Raise the persisted replay marks to `marks` (`peer_id -> highest sequence - /// number admitted`). A mark never moves down. - pub async fn raise_p2p_replay_marks(&mut self, marks: &[(String, i64)]) -> anyhow::Result<()> { - for (peer_id, highest) in marks { - sqlx::query( - "INSERT INTO p2p_replay_mark (peer_id, highest) VALUES (?, ?) \ - ON CONFLICT (peer_id) DO UPDATE SET highest = MAX(highest, excluded.highest)", - ) - .bind(peer_id) - .bind(highest) - .execute(self.conn()) - .await?; - } - Ok(()) - } - - pub async fn load_p2p_replay_marks(&mut self) -> anyhow::Result> { - let rows = sqlx::query("SELECT peer_id, highest FROM p2p_replay_mark") - .fetch_all(self.conn()) - .await?; - rows.iter() - .map(|row| Ok((row.try_get("peer_id")?, row.try_get("highest")?))) - .collect::, sqlx::Error>>() - .map_err(Into::into) - } - - /// Remove marks for unregistered peers, retaining verifier peer IDs in `keep`. - pub async fn prune_p2p_replay_marks(&mut self, keep: &[String]) -> anyhow::Result { - let keep_predicate = if keep.is_empty() { - String::new() - } else { - format!(" AND peer_id NOT IN ({})", create_place_holders(keep)) - }; - let query = format!( - "DELETE FROM p2p_replay_mark \ - WHERE peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer){keep_predicate}" - ); - let mut query = sqlx::query(&query); - for peer_id in keep { - query = query.bind(peer_id); - } - Ok(query.execute(self.conn()).await?.rows_affected()) - } - - /// Allow existing peers or a new unregistered row below `max_rows`. - /// At capacity, evict the oldest unregistered row older than `evict_stale_before`. - pub async fn node_row_admissible( - &mut self, - peer_id: &str, - max_rows: i64, - evict_stale_before: i64, - local_peer_id: &str, - ) -> anyhow::Result { - let row = sqlx::query( - "SELECT EXISTS(SELECT 1 FROM node WHERE peer_id = ?) AS known, \ - (SELECT COUNT(*) FROM node \ - WHERE peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer)) AS total", - ) - .bind(peer_id) - .fetch_one(self.conn()) - .await?; - let known: i64 = row.try_get("known")?; - let total: i64 = row.try_get("total")?; - if known != 0 || total < max_rows { - return Ok(true); - } - let evicted = sqlx::query( - "DELETE FROM node WHERE peer_id = ( \ - SELECT peer_id FROM node \ - WHERE updated_at < ? AND peer_id != ? \ - AND peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer) \ - ORDER BY updated_at ASC LIMIT 1)", - ) - .bind(evict_stale_before) - .bind(local_peer_id) - .execute(self.conn()) - .await?; - Ok(evicted.rows_affected() > 0) - } - - /// Drop the persisted replay marks of `peer_ids`. - pub async fn delete_p2p_replay_marks(&mut self, peer_ids: &[String]) -> anyhow::Result { - if peer_ids.is_empty() { - return Ok(0); - } - let query = format!( - "DELETE FROM p2p_replay_mark WHERE peer_id IN ({})", - create_place_holders(peer_ids) - ); - let mut query = sqlx::query(&query); - for peer_id in peer_ids { - query = query.bind(peer_id); - } - Ok(query.execute(self.conn()).await?.rows_affected()) - } - - /// Delete stale node rows except this node and confirmed registered peers. - pub async fn purge_stale_unregistered_nodes( - &mut self, - updated_before: i64, - local_peer_id: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "DELETE FROM node \ - WHERE updated_at < ? AND peer_id != ? \ - AND peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer)", - ) - .bind(updated_before) - .bind(local_peer_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - pub async fn renew_p2p_inbox_lease( - &mut self, - message_id: &str, - lease_token: &str, - lease_until: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox SET lease_until = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND lease_token = ?", - ) - .bind(lease_until) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(lease_token) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Release inbox claims during a graceful process shutdown without charging - /// them as abandoned executions. - pub async fn release_processing_p2p_inbox_messages(&mut self) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Pending', lease_until = 0, lease_token = '', \ - next_retry_at = 0, updated_at = ? \ - WHERE state = 'Processing'", - ) - .bind(get_current_timestamp_secs()) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// Startup sweep for inbox claims left behind by a process that no longer - /// exists. See [`Self::reclaim_processing_local_messages`]. - pub async fn reclaim_processing_p2p_inbox_messages( - &mut self, - now: i64, - backoff_secs: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Pending', abandon_count = abandon_count + 1, lease_until = 0, \ - lease_token = '', next_retry_at = ? + ? * (abandon_count + 1), \ - last_error = 'reclaimed at startup: previous process exited mid-dispatch', \ - updated_at = ? \ - WHERE state = 'Processing'", - ) - .bind(now) - .bind(backoff_secs) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - pub async fn requeue_p2p_inbox_message(&mut self, message_id: &str) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_inbox \ - SET state = 'Pending', abandon_count = 0, next_retry_at = 0, lease_until = 0, \ - lease_token = '', last_error = NULL, updated_at = ? \ - WHERE message_id = ? AND state = 'Quarantined' AND length(content) > 0", - ) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn insert_p2p_outbox_message( - &mut self, - message_id: &str, - msg_type: &str, - content: &[u8], - ) -> anyhow::Result { - let now = get_current_timestamp_secs(); - let result = sqlx::query( - "INSERT INTO p2p_outbox \ - (message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, created_at, updated_at) \ - VALUES (?, ?, ?, 'Pending', 0, 0, 0, ?, ?) \ - ON CONFLICT(message_id) DO NOTHING", - ) - .bind(message_id) - .bind(msg_type) - .bind(content) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Enqueue an outbound message, allowing a terminal message to be - /// deliberately announced again while preserving an in-flight attempt. - pub async fn enqueue_p2p_outbox_message( - &mut self, - message_id: &str, - msg_type: &str, - content: &[u8], - ) -> anyhow::Result { - let now = get_current_timestamp_secs(); - let result = sqlx::query( - "INSERT INTO p2p_outbox \ - (message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, created_at, updated_at) \ - VALUES (?, ?, ?, 'Pending', 0, 0, 0, ?, ?) \ - ON CONFLICT(message_id) DO UPDATE SET \ - msg_type = excluded.msg_type, content = excluded.content, state = 'Pending', \ - attempt_count = 0, next_retry_at = 0, lease_until = 0, last_error = NULL, updated_at = excluded.updated_at \ - WHERE p2p_outbox.state IN ('Processed', 'Failed')", - ) - .bind(message_id) - .bind(msg_type) - .bind(content) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Queue a graph-setup notification for a bounded retransmission window. - /// - /// A non-empty `ack_peer_id` is the only ACK that may stop this retry - /// loop early. Broadcast messages without an authenticated recipient keep - /// publishing until their window expires. - /// `first_publish_at`: first outbox delivery time; zero makes the row immediately due. - #[allow(clippy::too_many_arguments)] - pub async fn enqueue_p2p_outbox_retry_message( - &mut self, - message_id: &str, - delivery_id: Option<&str>, - msg_type: &str, - content: &[u8], - retry_until: i64, - retry_interval_secs: i64, - ack_peer_id: Option<&str>, - first_publish_at: i64, - ) -> anyhow::Result { - let now = get_current_timestamp_secs(); - let result = sqlx::query( - "INSERT INTO p2p_outbox \ - (message_id, delivery_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, retry_until, retry_interval_secs, ack_peer_id, created_at, updated_at) \ - VALUES (?, ?, ?, ?, 'Pending', 0, ?, 0, ?, ?, ?, ?, ?) \ - ON CONFLICT(message_id) DO NOTHING", - ) - .bind(message_id) - .bind(delivery_id.unwrap_or_default()) - .bind(msg_type) - .bind(content) - .bind(first_publish_at) - .bind(retry_until) - .bind(retry_interval_secs) - .bind(ack_peer_id.unwrap_or_default()) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Claim due outbox rows in ascending `next_retry_at` order. - pub async fn claim_p2p_outbox_messages( - &mut self, - now: i64, - lease_until: i64, - limit: i64, - ) -> anyhow::Result> { - let rows = sqlx::query( - "SELECT message_id, msg_type, content, state, attempt_count, next_retry_at, lease_until, last_error, retry_until, retry_interval_secs, ack_peer_id, publish_count, created_at \ - FROM p2p_outbox \ - WHERE ((state = 'Pending' AND next_retry_at <= ?) \ - OR (state = 'Processing' AND lease_until <= ?) \ - ) AND (retry_until = 0 OR retry_until > ?) \ - ORDER BY next_retry_at ASC, created_at ASC LIMIT ?", - ) - .bind(now) - .bind(now) - .bind(now) - .bind(limit) - .fetch_all(self.conn()) - .await?; - let mut claimed = Vec::with_capacity(rows.len()); - for row in rows { - let mut message = p2p_outbox_message_from_row(&row)?; - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Processing', attempt_count = attempt_count + 1, lease_until = ?, updated_at = ? \ - WHERE message_id = ? AND ((state = 'Pending' AND next_retry_at <= ?) \ - OR (state = 'Processing' AND lease_until <= ?))", - ) - .bind(lease_until) - .bind(now) - .bind(&message.message_id) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - if result.rows_affected() > 0 { - message.state = "Processing".to_owned(); - message.attempt_count += 1; - message.lease_until = lease_until; - claimed.push(message); - } - } - Ok(claimed) - } - - /// End a bounded-retry message once its delivery window is exhausted. - pub async fn expire_p2p_outbox_retry_messages(&mut self, now: i64) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'RetryExhausted', \ - content = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN content ELSE X'' END, \ - lease_until = 0, next_retry_at = 0, \ - retry_until = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN retry_until ELSE 0 END, \ - retry_interval_secs = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN retry_interval_secs ELSE 0 END, \ - ack_peer_id = CASE WHEN msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') THEN ack_peer_id ELSE '' END, \ - last_error = 'retry window expired without expected ACK', updated_at = ? \ - WHERE retry_until > 0 AND retry_until <= ? AND state IN ('Pending', 'Processing')", - ) - .bind(now) - .bind(now) - .execute(self.conn()) - .await?; - Ok(result.rows_affected()) - } - - /// At most one retained setup payload per pass, oldest recovery first. - pub async fn next_exhausted_setup( - &mut self, - now: i64, - ) -> anyhow::Result, i64, String)>> { - Ok(sqlx::query_as( - "SELECT message_id, content, created_at, ack_peer_id FROM p2p_outbox \ - WHERE state = 'RetryExhausted' AND length(content) > 0 \ - AND msg_type IN ('InitGraph','GenCircuits','CutCircuits','SolderingProofReady','CreateGraph') \ - AND updated_at <= ? ORDER BY updated_at, message_id LIMIT 1", - ) - .bind(now - 60) - .fetch_optional(self.conn()) - .await?) - } - - /// Called inside an immediate transaction: transfer bytes, never reopen a terminal row. - pub async fn recover_setup_delivery( - &mut self, - id: &str, - canonical_id: &str, - now: i64, - window: i64, - resume: bool, - ) -> anyhow::Result<()> { - if resume { - let next_id = format!("{canonical_id}:recovery:{now}"); - sqlx::query( - "INSERT INTO p2p_outbox (message_id,delivery_id,msg_type,content,state,attempt_count,next_retry_at,lease_until, \ - retry_until,retry_interval_secs,ack_peer_id,publish_count,created_at,updated_at) \ - SELECT ?,delivery_id,msg_type,content,'Pending',0,?,0,?,retry_interval_secs,ack_peer_id,publish_count,created_at,? \ - FROM p2p_outbox WHERE message_id = ? AND state = 'RetryExhausted' AND length(content) > 0", - ).bind(next_id).bind(now).bind(now + window).bind(now).bind(id).execute(self.conn()).await?; - } - sqlx::query("UPDATE p2p_outbox SET content = X'', updated_at = ? WHERE message_id = ? AND state = 'RetryExhausted'") - .bind(now).bind(id).execute(self.conn()).await?; - Ok(()) - } - - /// Record a publish that went through and set when the row is next due. - pub async fn schedule_p2p_outbox_retry( - &mut self, - message_id: &str, - next_retry_at: i64, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Pending', lease_until = 0, next_retry_at = ?, \ - publish_count = publish_count + 1, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing' AND retry_until > 0", - ) - .bind(next_retry_at) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Make a waiting row due at once. For a caller whose own publish of the - /// message failed: the row was scheduled on the assumption that it would not. - pub async fn expedite_p2p_outbox_message(&mut self, message_id: &str) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET next_retry_at = 0, updated_at = ? \ - WHERE message_id = ? AND state = 'Pending' AND next_retry_at > 0", - ) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// State and stored payload length of one outbox entry, for diagnostics and - /// tests. A terminal entry has dropped its payload. - pub async fn p2p_outbox_entry_state( - &mut self, - message_id: &str, - ) -> anyhow::Result> { - let row = sqlx::query( - "SELECT state, length(content) AS content_len FROM p2p_outbox WHERE message_id = ?", - ) - .bind(message_id) - .fetch_optional(self.conn()) - .await?; - row.map(|row| Ok((row.try_get("state")?, row.try_get("content_len")?))).transpose() - } - - pub async fn p2p_outbox_delivery_id( - &mut self, - message_id: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_scalar("SELECT delivery_id FROM p2p_outbox WHERE message_id = ?") - .bind(message_id) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn acknowledge_p2p_outbox_message( - &mut self, - delivery_id: &str, - peer_id: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Processed', content = X'', lease_until = 0, next_retry_at = 0, updated_at = ? \ - WHERE delivery_id = ? AND delivery_id != '' AND ack_peer_id = ? AND ack_peer_id != '' \ - AND state IN ('Pending', 'Processing', 'RetryExhausted')", - ) - .bind(get_current_timestamp_secs()) - .bind(delivery_id) - .bind(peer_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Stop a local setup notification once a later protocol step makes it - /// obsolete and release its potentially large payload immediately. - pub async fn cancel_p2p_outbox_message(&mut self, message_id: &str) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Cancelled', content = X'', lease_until = 0, next_retry_at = 0, \ - updated_at = ? \ - WHERE (message_id = ? OR substr(message_id, 1, length(?) + 10) = ? || ':recovery:') \ - AND state IN ('Pending', 'Processing', 'RetryExhausted')", - ) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .bind(message_id) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn complete_p2p_outbox_message(&mut self, message_id: &str) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Processed', content = X'', lease_until = 0, next_retry_at = 0, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing'", - ) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn retry_p2p_outbox_message( - &mut self, - message_id: &str, - next_retry_at: i64, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Pending', lease_until = 0, next_retry_at = ?, last_error = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing'", - ) - .bind(next_retry_at) - .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn fail_p2p_outbox_message( - &mut self, - message_id: &str, - error: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "UPDATE p2p_outbox SET state = 'Failed', content = X'', lease_until = 0, next_retry_at = 0, \ - last_error = ?, updated_at = ? \ - WHERE message_id = ? AND state = 'Processing'", - ) - .bind(error.chars().take(1024).collect::()) - .bind(get_current_timestamp_secs()) - .bind(message_id) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - /// Record that a chain-derived graph message has been durably enqueued. - /// - /// Queue rows are intentionally pruned after their retention period, but - /// replaying an old SyncGraph must not recreate already-completed protocol - /// actions. The caller inserts this marker and its queue row in one - /// transaction. - pub async fn insert_graph_compensation_marker( - &mut self, - graph_id: Uuid, - message_id: &str, - ) -> anyhow::Result { - let result = sqlx::query( - "INSERT INTO graph_compensation_marker (graph_id, message_id, created_at) \ - VALUES (?, ?, ?) ON CONFLICT(graph_id, message_id) DO NOTHING", - ) - .bind(graph_id) - .bind(message_id) - .bind(get_current_timestamp_secs()) - .execute(self.conn()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn upsert_pegin_instance_process_data( - &mut self, - pegin_instance_process_data: &PeginInstanceProcessData, - ) -> anyhow::Result { - let res = sqlx::query!( - "INSERT INTO pegin_instance_process_data - (instance_id, process_data, created_at, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(instance_id) DO UPDATE - SET process_data = excluded.process_data, - updated_at = excluded.updated_at", - pegin_instance_process_data.instance_id, - pegin_instance_process_data.process_data, - pegin_instance_process_data.created_at, - pegin_instance_process_data.updated_at - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected() > 0) - } - - pub async fn find_pegin_instance_process_data( - &mut self, - instance_id: &Uuid, - ) -> anyhow::Result> { - let row = sqlx::query_as!( - PeginInstanceProcessData, - "SELECT - instance_id AS \"instance_id:Uuid\", - process_data, - created_at, - updated_at - FROM pegin_instance_process_data - WHERE instance_id = ?", - instance_id - ) - .fetch_optional(self.conn()) - .await?; - Ok(row) - } - - pub async fn upsert_pegin_graph_process_data( - &mut self, - pegin_graph_process_data: &PeginGraphProcessData, - ) -> anyhow::Result { - let res = sqlx::query!( - "INSERT INTO pegin_graph_process_data - (graph_id, instance_id, process_data, is_endorsed, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(graph_id) DO UPDATE - SET process_data = excluded.process_data, - instance_id = excluded.instance_id, - is_endorsed = excluded.is_endorsed, - updated_at = excluded.updated_at", - pegin_graph_process_data.graph_id, - pegin_graph_process_data.instance_id, - pegin_graph_process_data.process_data, - pegin_graph_process_data.is_endorsed, - pegin_graph_process_data.created_at, - pegin_graph_process_data.updated_at - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected() > 0) - } - - pub async fn find_pegin_graph_process_data( - &mut self, - graph_id: &Uuid, - ) -> anyhow::Result> { - let row = sqlx::query_as!( - PeginGraphProcessData, - "SELECT - graph_id AS \"graph_id:Uuid\", - instance_id AS \"instance_id:Uuid\", - process_data, - is_endorsed, - created_at, - updated_at - FROM pegin_graph_process_data - WHERE graph_id = ?", - graph_id - ) - .fetch_optional(self.conn()) - .await?; - Ok(row) - } - - pub async fn get_pegin_graph_endorsed_len_by_instance_id( - &mut self, - instance_id: &Uuid, - is_endorsed: bool, - ) -> anyhow::Result { - let record = sqlx::query!( - r#"SELECT count(*) AS length - FROM pegin_graph_process_data - WHERE instance_id = ? AND is_endorsed = ?"#, - instance_id, - is_endorsed - ) - .fetch_one(self.conn()) - .await?; - Ok(record.length) - } - - pub async fn upsert_graph_raw_data( - &mut self, - instance_id: Uuid, - graph_raw_data: GraphRawData, - definition_hash: &str, - ) -> anyhow::Result { - if definition_hash.is_empty() { - anyhow::bail!( - "graph {} raw definition is missing its canonical definition hash", - graph_raw_data.graph_id - ); - } - let Some(graph) = self.find_graph(&graph_raw_data.graph_id).await? else { - anyhow::bail!( - "cannot store raw definition for missing graph {}", - graph_raw_data.graph_id - ); - }; - if graph.instance_id != instance_id { - anyhow::bail!( - "raw definition instance does not match graph {}: stored={}, incoming={instance_id}", - graph_raw_data.graph_id, - graph.instance_id - ); - } - if graph.definition_hash != definition_hash { - anyhow::bail!( - "raw definition hash does not match graph {}: stored={}, incoming={definition_hash}", - graph_raw_data.graph_id, - graph.definition_hash - ); - } - - let result = sqlx::query( - "INSERT INTO graph_raw_data (graph_id, raw_data, created_at, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(graph_id) DO UPDATE SET - raw_data = excluded.raw_data, - updated_at = excluded.updated_at", - ) - .bind(graph_raw_data.graph_id) - .bind(graph_raw_data.raw_data) - .bind(graph_raw_data.created_at) - .bind(graph_raw_data.updated_at) - .execute(self.conn()) - .await?; - - Ok(result.rows_affected()) - } - - pub async fn find_graph_raw_data( - &mut self, - graph_id: &Uuid, - ) -> anyhow::Result> { - let row = sqlx::query_as!( - GraphRawData, - "SELECT - graph_id AS \"graph_id:Uuid\", - raw_data, - created_at, - updated_at - FROM graph_raw_data WHERE graph_id = ?", - graph_id - ) - .fetch_optional(self.conn()) - .await?; - - Ok(row) - } - - /// Check graph data exists under both IDs without reading the blob. - pub async fn has_graph_of_instance( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - ) -> anyhow::Result { - let row = sqlx::query( - "SELECT 1 FROM graph \ - WHERE graph_id = ? AND instance_id = ? \ - AND EXISTS (SELECT 1 FROM graph_raw_data WHERE graph_raw_data.graph_id = graph.graph_id) \ - LIMIT 1", - ) - .bind(graph_id) - .bind(instance_id) - .fetch_optional(self.conn()) - .await?; - Ok(row.is_some()) - } - - pub async fn find_watch_contract( - &mut self, - addr: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as!( - WatchContract, - "SELECT * - FROM watch_contract - WHERE contract_addr = ?", - addr - ) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn upsert_watch_contract( - &mut self, - watch_contract: &WatchContract, - ) -> anyhow::Result<()> { - let _ = sqlx::query!( - "INSERT INTO watch_contract (contract_addr, the_graph_url, gap, from_height, status, extra, updated_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (contract_addr) DO UPDATE SET the_graph_url = excluded.the_graph_url, - gap = excluded.gap, - from_height = excluded.from_height, - status = excluded.status, - extra = excluded.extra, - updated_at = excluded.updated_at", - watch_contract.contract_addr, - watch_contract.the_graph_url, - watch_contract.gap, - watch_contract.from_height, - watch_contract.status, - watch_contract.extra, - watch_contract.updated_at, - watch_contract.created_at - ).execute(self.conn()).await; - Ok(()) - } - - pub async fn update_watch_contract_status( - &mut self, - contract_addr: &str, - status: &str, - updated_at: i64, - ) -> anyhow::Result<()> { - let _ = sqlx::query!( - "UPDATE watch_contract SET status = ?, updated_at = ? WHERE contract_addr = ?", - status, - updated_at, - contract_addr, - ) - .execute(self.conn()) - .await; - Ok(()) - } - - pub async fn upsert_goat_tx_record( - &mut self, - goat_tx_record: &GoatTxRecord, - ) -> anyhow::Result<()> { - let mut update_goat_tx_record = goat_tx_record.clone(); - if let Some(goat_tx_record_store) = self - .find_graph_goat_tx_record( - &goat_tx_record.instance_id, - &goat_tx_record.graph_id, - &goat_tx_record.tx_type, - ) - .await? - { - update_goat_tx_record.created_at = goat_tx_record_store.created_at; - update_goat_tx_record.is_local = goat_tx_record_store.is_local; - if goat_tx_record_store.is_processed() { - update_goat_tx_record.processing_status = goat_tx_record_store.processing_status; - } - if update_goat_tx_record.height < goat_tx_record_store.height { - update_goat_tx_record.height = goat_tx_record_store.height; - } - if goat_tx_record_store.extra.is_some() && update_goat_tx_record.extra.is_none() { - update_goat_tx_record.extra = goat_tx_record_store.extra.clone(); - } - - if !goat_tx_record_store.tx_hash.is_empty() { - update_goat_tx_record.tx_hash = goat_tx_record_store.tx_hash.clone(); - } - } - sqlx::query!( - "INSERT OR - REPLACE INTO goat_tx_record (instance_id, - graph_id, - tx_type, - tx_hash, - height, - is_local, - processing_status, - extra, - created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - update_goat_tx_record.instance_id, - update_goat_tx_record.graph_id, - update_goat_tx_record.tx_type, - update_goat_tx_record.tx_hash, - update_goat_tx_record.height, - update_goat_tx_record.is_local, - update_goat_tx_record.processing_status, - update_goat_tx_record.extra, - update_goat_tx_record.created_at - ) - .execute(self.conn()) - .await?; - - Ok(()) - } - - pub async fn find_graph_goat_tx_record( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - tx_type: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as!( - GoatTxRecord, - "SELECT instance_id AS \"instance_id:Uuid\", - graph_id AS \"graph_id:Uuid\", - tx_type, - tx_hash, - height, - is_local, - processing_status, - extra, - created_at - FROM goat_tx_record - WHERE instance_id = ? - AND graph_id = ? - AND tx_type = ?", - instance_id, - graph_id, - tx_type - ) - .fetch_optional(self.conn()) - .await?) - } - - pub async fn get_goat_tx_record_by_processing_status( - &mut self, - tx_type: &str, - processing_status: &str, - ) -> anyhow::Result> { - Ok(sqlx::query_as!( - GoatTxRecord, - "SELECT instance_id AS \"instance_id:Uuid\", - graph_id AS \"graph_id:Uuid\", - tx_type, tx_hash, - height, - is_local, - processing_status, - extra, - created_at - FROM goat_tx_record - WHERE tx_type = ? - AND processing_status = ? - ORDER BY height ASC", - tx_type, - processing_status - ) - .fetch_all(self.conn()) - .await?) - } - - pub async fn update_goat_tx_record_processing_status( - &mut self, - graph_id: &Uuid, - instance_id: &Uuid, - tx_type: &str, - status: &str, - ) -> anyhow::Result<()> { - sqlx::query!( - "UPDATE goat_tx_record - SET processing_status = ? - where instance_id = ? - AND graph_id = ? - AND tx_type = ?", - status, - instance_id, - graph_id, - tx_type - ) - .execute(self.conn()) - .await?; - Ok(()) - } - - pub async fn upsert_graph_btc_tx_vout_monitor( - &mut self, - monitor: &GraphBtcTxVoutMonitor, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - - let res = sqlx::query!( - r#" - INSERT OR REPLACE INTO graph_btc_tx_vout_monitor - (graph_id, tx_name, txid, height, vout_len, monitor_data, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - "#, - monitor.graph_id, - monitor.tx_name, - monitor.txid, - monitor.height, - monitor.vout_len, - monitor.monitor_data, - monitor.created_at, - current_time - ) - .execute(self.conn()) - .await?; - - Ok(res.rows_affected()) - } - - pub async fn find_graph_btc_tx_vout_monitor( - &mut self, - graph_id: &Uuid, - txid: &SerializableTxid, - ) -> anyhow::Result> { - let row = sqlx::query_as!( - GraphBtcTxVoutMonitor, - r#" - SELECT - graph_id AS "graph_id: Uuid", - tx_name, - txid AS "txid: SerializableTxid", - height, - vout_len, - monitor_data, - created_at, - updated_at - FROM graph_btc_tx_vout_monitor - WHERE graph_id = ? AND txid = ? - "#, - graph_id, - txid - ) - .fetch_optional(self.conn()) - .await?; - - Ok(row) - } - - pub async fn create_long_running_task_proof( - &mut self, - long_running_task_proof: &LongRunningTaskProof, - ) -> anyhow::Result { - let res = sqlx::query!( - "INSERT - INTO long_running_task_proof (block_start, block_end, chain_name, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, - zkm_version, extra, updated_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - long_running_task_proof.block_start, - long_running_task_proof.block_end, - long_running_task_proof.chain_name, - long_running_task_proof.path_to_proof, - long_running_task_proof.public_value_hex, - long_running_task_proof.proof_size, - long_running_task_proof.cycles, - long_running_task_proof.proof_state, - long_running_task_proof.total_time_to_proof, - long_running_task_proof.proving_time, - long_running_task_proof.zkm_version, - long_running_task_proof.extra, - long_running_task_proof.updated_at, - long_running_task_proof.created_at, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - /// Deletes all persisted proof tasks for one proof chain. - pub async fn delete_long_running_task_proofs_by_name( - &mut self, - chain_name: &str, - ) -> anyhow::Result { - let res = sqlx::query("DELETE FROM long_running_task_proof WHERE chain_name = ?") - .bind(chain_name) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - #[allow(clippy::too_many_arguments)] - pub async fn update_long_running_task_proof_success( - &mut self, - block_start: i64, - chain_name: &str, - batch_size: i64, - path_to_proof: String, - public_value_hex: String, - proof_size: i64, - cycles: i64, - proof_state: i64, - total_time_to_proof: i64, - proving_time: i64, - zkm_version: &str, - ) -> anyhow::Result { - let block_end = block_start + batch_size; - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE long_running_task_proof - SET path_to_proof = ?, - cycles = ?, - proof_state = ?, - total_time_to_proof = ?, - proving_time = ?, - zkm_version = ?, - block_end = ?, - public_value_hex = ?, - proof_size = ?, - updated_at = ? - WHERE block_start = ? AND chain_name = ?", - path_to_proof, - cycles, - proof_state, - total_time_to_proof, - proving_time, - zkm_version, - block_end, - public_value_hex, - proof_size, - current_time, - block_start, - chain_name, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn update_long_running_task_proof_state( - &mut self, - block_start: i64, - chain_name: &str, - batch_size: i64, - proof_state: i64, - ) -> anyhow::Result { - let block_end = block_start + batch_size; - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE long_running_task_proof - SET proof_state = ?, - block_end = ?, - updated_at = ? - WHERE block_start = ? AND chain_name = ?", - proof_state, - block_end, - current_time, - block_start, - chain_name, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn find_all_running_task_proofs_by_name( - &mut self, - chain_name: String, - ) -> anyhow::Result> { - let res = sqlx::query_as!( - LongRunningTaskProof, - "SELECT block_start, block_end, chain_name, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, - zkm_version, extra, updated_at, created_at FROM long_running_task_proof - WHERE chain_name = ?", - chain_name, - ) - .fetch_all(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_long_running_task_proof_including_block_number( - &mut self, - block_number: i64, - chain_name: String, - ) -> anyhow::Result> { - let res = sqlx::query_as!( - LongRunningTaskProof, - "SELECT block_start, block_end, chain_name, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, - zkm_version, extra, updated_at, created_at FROM long_running_task_proof - - WHERE block_end > ? and block_start <= ? AND chain_name = ? LIMIT 1", - block_number, - block_number, - chain_name, - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_latest_long_running_task_proof_by_name_and_state( - &mut self, - chain_name: String, - proof_state: i64, - ) -> anyhow::Result> { - let res = sqlx::query_as!( - LongRunningTaskProof, - "SELECT - block_start, - block_end, - chain_name, - path_to_proof, - public_value_hex, - proof_size, - cycles, - proof_state, - total_time_to_proof, - proving_time, - zkm_version, - extra, - updated_at, - created_at - FROM long_running_task_proof - WHERE chain_name = ? - AND proof_state = ? - ORDER BY block_start DESC - LIMIT 1", - chain_name, - proof_state, - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_latest_long_running_task_proof_by_name( - &mut self, - chain_name: String, - ) -> anyhow::Result> { - let res = sqlx::query_as!( - LongRunningTaskProof, - "SELECT - block_start, - block_end, - chain_name, - path_to_proof, - public_value_hex, - proof_size, - cycles, - proof_state, - total_time_to_proof, - proving_time, - zkm_version, - extra, - updated_at, - created_at - FROM long_running_task_proof - WHERE chain_name = ? - ORDER BY block_start DESC - LIMIT 1", - chain_name, - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn create_operator_proof( - &mut self, - operator_proof: &OperatorProof, - ) -> anyhow::Result { - let res = sqlx::query!( - "INSERT - INTO operator_proof (instance_id, graph_id, execution_layer_block_number, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, - zkm_version, extra, updated_at, created_at, operator_committed_blockhash) - VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - operator_proof.instance_id, - operator_proof.graph_id, - operator_proof.execution_layer_block_number, - operator_proof.path_to_proof, - operator_proof.public_value_hex, - operator_proof.proof_size, - operator_proof.cycles, - operator_proof.proof_state, - operator_proof.total_time_to_proof, - operator_proof.proving_time, - operator_proof.zkm_version, - operator_proof.extra, - operator_proof.updated_at, - operator_proof.created_at, - operator_proof.operator_committed_blockhash, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - #[allow(clippy::too_many_arguments)] - pub async fn update_operator_proof( - &mut self, - id: i64, - path_to_proof: String, - public_value_hex: String, - proof_size: i64, - cycles: i64, - proof_state: i64, - total_time_to_proof: i64, - proving_time: i64, - zkm_version: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE operator_proof - SET path_to_proof = ?, - public_value_hex = ?, - proof_size = ?, - cycles = ?, - proof_state = ?, - total_time_to_proof = ?, - proving_time = ?, - zkm_version = ?, - updated_at = ? - WHERE id = ?", - path_to_proof, - public_value_hex, - proof_size, - cycles, - proof_state, - total_time_to_proof, - proving_time, - zkm_version, - current_time, - id, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn update_operator_proof_state_with_instance_graph( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - old_proof_state: i64, - new_proof_state: i64, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE operator_proof - SET proof_state = ?, - updated_at = ? - WHERE instance_id = ? AND graph_id = ? AND proof_state = ?", - new_proof_state, - current_time, - instance_id, - graph_id, - old_proof_state - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn find_operator_proof_by_instance_and_graph( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, OperatorProof>( - "SELECT * FROM operator_proof - WHERE instance_id = ? - AND graph_id = ?", - ) - .bind(instance_id) - .bind(graph_id) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_next_operator_proof(&mut self) -> anyhow::Result> { - let res = sqlx::query_as::<_, OperatorProof>( - "SELECT * FROM operator_proof - WHERE proof_state == 0 - ORDER BY id ASC - LIMIT 1", - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - #[allow(clippy::too_many_arguments)] - pub async fn update_watchtower_proof( - &mut self, - id: i64, - path_to_proof: String, - public_value_hex: String, - proof_size: i64, - cycles: i64, - proof_state: i64, - total_time_to_proof: i64, - proving_time: i64, - zkm_version: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE watchtower_proof - SET path_to_proof = ?, - public_value_hex = ?, - proof_size = ?, - cycles = ?, - proof_state = ?, - total_time_to_proof = ?, - proving_time = ?, - zkm_version = ?, - updated_at = ? - WHERE id = ?", - path_to_proof, - public_value_hex, - proof_size, - cycles, - proof_state, - total_time_to_proof, - proving_time, - zkm_version, - current_time, - id, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - /// Finishes an on-demand proof task only while it is still `New`. - /// - /// A timeout request may mark the task `Failed` while its proof is queued or - /// running; that state wins and this update affects no row. - pub async fn complete_on_demand_proof_if_new( - &mut self, - table: OnDemandProofTable, - id: i64, - completion: &OnDemandProofCompletion, - ) -> anyhow::Result { - let sql = format!( - "UPDATE {} - SET path_to_proof = ?, - public_value_hex = ?, - proof_size = ?, - cycles = ?, - proof_state = ?, - total_time_to_proof = ?, - proving_time = ?, - zkm_version = ?, - updated_at = ? - WHERE id = ? AND proof_state = ?", - table.table_name() - ); - let res = sqlx::query(&sql) - .bind(&completion.path_to_proof) - .bind(&completion.public_value_hex) - .bind(completion.proof_size) - .bind(completion.cycles) - .bind(completion.proof_state) - .bind(completion.total_time_to_proof) - .bind(completion.proving_time) - .bind(&completion.zkm_version) - .bind(get_current_timestamp_secs()) - .bind(id) - .bind(crate::ProofState::New.to_i64()) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - /// Returns the current proof state of an on-demand proof task. - pub async fn get_on_demand_proof_state( - &mut self, - table: OnDemandProofTable, - id: i64, - ) -> anyhow::Result> { - let sql = format!("SELECT proof_state FROM {} WHERE id = ?", table.table_name()); - let state = sqlx::query_scalar::<_, i64>(&sql).bind(id).fetch_optional(self.conn()).await?; - Ok(state) - } - - /// Counts unfinished (`New` or `Proving`) tasks of one proof chain starting at or after - /// `from_block`. - pub async fn count_pending_long_running_task_proofs( - &mut self, - chain_name: &str, - from_block: i64, - ) -> anyhow::Result { - let count = sqlx::query_scalar::<_, i64>( - "SELECT COUNT(*) FROM long_running_task_proof - WHERE chain_name = ? AND block_start >= ? AND proof_state IN (?, ?)", - ) - .bind(chain_name) - .bind(from_block) - .bind(crate::ProofState::New.to_i64()) - .bind(crate::ProofState::Proving.to_i64()) - .fetch_one(self.conn()) - .await?; - Ok(count) - } - - pub async fn update_watchtower_proof_state_with_instance_graph_pubkey( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - public_key: &str, - old_proof_state: i64, - new_proof_state: i64, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE watchtower_proof - SET proof_state = ?, - updated_at = ? - WHERE instance_id = ? - AND graph_id = ? - AND public_key = ? AND proof_state = ?", - new_proof_state, - current_time, - instance_id, - graph_id, - public_key, - old_proof_state - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn update_watchtower_proof_challenge_txid( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - node_index: i32, - challenge_txid: &str, - included: bool, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query!( - "UPDATE watchtower_proof - SET challenge_txid = ?, - included =?, - updated_at = ? - WHERE instance_id = ? - AND graph_id = ? - AND node_index = ?", - challenge_txid, - included, - current_time, - instance_id, - graph_id, - node_index, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn update_watchtower_proof_node_index( - &mut self, - id: i64, - instance_id: &Uuid, - graph_id: &Uuid, - node_index: i32, - ) -> anyhow::Result { - let res = sqlx::query( - "UPDATE watchtower_proof - SET node_index = ? - WHERE id = ? - AND instance_id = ? - AND graph_id = ?", - ) - .bind(node_index) - .bind(id) - .bind(instance_id) - .bind(graph_id) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn find_watchtower_proof_by_instance_and_graph( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, WatchtowerProof>( - "SELECT * - FROM watchtower_proof - WHERE instance_id = ? - AND graph_id = ? - ORDER BY node_index ASC, id ASC", - ) - .bind(instance_id) - .bind(graph_id) - .fetch_all(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_watchtower_proof_by_instance_and_graph_and_pubkey( - &mut self, - instance_id: &Uuid, - graph_id: &Uuid, - public_key: &str, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, WatchtowerProof>( - "SELECT * - FROM watchtower_proof - WHERE instance_id = ? - AND graph_id = ? - AND public_key = ?", - ) - .bind(instance_id) - .bind(graph_id) - .bind(public_key) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_next_watchtower_proof(&mut self) -> anyhow::Result> { - let res = sqlx::query_as::<_, WatchtowerProof>( - "SELECT * - FROM watchtower_proof - WHERE proof_state == 0 - ORDER BY id ASC - lIMIT 1", - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn create_watchtower_proof( - &mut self, - watchtower_proof: &WatchtowerProof, - ) -> anyhow::Result { - let res = sqlx::query!( - "INSERT - INTO watchtower_proof (instance_id, graph_id, public_key, challenge_txid, challenge_init_txid, execution_layer_block_number, - path_to_proof, - public_value_hex, proof_size, - cycles, proof_state, total_time_to_proof, proving_time, - zkm_version, node_index, included, - extra, updated_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - watchtower_proof.instance_id, - watchtower_proof.graph_id, - watchtower_proof.public_key, - watchtower_proof.challenge_txid, - watchtower_proof.challenge_init_txid, - watchtower_proof.execution_layer_block_number, - watchtower_proof.path_to_proof, - watchtower_proof.public_value_hex, - watchtower_proof.proof_size, - watchtower_proof.cycles, - watchtower_proof.proof_state, - watchtower_proof.total_time_to_proof, - watchtower_proof.proving_time, - watchtower_proof.zkm_version, - watchtower_proof.node_index, - watchtower_proof.included, - watchtower_proof.extra, - watchtower_proof.updated_at, - watchtower_proof.created_at, - ) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn upsert_bridge_out_global_stats( - &mut self, - bridge_out_stats: &BridgeOutGlobalStats, - ) -> anyhow::Result<()> { - sqlx::query!( - r#"INSERT INTO bridge_out_global_stats (id, initial_txn, initial_amount, claim_txn, claim_amount, refund_txn, - refund_amount,updated_at, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (id) DO UPDATE SET initial_txn = excluded.initial_txn, - initial_amount = excluded.initial_amount, - claim_txn = excluded.claim_txn, - claim_amount = excluded.claim_amount, - refund_txn = excluded.refund_txn, - refund_amount = excluded.refund_amount, - updated_at = excluded.updated_at"#, - bridge_out_stats.id, - bridge_out_stats.initial_txn, - bridge_out_stats.initial_amount, - bridge_out_stats.claim_txn, - bridge_out_stats.claim_amount, - bridge_out_stats.refund_txn, - bridge_out_stats.refund_amount, - bridge_out_stats.updated_at, - bridge_out_stats.created_at, - ).execute(self.conn()) - .await?; - Ok(()) - } - pub async fn find_bridge_out_global_stats_by_id( - &mut self, - id: i64, - ) -> anyhow::Result> { - let res = sqlx::query_as!( - BridgeOutGlobalStats, - r#"SELECT * from bridge_out_global_stats WHERE id = ?"#, - id - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn upsert_sequencer_set_hash_change( - &mut self, - cosmos_block_height: i64, - goat_block_height: i64, - validators_hash: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query( - r#"INSERT INTO sequencer_set_hash_changes (cosmos_block_height, goat_block_height, validators_hash, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT (cosmos_block_height) DO UPDATE - SET goat_block_height = excluded.goat_block_height, - validators_hash = excluded.validators_hash, - updated_at = excluded.updated_at"#, - ) - .bind(cosmos_block_height) - .bind(goat_block_height) - .bind(validators_hash) - .bind(current_time) - .bind(current_time) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } - - pub async fn find_latest_sequencer_set_hash_change( - &mut self, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, SequencerSetHashChange>( - "SELECT * FROM sequencer_set_hash_changes ORDER BY cosmos_block_height DESC LIMIT 1", - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn find_first_sequencer_set_hash_change_by_goat_block_at_or_before( - &mut self, - goat_block_height: i64, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, SequencerSetHashChange>( - "SELECT * FROM sequencer_set_hash_changes WHERE goat_block_height <= ? ORDER BY goat_block_height DESC LIMIT 1", - ) - .bind(goat_block_height) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn get_sequencer_set_scan_state( - &mut self, - ) -> anyhow::Result> { - let res = sqlx::query_as::<_, SequencerSetScanState>( - "SELECT * FROM sequencer_set_scan_state WHERE id = 1 LIMIT 1", - ) - .fetch_optional(self.conn()) - .await?; - Ok(res) - } - - pub async fn upsert_sequencer_set_scan_state( - &mut self, - next_cosmos_block_height: i64, - latest_goat_block_height: i64, - latest_validators_hash: &str, - ) -> anyhow::Result { - let current_time = get_current_timestamp_secs(); - let res = sqlx::query( - r#"INSERT INTO sequencer_set_scan_state (id, next_cosmos_block_height, latest_goat_block_height, latest_validators_hash, created_at, updated_at) - VALUES (1, ?, ?, ?, ?, ?) - ON CONFLICT (id) DO UPDATE - SET next_cosmos_block_height = excluded.next_cosmos_block_height, - latest_goat_block_height = excluded.latest_goat_block_height, - latest_validators_hash = excluded.latest_validators_hash, - updated_at = excluded.updated_at"#, - ) - .bind(next_cosmos_block_height) - .bind(latest_goat_block_height) - .bind(latest_validators_hash) - .bind(current_time) - .bind(current_time) - .execute(self.conn()) - .await?; - Ok(res.rows_affected()) - } -} - -// fn truncate_string(s: &str, max_len: usize) -> &str { -// if s.len() > max_len { &s[..max_len] } else { s } -// } - -pub async fn create_local_db(db_path: &str) -> LocalDB { - let local_db = LocalDB::new(db_path, true).await; - local_db.migrate().await; - local_db -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::MessageState; - - async fn setup_db() -> LocalDB { - create_local_db("sqlite::memory:").await - } - - fn pegin_instance(instance_id: Uuid, status: &str) -> Instance { - Instance { - instance_id, - network: "testnet".to_string(), - input_utxos: "[]".to_string(), - status: status.to_string(), - created_at: 100, - updated_at: 100, - ..Default::default() - } - } - - fn pegin_request_statuses() -> Vec { - vec![ - crate::InstanceBridgeInStatus::UserIniting.to_string(), - crate::InstanceBridgeInStatus::UserInited.to_string(), - ] - } - - #[tokio::test] - async fn test_upsert_pegin_request_instance_initializes_and_promotes() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let instance_id = Uuid::new_v4(); - - let mut initing = pegin_instance(instance_id, "UserIniting"); - initing.to_addr = "0xuser".to_string(); - initing.from_addr = "bcrt1quser".to_string(); - assert!(s.insert_instance_if_absent(&initing).await.unwrap()); - - let mut request = pegin_instance(instance_id, "UserInited"); - request.to_addr = "0xuser".to_string(); - request.amount = 10_000; - request.goat_tx_hash = "0xrequest".to_string(); - request.goat_tx_height = 500; - request.status_updated_at = 200; - request.created_at = 999; - request.updated_at = 999; - assert!( - s.upsert_pegin_request_instance(&request, &pegin_request_statuses()).await.unwrap() - ); - - let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); - assert_eq!(stored.status, "UserInited"); - assert_eq!(stored.amount, 10_000); - assert_eq!(stored.goat_tx_hash, "0xrequest"); - assert_eq!(stored.goat_tx_height, 500); - assert_eq!(stored.status_updated_at, 200); - // An undecodable from_addr must not erase the one the request tag stored. - assert_eq!(stored.from_addr, "bcrt1quser"); - // created_at belongs to whoever created the row. - assert_eq!(stored.created_at, 100); - } - - #[tokio::test] - async fn test_upsert_pegin_request_instance_replay_keeps_accrued_state() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let instance_id = Uuid::new_v4(); - - let mut inited = pegin_instance(instance_id, "UserInited"); - inited.goat_tx_hash = "0xrequest".to_string(); - inited.goat_tx_height = 500; - inited.committees_answers = IndexMap::from([("0xcommittee".to_string(), vec![1u8, 2, 3])]); - inited.parameters = Some("{}".to_string()); - assert!(s.insert_instance_if_absent(&inited).await.unwrap()); - - // A re-delivered request refreshes the row in place; everything the - // instance accrued after the request must survive it. - let mut replay = pegin_instance(instance_id, "UserInited"); - replay.goat_tx_hash = "0xrequest".to_string(); - replay.goat_tx_height = 500; - assert!(s.upsert_pegin_request_instance(&replay, &pegin_request_statuses()).await.unwrap()); - let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); - assert_eq!(stored.committees_answers.get("0xcommittee"), Some(&vec![1u8, 2, 3])); - assert_eq!(stored.parameters, Some("{}".to_string())); - assert_eq!(stored.goat_tx_height, 500); - } - - #[tokio::test] - async fn test_upsert_pegin_request_instance_rejects_regression() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let instance_id = Uuid::new_v4(); - - let mut minted = pegin_instance(instance_id, "RelayerL2Minted"); - minted.goat_tx_hash = "0xrequest".to_string(); - minted.goat_tx_height = 500; - minted.parameters = Some("{}".to_string()); - minted.post_pegin_txhash = Some("0xmint".to_string()); - assert!(s.insert_instance_if_absent(&minted).await.unwrap()); - - // Once the instance moves on, a re-delivered request may not pull it back. - let replay = pegin_instance(instance_id, "UserInited"); - assert!( - !s.upsert_pegin_request_instance(&replay, &pegin_request_statuses()).await.unwrap() - ); - let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); - assert_eq!(stored.status, "RelayerL2Minted"); - assert_eq!(stored.parameters, Some("{}".to_string())); - assert_eq!(stored.post_pegin_txhash, Some("0xmint".to_string())); - assert_eq!(stored.goat_tx_height, 500); - } - - #[tokio::test] - async fn test_upsert_pegin_request_instance_rejects_progressed_instance() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let instance_id = Uuid::new_v4(); - - let progressed = pegin_instance(instance_id, "CommitteesAnswered"); - assert!(s.insert_instance_if_absent(&progressed).await.unwrap()); - - let request = pegin_instance(instance_id, "UserInited"); - assert!( - !s.upsert_pegin_request_instance(&request, &pegin_request_statuses()).await.unwrap() - ); - let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); - assert_eq!(stored.status, "CommitteesAnswered"); - } - - #[tokio::test] - async fn test_instance_update_rejects_stale_status_transition() { - let db = setup_db().await; - let instance_id = Uuid::new_v4(); - let mut storage = db.acquire().await.unwrap(); - assert!( - storage - .insert_instance_if_absent(&pegin_instance(instance_id, "RelayerL2Minted")) - .await - .unwrap() - ); - - let updated = storage - .update_instance( - &InstanceUpdate::new_with_instance_id(instance_id) - .with_status("Timeout".to_string()) - .with_only_if_status_in(vec!["Presigned".to_string()]), - ) - .await - .unwrap(); - assert!(!updated); - assert_eq!( - storage.find_instance(&instance_id).await.unwrap().unwrap().status, - "RelayerL2Minted" - ); - } - - #[tokio::test] - async fn test_node_metrics_state_counts() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - - sqlx::query( - "INSERT INTO instance (instance_id, status, created_at, updated_at) VALUES ('in-1', 'Pending', 20, 20), ('in-2', 'Pending', 10, 10)", - ) - .execute(s.conn()) - .await - .unwrap(); - sqlx::query( - "INSERT INTO swap_escrow (escrow_hash, status, created_at, updated_at) VALUES ('0xaa', 'Initialize', 30, 30)", - ) - .execute(s.conn()) - .await - .unwrap(); - sqlx::query( - "INSERT INTO graph (graph_id, instance_id, status, created_at, updated_at) VALUES ('graph-1', 'in-1', 'Created', 40, 40), ('graph-2', 'in-2', 'Created', 25, 25)", - ) - .execute(s.conn()) - .await - .unwrap(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, created_at, updated_at) VALUES ('message-1', 'in-1', 'operator', 'test', X'00', 'Pending', 15, 15)", - ) - .execute(s.conn()) - .await - .unwrap(); - - let counts = s.node_metrics_state_counts().await.unwrap(); - assert_eq!( - counts.iter().find(|count| count.category == "instance" && count.state == "Pending"), - Some(&MetricsStateCount { - category: "instance".to_string(), - state: "Pending".to_string(), - count: 2, - oldest_created_at: Some(10), - last_success_at: None, - }) - ); - assert_eq!( - counts - .iter() - .find(|count| count.category == "swap_escrow" && count.state == "Initialize") - .unwrap() - .count, - 1 - ); - assert_eq!( - counts - .iter() - .find(|count| count.category == "graph" && count.state == "Created") - .unwrap() - .oldest_created_at, - Some(25) - ); - assert_eq!( - counts - .iter() - .find(|count| count.category == "message" && count.state == "Pending") - .unwrap() - .count, - 1 - ); - } - - /// A retired row must stay retired: a periodic producer re-upserting the same - /// deterministic message_id would otherwise resurrect it on every tick. - #[tokio::test] - async fn upsert_does_not_resurrect_a_retired_message() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - let msg = Message { - message_id: "retired-1".to_string(), - business_id, - actor: "Operator".to_string(), - from_peer: "self".to_string(), - msg_type: "AssertReady".to_string(), - content: vec![1, 2], - state: MessageState::Pending.to_string(), - ..Default::default() - }; - assert!(s.upsert_message(msg.clone()).await.unwrap()); - - let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert!( - s.fail_local_message("retired-1", claimed[0].message_version, "handler panicked") - .await - .unwrap() - ); - - // The producer tries again with the same deterministic id. - assert!(!s.upsert_message(msg).await.unwrap(), "a retired row must not be revived"); - let row = s.find_messages_by_id("retired-1").await.unwrap().unwrap(); - assert_eq!(row.state, "Failed"); - assert!(row.content.is_empty(), "the retired payload must stay released"); - } - - #[tokio::test] - async fn upsert_does_not_replace_an_active_local_claim() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - let mut message = Message { - message_id: "active-1".to_owned(), - business_id, - actor: "Operator".to_owned(), - from_peer: "self".to_owned(), - msg_type: "AssertReady".to_owned(), - content: vec![1, 2], - state: MessageState::Pending.to_string(), - ..Default::default() - }; - assert!(s.upsert_message(message.clone()).await.unwrap()); - let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); - assert_eq!(claimed.len(), 1); - - message.content = vec![9, 9]; - assert!(!s.upsert_message(message).await.unwrap()); - let stored = s.find_messages_by_id("active-1").await.unwrap().unwrap(); - assert_eq!(stored.state, "Processing"); - assert_eq!(stored.content, vec![1, 2]); - assert_eq!(stored.message_version, claimed[0].message_version); - } - - #[tokio::test] - async fn producer_replay_does_not_forgive_an_abandoned_local_claim() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let message = Message { - message_id: "abandoned-replay-1".to_owned(), - business_id: Uuid::new_v4(), - actor: "Operator".to_owned(), - from_peer: "self".to_owned(), - msg_type: "AssertReady".to_owned(), - content: vec![1, 2], - state: MessageState::Pending.to_string(), - ..Default::default() - }; - assert!(s.upsert_message(message.clone()).await.unwrap()); - sqlx::query("UPDATE message SET abandon_count = 2 WHERE message_id = ?") - .bind(&message.message_id) - .execute(s.conn()) - .await - .unwrap(); - - assert!(s.upsert_message(message.clone()).await.unwrap()); - let stored = s.find_messages_by_id(&message.message_id).await.unwrap().unwrap(); - assert_eq!(stored.abandon_count, 2); - } - - #[tokio::test] - async fn self_defer_cannot_be_overwritten_by_stale_completion() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES ('self-defer-1', ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", - ) - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); - assert!( - s.self_defer_local_message( - "self-defer-1", - claimed[0].message_version, - 150, - "not ready" - ) - .await - .unwrap() - ); - assert!( - !s.complete_local_message("self-defer-1", claimed[0].message_version).await.unwrap() - ); - let stored = s.find_messages_by_id("self-defer-1").await.unwrap().unwrap(); - assert_eq!(stored.state, "Pending"); - assert_eq!(stored.attempt_count, 1); - } - - #[tokio::test] - async fn cancellation_reaches_pending_and_processing_local_messages() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - for (message_id, msg_type, state, lock_time_until) in [ - ("cancel-processing", "AssertReady", "Processing", 500), - ("cancel-pending", "PostReady", "Pending", 400), - ("keep-processed", "KickoffReady", "Processed", 0), - ] { - sqlx::query( - "INSERT INTO message \ - (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES (?, ?, 'Operator', ?, X'01', ?, ?, 10, 10)", - ) - .bind(message_id) - .bind(business_id) - .bind(msg_type) - .bind(state) - .bind(lock_time_until) - .execute(s.conn()) - .await - .unwrap(); - } - - assert_eq!( - s.cancel_messages_by_business_id(&business_id, Some("AssertReady".to_owned())) - .await - .unwrap(), - 1 - ); - let processing = s.find_messages_by_id("cancel-processing").await.unwrap().unwrap(); - assert_eq!(processing.state, "Cancelled"); - assert_eq!(processing.lock_time_until, 0); - assert_eq!(s.cancel_messages_by_business_id(&business_id, None).await.unwrap(), 1); - assert_eq!( - s.find_messages_by_id("cancel-pending").await.unwrap().unwrap().state, - "Cancelled" - ); - assert_eq!( - s.find_messages_by_id("keep-processed").await.unwrap().unwrap().state, - "Processed" - ); - } - - /// Every `message` SELECT must fetch the full column set `message_from_row` - /// reads. Adding a column and updating only some of the hand-written SELECT - /// lists fails at runtime, not at compile time, so pin all of them here. - #[tokio::test] - async fn every_message_query_hydrates_the_full_row() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES (?, ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", - ) - .bind("hydrate-1") - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - - let by_id = s.find_messages_by_id("hydrate-1").await.unwrap().expect("row by id"); - assert_eq!(by_id.attempt_count, 0); - assert_eq!(by_id.abandon_count, 0); - assert!(by_id.last_error.is_none()); - - let by_business = s - .find_message_by_business_id(&business_id, "AssertReady") - .await - .unwrap() - .expect("row by business id"); - assert_eq!(by_business.message_id, "hydrate-1"); - - let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].message_id, "hydrate-1"); - } - - fn admission_inbox_message( - message_id: &str, - from_peer: &str, - admission_class: P2pInboxAdmissionClass, - content: Vec, - ) -> P2pInboxMessage { - P2pInboxMessage { - message_id: message_id.to_string(), - actor: "Committee".to_string(), - from_peer: from_peer.to_string(), - msg_type: "KickoffSent".to_string(), - content_size: content.len() as i64, - content_hash: Some(content.clone()), - content, - admission_class: admission_class.to_string(), - ..Default::default() - } - } - - /// Quota accounting must see what a sender currently makes the node store: - /// queued rows only, split by class, with the sender's own share. - #[tokio::test] - async fn inbox_usage_counts_queued_rows_per_class_and_sender() { - use P2pInboxAdmissionClass::{Registered, Unregistered}; - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - for (message_id, from_peer, class, size) in [ - ("usage-1", "peer-a", Registered, 10), - ("usage-2", "peer-a", Unregistered, 20), - ("usage-3", "peer-b", Unregistered, 30), - ("usage-4", "peer-b", Unregistered, 40), - ] { - let message = admission_inbox_message(message_id, from_peer, class, vec![7; size]); - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - } - // A terminal row has dropped its payload and must stop counting. - let claimed = s.claim_p2p_inbox_message("usage-4", 100, 200).await.unwrap().unwrap(); - assert!(s.complete_p2p_inbox_message("usage-4", &claimed.lease_token).await.unwrap()); - // A claimed row still holds its payload. - assert!(s.claim_p2p_inbox_message("usage-3", 100, 200).await.unwrap().is_some()); - - let mut usage = s.p2p_inbox_usage("peer-b").await.unwrap(); - usage.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); - assert_eq!( - usage, - vec![ - P2pInboxClassUsage { - admission_class: "Registered".to_string(), - rows: 1, - bytes: 10, - peer_rows: 0, - peer_bytes: 0, - }, - P2pInboxClassUsage { - admission_class: "Unregistered".to_string(), - rows: 2, - bytes: 50, - peer_rows: 1, - peer_bytes: 30, - }, - ] - ); - } - - /// The usage query runs for every inbound message; it must be answered from - /// the covering index, never by reading rows that carry multi-megabyte blobs. - #[tokio::test] - async fn inbox_usage_is_answered_from_the_covering_index() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let plan: Vec = sqlx::query( - "EXPLAIN QUERY PLAN \ - SELECT admission_class, COUNT(*), COALESCE(SUM(content_size), 0), \ - COALESCE(SUM(CASE WHEN from_peer = 'p' THEN content_size ELSE 0 END), 0) \ - FROM p2p_inbox WHERE state IN ('Pending', 'Processing', 'Quarantined') \ - GROUP BY admission_class", - ) - .fetch_all(s.conn()) - .await - .unwrap() - .iter() - .map(|row| row.get::("detail")) - .collect(); - assert!( - plan.iter().any(|step| step.contains("COVERING INDEX idx_p2p_inbox_admission")), - "unexpected plan: {plan:?}" - ); - } - - /// A row written without a classification defaults to `Unregistered` rather - /// than silently joining the registered pool. - #[tokio::test] - async fn unclassified_inbox_rows_default_to_unregistered() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let message = P2pInboxMessage { - message_id: "unclassified-1".to_string(), - actor: "Operator".to_string(), - from_peer: "peer".to_string(), - msg_type: "CreateGraph".to_string(), - content: vec![1, 2, 3], - content_size: 3, - ..Default::default() - }; - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - let usage = s.p2p_inbox_usage("peer").await.unwrap(); - assert_eq!(usage.len(), 1); - assert_eq!(usage[0].admission_class, "Unregistered"); - assert!(!s.has_queued_p2p_inbox_payload("peer", &[1, 2, 3]).await.unwrap()); - } - - /// Verify quota accounting and cleanup of quarantined payloads and terminal rows. - #[tokio::test] - async fn quarantined_payloads_count_and_are_capped() { - use P2pInboxAdmissionClass::Unregistered; - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - for (message_id, updated_at) in [("q-old", 100), ("q-new", 200)] { - let message = admission_inbox_message(message_id, "peer", Unregistered, vec![9; 1000]); - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - sqlx::query( - "UPDATE p2p_inbox SET state = 'Quarantined', updated_at = ? WHERE message_id = ?", - ) - .bind(updated_at) - .bind(message_id) - .execute(s.conn()) - .await - .unwrap(); - } - let usage = s.p2p_inbox_usage("peer").await.unwrap(); - assert_eq!(usage.iter().map(|u| u.bytes).sum::(), 2000, "quarantine counts"); - - // Keep only ~1000 bytes: the oldest quarantined payload is cleared. - assert_eq!(s.trim_quarantined_p2p_inbox_payloads(1000).await.unwrap(), 1); - let cleared: i64 = - sqlx::query("SELECT length(content) AS len FROM p2p_inbox WHERE message_id = 'q-old'") - .fetch_one(s.conn()) - .await - .unwrap() - .get("len"); - assert_eq!(cleared, 0, "the oldest payload was cleared"); - let kept: i64 = - sqlx::query("SELECT length(content) AS len FROM p2p_inbox WHERE message_id = 'q-new'") - .fetch_one(s.conn()) - .await - .unwrap() - .get("len"); - assert_eq!(kept, 1000, "the newest payload is retained"); - - // The row cap deletes the oldest terminal rows beyond the ceiling. - assert_eq!(s.purge_p2p_inbox_over_terminal_cap(1, 10).await.unwrap(), 1); - let remaining: i64 = sqlx::query("SELECT COUNT(*) AS n FROM p2p_inbox") - .fetch_one(s.conn()) - .await - .unwrap() - .get("n"); - assert_eq!(remaining, 1); - } - - /// Verify split usage queries match the combined query and use the covering index. - #[tokio::test] - async fn split_inbox_usage_matches_and_seeks_the_sender() { - use P2pInboxAdmissionClass::{Registered, Unregistered}; - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - for (message_id, from_peer, class, size) in [ - ("split-1", "peer-a", Registered, 10), - ("split-2", "peer-a", Unregistered, 20), - ("split-3", "peer-b", Unregistered, 30), - ] { - let message = admission_inbox_message(message_id, from_peer, class, vec![7; size]); - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - } - let mut combined = s.p2p_inbox_usage("peer-a").await.unwrap(); - combined.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); - let mut totals = s.p2p_inbox_class_totals().await.unwrap(); - totals.sort_by(|a, b| a.admission_class.cmp(&b.admission_class)); - for peer in s.p2p_inbox_peer_usage("peer-a").await.unwrap() { - let total = - totals.iter_mut().find(|t| t.admission_class == peer.admission_class).unwrap(); - total.peer_rows = peer.peer_rows; - total.peer_bytes = peer.peer_bytes; - } - assert_eq!(totals, combined); - - let plan: Vec = sqlx::query( - "EXPLAIN QUERY PLAN \ - SELECT admission_class, COUNT(*), COALESCE(SUM(content_size), 0) \ - FROM p2p_inbox \ - WHERE state IN ('Pending', 'Processing', 'Quarantined') \ - AND admission_class IN ('Registered', 'Unregistered') \ - AND from_peer = 'p' \ - GROUP BY admission_class", - ) - .fetch_all(s.conn()) - .await - .unwrap() - .iter() - .map(|row| row.get::("detail")) - .collect(); - assert!( - plan.iter().any(|step| step.contains("COVERING INDEX idx_p2p_inbox_admission") - && step.contains("from_peer=?")), - "the sender's rows must be reached by an index seek: {plan:?}" - ); - } - - /// Registered rows take the batch ahead of older unregistered ones, but a - /// reserved share keeps the unregistered queue moving. - #[tokio::test] - async fn claimable_listing_drains_registered_rows_first() { - use P2pInboxAdmissionClass::{Committee, Registered, Unregistered}; - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let mut created_at = 100; - let mut insert = async |s: &mut StorageProcessor<'_>, id: String, class| { - let message = admission_inbox_message(&id, "peer", class, id.clone().into_bytes()); - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - sqlx::query("UPDATE p2p_inbox SET created_at = ? WHERE message_id = ?") - .bind(created_at) - .bind(&id) - .execute(s.conn()) - .await - .unwrap(); - created_at += 1; - }; - // The flood arrived first ... - for index in 0..8 { - insert(&mut s, format!("flood-{index}"), Unregistered).await; - } - // ... and the registered messages queue up behind it, the committee's - // last of all. - for index in 0..4 { - insert(&mut s, format!("member-{index}"), Registered).await; - } - for index in 4..6 { - insert(&mut s, format!("member-{index}"), Committee).await; - } - let ids = |rows: Vec| -> Vec { - rows.into_iter().map(|row| row.message_id).collect() - }; - - let plain = ids(s.list_claimable_p2p_inbox_messages(1000, 4, 3, &[]).await.unwrap()); - assert_eq!(plain, ["flood-0", "flood-1", "flood-2", "flood-3"], "oldest first"); - - let fair = - ids(s.list_claimable_p2p_inbox_messages_by_class(1000, 4, 1, 1, 3, &[]).await.unwrap()); - assert_eq!(fair, ["member-4", "member-5", "member-0", "flood-0"], "committee first"); - - // With fewer registered rows than the batch, the rest goes to the others. - let fair = ids(s - .list_claimable_p2p_inbox_messages_by_class(1000, 10, 2, 2, 3, &[]) - .await - .unwrap()); - assert_eq!(fair.len(), 10); - assert_eq!(fair.iter().filter(|id| id.starts_with("member-")).count(), 6); - assert_eq!(&fair[6..], ["flood-0", "flood-1", "flood-2", "flood-3"]); - - // A committee backlog that would fill every batch does not starve the - // other registered senders, nor the unregistered ones. - for index in 0..8 { - insert(&mut s, format!("committee-backlog-{index}"), Committee).await; - } - let fair = - ids(s.list_claimable_p2p_inbox_messages_by_class(1000, 4, 1, 1, 3, &[]).await.unwrap()); - assert_eq!(fair, ["member-4", "member-5", "member-0", "flood-0"]); - - // Lend unused reserved capacity to committee first. - let fair = ids(s - .list_claimable_p2p_inbox_messages_by_class(1000, 16, 6, 4, 3, &[]) - .await - .unwrap()); - assert_eq!(fair.len(), 16); - assert_eq!(fair.iter().filter(|id| id.starts_with("flood-")).count(), 4); - assert_eq!(fair.iter().filter(|id| id.starts_with("committee-backlog-")).count(), 6); - let mut unique = fair.clone(); - unique.sort(); - unique.dedup(); - assert_eq!( - unique.len(), - 16, - "borrowing continues a class's listing, it does not repeat it" - ); - } - - /// Verify confirmed operator bindings are loaded before unconfirmed identities. - #[tokio::test] - async fn confirmed_operator_bindings_are_loaded_first() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let operator = |peer_id: &str, updated_at: i64| Node { - peer_id: peer_id.to_string(), - actor: "Operator".to_string(), - btc_pub_key: format!("key-of-{peer_id}"), - binding_sig: "sig".to_string(), - binding_issued_at: 1, - updated_at, - created_at: updated_at, - ..Default::default() - }; - s.upsert_node(&operator("confirmed", 10)).await.unwrap(); - for index in 0..4 { - s.upsert_node(&operator(&format!("recent-{index}"), 900 + index)).await.unwrap(); - } - // Not an operator, and an operator without a binding: never loaded. - s.upsert_node(&Node { actor: "Committee".to_string(), ..operator("member", 999) }) - .await - .unwrap(); - s.upsert_node(&Node { binding_sig: String::new(), ..operator("unbound", 999) }) - .await - .unwrap(); - s.upsert_p2p_registered_peer("confirmed", "Operator", "aa").await.unwrap(); - - let loaded: Vec = s - .load_p2p_peer_bindings(2) - .await - .unwrap() - .into_iter() - .map(|node| node.peer_id) - .collect(); - assert_eq!(loaded, ["confirmed", "recent-3"]); - } - - /// A sync request names both ids. A real graph under another instance's id - /// is not a graph this node holds for that request. - #[tokio::test] - async fn graph_is_held_only_under_its_own_instance() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let (instance, graph, other) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); - sqlx::query( - "INSERT INTO graph (graph_id, instance_id, status, created_at, updated_at) \ - VALUES (?, ?, 'Created', 1, 1)", - ) - .bind(graph) - .bind(instance) - .execute(s.conn()) - .await - .unwrap(); - assert!(!s.has_graph_of_instance(&instance, &graph).await.unwrap(), "no data stored yet"); - sqlx::query("INSERT INTO graph_raw_data (graph_id, raw_data) VALUES (?, 'x')") - .bind(graph) - .execute(s.conn()) - .await - .unwrap(); - assert!(s.has_graph_of_instance(&instance, &graph).await.unwrap()); - assert!(!s.has_graph_of_instance(&other, &graph).await.unwrap(), "wrong instance"); - assert!(!s.has_graph_of_instance(&instance, &other).await.unwrap(), "unknown graph"); - } - - #[tokio::test] - async fn registered_peers_persist_and_shield_their_node_rows() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - s.upsert_p2p_registered_peer("member", "Committee", "").await.unwrap(); - s.upsert_p2p_registered_peer("member", "Committee", "").await.unwrap(); - s.upsert_p2p_registered_peer("other-member", "Committee", "").await.unwrap(); - s.upsert_p2p_registered_peer("operator", "Operator", "aa").await.unwrap(); - let load = async |s: &mut StorageProcessor<'_>| { - let mut peers = s.load_p2p_registered_peers().await.unwrap(); - peers.sort(); - peers - }; - let row = |peer: &str, kind: &str, key: &str| { - (peer.to_string(), kind.to_string(), key.to_string()) - }; - assert_eq!( - load(&mut s).await, - [ - row("member", "Committee", ""), - row("operator", "Operator", "aa"), - row("other-member", "Committee", "") - ], - "keyless kinds never displace each other" - ); - // Move the key's registration to the new peer. - s.upsert_p2p_registered_peer("operator-moved", "Operator", "aa").await.unwrap(); - // A peer confirmed for another key keeps one row, for the new key. - s.upsert_p2p_registered_peer("operator-moved", "Operator", "bb").await.unwrap(); - assert_eq!( - load(&mut s).await, - [ - row("member", "Committee", ""), - row("operator-moved", "Operator", "bb"), - row("other-member", "Committee", "") - ] - ); - s.delete_p2p_registered_peer("operator-moved", "Operator").await.unwrap(); - s.delete_p2p_registered_peer("other-member", "Committee").await.unwrap(); - assert_eq!(load(&mut s).await.len(), 1); - - // Replay marks only move up, and are kept for registered peers and the - // verifiers the caller names. - s.raise_p2p_replay_marks(&[("member".into(), 50), ("verifier".into(), 7)]).await.unwrap(); - s.raise_p2p_replay_marks(&[("member".into(), 40), ("gone".into(), 9)]).await.unwrap(); - assert_eq!(s.prune_p2p_replay_marks(&["verifier".to_string()]).await.unwrap(), 1); - let mut marks = s.load_p2p_replay_marks().await.unwrap(); - marks.sort(); - assert_eq!(marks, [("member".to_string(), 50), ("verifier".to_string(), 7)]); - - for (peer_id, updated_at) in [("member", 10), ("sybil", 10), ("local", 10), ("live", 900)] { - s.upsert_node(&Node { - peer_id: peer_id.to_string(), - actor: "Operator".to_string(), - updated_at, - created_at: updated_at, - ..Default::default() - }) - .await - .unwrap(); - } - // Only unregistered rows count toward capacity; stale rows may be evicted. - assert!(s.node_row_admissible("sybil", 3, 0, "local").await.unwrap()); - assert!(s.node_row_admissible("newcomer", 4, 0, "local").await.unwrap()); - assert!( - !s.node_row_admissible("newcomer", 3, 5, "local").await.unwrap(), - "full, and every row was refreshed after the eviction horizon" - ); - assert!( - s.node_row_admissible("newcomer", 3, 500, "local").await.unwrap(), - "the stalest unregistered row makes room" - ); - let known = async |s: &mut StorageProcessor<'_>, peer_id: &str| { - s.node_row_admissible(peer_id, 0, 0, "local").await.unwrap() - }; - assert!(!known(&mut s, "sybil").await, "evicted: stale, unregistered, not this node"); - for kept in ["member", "local", "live"] { - assert!(known(&mut s, kept).await, "{kept}"); - } - - s.upsert_node(&Node { - peer_id: "left".to_string(), - actor: "Watchtower".to_string(), - updated_at: 20, - created_at: 20, - ..Default::default() - }) - .await - .unwrap(); - assert_eq!(s.purge_stale_unregistered_nodes(500, "local").await.unwrap(), 1); - assert!(!known(&mut s, "left").await); - assert!(known(&mut s, "member").await && known(&mut s, "local").await); - - s.raise_p2p_replay_marks(&[("locked-out".into(), 99)]).await.unwrap(); - assert_eq!(s.delete_p2p_replay_marks(&["locked-out".to_string()]).await.unwrap(), 1); - assert_eq!(s.delete_p2p_replay_marks(&[]).await.unwrap(), 0); - } - - /// Only a copy that is still queued suppresses a re-publish; once it reached - /// a terminal state the same payload may be delivered again. - #[tokio::test] - async fn queued_payload_lookup_ignores_other_senders_and_terminal_rows() { - use P2pInboxAdmissionClass::Registered; - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let message = admission_inbox_message("dup-1", "peer-a", Registered, vec![9; 8]); - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - - assert!(s.has_queued_p2p_inbox_payload("peer-a", &[9; 8]).await.unwrap()); - assert!(!s.has_queued_p2p_inbox_payload("peer-b", &[9; 8]).await.unwrap()); - assert!(!s.has_queued_p2p_inbox_payload("peer-a", &[8; 8]).await.unwrap()); - - let claimed = s.claim_p2p_inbox_message("dup-1", 100, 200).await.unwrap().unwrap(); - assert!( - s.has_queued_p2p_inbox_payload("peer-a", &[9; 8]).await.unwrap(), - "a claimed row is still in flight" - ); - assert!(s.complete_p2p_inbox_message("dup-1", &claimed.lease_token).await.unwrap()); - assert!(!s.has_queued_p2p_inbox_payload("peer-a", &[9; 8]).await.unwrap()); - } - - /// Verify Pending expiry uses `created_at`, not the last retry time. - #[tokio::test] - async fn pending_inbox_rows_expire_by_class_and_age() { - use P2pInboxAdmissionClass::{Registered, Unregistered}; - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - for (message_id, class, created_at) in [ - ("expire-old-unregistered", Unregistered, 100), - ("expire-new-unregistered", Unregistered, 900), - ("expire-old-registered", Registered, 100), - ("expire-claimed-unregistered", Unregistered, 100), - ] { - let message = admission_inbox_message(message_id, "peer", class, vec![5; 16]); - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - sqlx::query( - "UPDATE p2p_inbox SET created_at = ?, updated_at = 5000 WHERE message_id = ?", - ) - .bind(created_at) - .bind(message_id) - .execute(s.conn()) - .await - .unwrap(); - } - let now = get_current_timestamp_secs(); - assert!( - s.claim_p2p_inbox_message("expire-claimed-unregistered", now, now + 300) - .await - .unwrap() - .is_some() - ); - - assert_eq!(s.expire_pending_p2p_inbox_messages(Some(Unregistered), 500).await.unwrap(), 1); - let states: Vec<(String, String, i64)> = sqlx::query( - "SELECT message_id, state, length(content) AS len FROM p2p_inbox ORDER BY message_id", - ) - .fetch_all(s.conn()) - .await - .unwrap() - .iter() - .map(|row| (row.get("message_id"), row.get("state"), row.get("len"))) - .collect(); - assert_eq!( - states, - vec![ - ("expire-claimed-unregistered".to_string(), "Processing".to_string(), 16), - ("expire-new-unregistered".to_string(), "Pending".to_string(), 16), - ("expire-old-registered".to_string(), "Pending".to_string(), 16), - ("expire-old-unregistered".to_string(), "Failed".to_string(), 0), - ] - ); - - // The class-independent ceiling reaches every remaining pending row. - assert_eq!(s.expire_pending_p2p_inbox_messages(None, 1000).await.unwrap(), 2); - assert_eq!(s.p2p_inbox_usage("peer").await.unwrap().iter().map(|u| u.rows).sum::(), 1); - } - - /// Verify only unfinished claims increment abandon_count. - #[tokio::test] - async fn test_inbox_abandon_is_charged_only_for_unfinished_claims() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let message = P2pInboxMessage { - message_id: "inbox-abandon-1".to_string(), - actor: "Operator".to_string(), - from_peer: "peer".to_string(), - msg_type: "CreateGraph".to_string(), - content: vec![1, 2, 3], - content_size: 3, - ..Default::default() - }; - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - - // First claim of a Pending row: a retry attempt, not an abandon. - let claimed = s.claim_p2p_inbox_messages(100, 200, 10, 3, &[]).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].attempt_count, 1); - assert_eq!(claimed[0].abandon_count, 0); - - // The handler returned a retryable error and the row went back to Pending. - assert!( - s.retry_p2p_inbox_message( - &message.message_id, - &claimed[0].lease_token, - 150, - "storage busy" - ) - .await - .unwrap() - ); - let claimed = s.claim_p2p_inbox_messages(160, 260, 10, 3, &[]).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].attempt_count, 2, "a retry is recorded"); - assert_eq!(claimed[0].abandon_count, 0, "a retry must not charge the abandon budget"); - - // Now simulate a worker that died mid-dispatch: the row is still - // Processing and its lease has expired. - let claimed = s.claim_p2p_inbox_messages(400, 500, 10, 3, &[]).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].attempt_count, 3); - assert_eq!(claimed[0].abandon_count, 1, "an unfinished claim charges the abandon budget"); - - assert!( - s.retry_p2p_inbox_message( - &message.message_id, - &claimed[0].lease_token, - 450, - "dependency pending", - ) - .await - .unwrap() - ); - let claimed = s.claim_p2p_inbox_messages(460, 560, 10, 3, &[]).await.unwrap(); - assert_eq!(claimed[0].abandon_count, 0, "a reported outcome resets consecutive abandons"); - } - - /// Verify repeated abandonment quarantines the retained payload. - #[tokio::test] - async fn test_inbox_quarantines_repeatedly_abandoned_message() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let message = P2pInboxMessage { - message_id: "inbox-poison-1".to_string(), - actor: "Operator".to_string(), - from_peer: "peer".to_string(), - msg_type: "GraphFinalize".to_string(), - content: vec![9; 64], - content_size: 64, - ..Default::default() - }; - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - - // Claims that never report an outcome, each one lease apart. On the - // sweep after the third expired claim, that final abandon is recorded - // as part of the quarantine transition. - let mut now = 100; - for attempt in 1..=3 { - let claimed = s.claim_p2p_inbox_messages(now, now + 10, 10, 3, &[]).await.unwrap(); - assert_eq!(claimed.len(), 1, "claim {attempt} should still be served"); - assert_eq!( - claimed[0].abandon_count, - attempt - 1, - "claim {attempt} charges one abandon per unfinished predecessor" - ); - now += 100; - } - - let quarantined = s.quarantine_p2p_inbox_messages(now, 3).await.unwrap(); - assert_eq!(quarantined, 1); - - let claimed = s.claim_p2p_inbox_messages(now, now + 10, 10, 3, &[]).await.unwrap(); - assert!(claimed.is_empty(), "a quarantined message must not be claimed again"); - - let row = - sqlx::query("SELECT state, content, last_error FROM p2p_inbox WHERE message_id = ?") - .bind(&message.message_id) - .fetch_one(s.conn()) - .await - .unwrap(); - assert_eq!(row.get::("state"), "Quarantined"); - assert_eq!(row.get::, _>("content"), message.content); - assert!(row.get::, _>("last_error").is_some()); - - assert!(s.requeue_p2p_inbox_message(&message.message_id).await.unwrap()); - let requeued = s.claim_p2p_inbox_messages(now, now + 10, 10, 3, &[]).await.unwrap(); - assert_eq!(requeued.len(), 1); - assert_eq!(requeued[0].abandon_count, 0); - } - - /// Verify local claims hold a lease. - #[tokio::test] - async fn test_local_claim_holds_lease_and_charges_abandon() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES (?, ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", - ) - .bind("local-claim-1") - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - - let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].abandon_count, 0); - - // Still inside the lease: the message must not be handed out again. - let claimed_again = s.claim_local_messages(150, 250, 0, 10, 3).await.unwrap(); - assert!(claimed_again.is_empty(), "a leased message must not be re-claimed"); - - // Lease expired with no outcome reported: that is an abandon. - let reclaimed = s.claim_local_messages(300, 400, 0, 10, 3).await.unwrap(); - assert_eq!(reclaimed.len(), 1); - assert_eq!(reclaimed[0].abandon_count, 1); - } - - #[tokio::test] - async fn graceful_shutdown_releases_claims_without_charging_abandon() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES ('shutdown-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 10, 10)", - ) - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - let inbox = P2pInboxMessage { - message_id: "shutdown-inbox".to_owned(), - actor: "Operator".to_owned(), - from_peer: "peer".to_owned(), - msg_type: "CreateGraph".to_owned(), - content: vec![1], - content_size: 1, - ..Default::default() - }; - assert!(s.insert_p2p_inbox_message(&inbox).await.unwrap()); - assert_eq!(s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap().len(), 1); - assert_eq!(s.claim_p2p_inbox_messages(100, 200, 10, 3, &[]).await.unwrap().len(), 1); - - assert_eq!(s.release_processing_local_messages().await.unwrap(), 1); - assert_eq!(s.release_processing_p2p_inbox_messages().await.unwrap(), 1); - - let local = s.find_messages_by_id("shutdown-local").await.unwrap().unwrap(); - assert_eq!(local.state, "Pending"); - assert_eq!(local.abandon_count, 0); - let inbox = sqlx::query( - "SELECT state, abandon_count, lease_token FROM p2p_inbox WHERE message_id = ?", - ) - .bind("shutdown-inbox") - .fetch_one(s.conn()) - .await - .unwrap(); - assert_eq!(inbox.get::("state"), "Pending"); - assert_eq!(inbox.get::("abandon_count"), 0); - assert!(inbox.get::("lease_token").is_empty()); - } - - /// Deferring a local message records the retry, but only abandoned claims - /// contribute to quarantine. - #[tokio::test] - async fn test_local_defer_charges_attempts_but_only_abandons_quarantine() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES (?, ?, 'Operator', 'AssertReady', X'0102', 'Pending', 0, 10, 10)", - ) - .bind("local-defer-1") - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - - let claimed = s.claim_local_messages(100, 200, 0, 10, 3).await.unwrap(); - assert_eq!(claimed.len(), 1); - assert!( - s.defer_local_message(&claimed[0].message_id, claimed[0].message_version, 150, "boom") - .await - .unwrap() - ); - - let reclaimed = s.claim_local_messages(160, 260, 0, 10, 3).await.unwrap(); - assert_eq!(reclaimed.len(), 1); - assert_eq!(reclaimed[0].attempt_count, 1, "defer records the attempt"); - assert_eq!(reclaimed[0].abandon_count, 0, "defer must not charge the abandon budget"); - - // A deferred message is NOT quarantined however many times it errors: - // the local queue has no error budget, because a surviving row makes the - // producer treat the work as already created. - assert_eq!(s.quarantine_local_messages(300, 3).await.unwrap(), 0); - - // Abandoned claims are what retires it. The quarantine sweep counts the - // final expired Processing lease. - let mut now = 400; - for _ in 0..3 { - s.claim_local_messages(now, now + 10, 0, 10, 3).await.unwrap(); - now += 100; - } - assert_eq!(s.quarantine_local_messages(now, 3).await.unwrap(), 1); - let row = sqlx::query("SELECT state, content FROM message WHERE message_id = ?") - .bind("local-defer-1") - .fetch_one(s.conn()) - .await - .unwrap(); - assert_eq!(row.get::("state"), "Quarantined"); - assert_eq!(row.get::, _>("content"), vec![1, 2]); - } - - /// Rows are claimed one at a time immediately before dispatch, so an abort - /// mid-dispatch charges only the row that was actually running. - #[tokio::test] - async fn local_claims_are_taken_per_message_not_per_batch() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - for (message_id, created_at) in [("first", 10), ("second", 20)] { - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES (?, ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, ?, ?)", - ) - .bind(message_id) - .bind(business_id) - .bind(created_at) - .bind(created_at) - .execute(s.conn()) - .await - .unwrap(); - } - - let candidates = s.list_claimable_local_messages(100, 0, 10, 3).await.unwrap(); - assert_eq!( - candidates.iter().map(|message| message.message_id.as_str()).collect::>(), - ["first", "second"] - ); - assert!( - candidates.iter().all(|message| message.state == "Pending"), - "listing must not write" - ); - - let claimed = s - .claim_local_message("first", candidates[0].message_version, 100, 200) - .await - .unwrap() - .expect("first claim"); - assert_eq!(claimed.state, "Processing"); - assert_eq!(claimed.lock_time_until, 200); - assert_eq!(s.find_messages_by_id("second").await.unwrap().unwrap().state, "Pending"); - - // Inside the lease the same row is not claimable again. - assert!( - s.claim_local_message("first", claimed.message_version, 150, 250) - .await - .unwrap() - .is_none() - ); - // A row re-armed under a new version since it was listed is skipped too. - assert!( - s.claim_local_message("second", candidates[1].message_version + 1, 100, 200) - .await - .unwrap() - .is_none() - ); - // Once the lease lapses, the reclaim charges the unfinished attempt. - let reclaimed = s - .claim_local_message("first", claimed.message_version, 300, 400) - .await - .unwrap() - .expect("reclaim"); - assert_eq!(reclaimed.abandon_count, 1); - } - - #[tokio::test] - async fn inbox_claims_are_taken_per_message_not_per_batch() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - for message_id in ["inbox-first", "inbox-second"] { - let message = P2pInboxMessage { - message_id: message_id.to_owned(), - actor: "Operator".to_owned(), - from_peer: "peer".to_owned(), - msg_type: "CreateGraph".to_owned(), - content: vec![1], - content_size: 1, - ..Default::default() - }; - assert!(s.insert_p2p_inbox_message(&message).await.unwrap()); - } - let candidates = s.list_claimable_p2p_inbox_messages(100, 10, 3, &[]).await.unwrap(); - assert_eq!(candidates.len(), 2); - assert!( - candidates - .iter() - .all(|message| message.state == "Pending" && message.lease_token.is_empty()), - "listing must not write" - ); - - let claimed = - s.claim_p2p_inbox_message("inbox-first", 100, 200).await.unwrap().expect("claim"); - assert_eq!(claimed.state, "Processing"); - assert_eq!(claimed.attempt_count, 1); - assert_eq!(claimed.lease_until, 200); - assert!(!claimed.lease_token.is_empty()); - assert!(s.claim_p2p_inbox_message("inbox-first", 150, 250).await.unwrap().is_none()); - let second = sqlx::query("SELECT state FROM p2p_inbox WHERE message_id = 'inbox-second'") - .fetch_one(s.conn()) - .await - .unwrap(); - assert_eq!(second.get::("state"), "Pending"); - } - - /// An unclean exit leaves claims behind. At startup they are provably - /// abandoned, so they are charged and released immediately with a backoff - /// instead of sitting locked until the lease lapses. - #[tokio::test] - async fn startup_reclaim_charges_abandon_and_applies_backoff() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES ('startup-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 10, 10)", - ) - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - let inbox = P2pInboxMessage { - message_id: "startup-inbox".to_owned(), - actor: "Operator".to_owned(), - from_peer: "peer".to_owned(), - msg_type: "CreateGraph".to_owned(), - content: vec![1], - content_size: 1, - ..Default::default() - }; - assert!(s.insert_p2p_inbox_message(&inbox).await.unwrap()); - assert_eq!(s.claim_local_messages(100, 700, 0, 10, 3).await.unwrap().len(), 1); - assert_eq!(s.claim_p2p_inbox_messages(100, 400, 10, 3, &[]).await.unwrap().len(), 1); - - assert_eq!(s.reclaim_processing_local_messages(1000, 60).await.unwrap(), 1); - assert_eq!(s.reclaim_processing_p2p_inbox_messages(1000, 60).await.unwrap(), 1); - - let local = s.find_messages_by_id("startup-local").await.unwrap().unwrap(); - assert_eq!(local.state, "Pending"); - assert_eq!(local.abandon_count, 1); - assert_eq!(local.lock_time_until, 1060, "the first abandon backs off by one interval"); - let inbox_row = sqlx::query( - "SELECT state, abandon_count, next_retry_at, lease_token FROM p2p_inbox WHERE message_id = 'startup-inbox'", - ) - .fetch_one(s.conn()) - .await - .unwrap(); - assert_eq!(inbox_row.get::("state"), "Pending"); - assert_eq!(inbox_row.get::("abandon_count"), 1); - assert_eq!(inbox_row.get::("next_retry_at"), 1060); - assert!(inbox_row.get::("lease_token").is_empty()); - - // Nothing is claimable until the backoff has passed. - assert!(s.list_claimable_local_messages(1030, 0, 10, 3).await.unwrap().is_empty()); - assert_eq!(s.list_claimable_local_messages(1060, 0, 10, 3).await.unwrap().len(), 1); - assert!(s.list_claimable_p2p_inbox_messages(1030, 10, 3, &[]).await.unwrap().is_empty()); - assert_eq!(s.list_claimable_p2p_inbox_messages(1060, 10, 3, &[]).await.unwrap().len(), 1); - } - - /// A handler that reschedules its own row and then panics must accumulate - /// abandons across restarts. The self-defer must not reset the counter, - /// because the handler is still running when it happens; only the - /// dispatcher's confirmation after a normal return may reset it. - #[tokio::test] - async fn panic_after_self_defer_accumulates_abandons_until_quarantine() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) \ - VALUES ('panic-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 10, 10)", - ) - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - - let mut now = 100; - for round in 1..=3 { - // The row is Pending, so the restart's reclaim sweep leaves it alone - // and the next tick claims it without a charge. - let candidate = s.find_messages_by_id("panic-local").await.unwrap().unwrap(); - let claimed = s - .claim_local_message("panic-local", candidate.message_version, now, now + 600) - .await - .unwrap() - .expect("claim"); - assert_eq!(claimed.abandon_count, round - 1); - // The handler reschedules itself, then panics. - assert!( - s.self_defer_local_message( - "panic-local", - claimed.message_version, - now + 5, - "not ready" - ) - .await - .unwrap() - ); - assert!( - s.abandon_local_message( - "panic-local", - claimed.message_version, - now, - 60, - "panicked" - ) - .await - .unwrap() - ); - let row = s.find_messages_by_id("panic-local").await.unwrap().unwrap(); - assert_eq!(row.state, "Pending"); - assert_eq!(row.abandon_count, round, "round {round} adds to the preserved count"); - assert_eq!(row.lock_time_until, now + 60 * round, "backoff grows with the count"); - now = row.lock_time_until + 1; - } - - // The sweep on the next tick retires it instead of dispatching again. - assert_eq!(s.quarantine_local_messages(now, 3).await.unwrap(), 1); - assert_eq!( - s.find_messages_by_id("panic-local").await.unwrap().unwrap().state, - "Quarantined" - ); - } - - /// A self-defer is only a reported outcome once the handler has returned, - /// so the counter survives the self-defer and resets on confirmation. - #[tokio::test] - async fn self_defer_keeps_abandons_until_the_dispatcher_confirms_it() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, abandon_count, created_at, updated_at) \ - VALUES ('confirm-local', ?, 'Operator', 'AssertReady', X'01', 'Pending', 0, 2, 10, 10)", - ) - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - let candidate = s.find_messages_by_id("confirm-local").await.unwrap().unwrap(); - let claimed = s - .claim_local_message("confirm-local", candidate.message_version, 100, 700) - .await - .unwrap() - .expect("claim"); - assert_eq!(claimed.abandon_count, 2, "a claim from Pending is not charged"); - - assert!( - s.self_defer_local_message("confirm-local", claimed.message_version, 150, "not ready") - .await - .unwrap() - ); - let row = s.find_messages_by_id("confirm-local").await.unwrap().unwrap(); - assert_eq!(row.state, "Pending"); - assert_eq!(row.attempt_count, 1); - assert_eq!(row.abandon_count, 2, "the self-defer must not reset the counter"); - - // The dispatcher confirms once the handler returned normally. - assert!( - s.confirm_local_message_self_defer("confirm-local", claimed.message_version) - .await - .unwrap() - ); - assert_eq!(s.find_messages_by_id("confirm-local").await.unwrap().unwrap().abandon_count, 0); - // A different claim generation, or a row that is not Pending, is not confirmed. - assert!( - !s.confirm_local_message_self_defer("confirm-local", claimed.message_version + 1) - .await - .unwrap() - ); - } - - #[tokio::test] - async fn test_message_debug_reasons_are_deduplicated() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let business_id = Uuid::new_v4(); - - sqlx::query( - "INSERT INTO message (message_id, business_id, actor, msg_type, content, state, lock_time_until, created_at, updated_at) VALUES (?, ?, 'Operator', 'AssertReady', X'00', 'Pending', 30, 10, 20)", - ) - .bind("message-debug-1") - .bind(business_id) - .execute(s.conn()) - .await - .unwrap(); - - for _ in 0..2 { - s.upsert_message_debug_reason( - "message-debug-1", - "operator_proof_pending", - "operator proof is not ready", - ) - .await - .unwrap(); - } - s.upsert_message_debug_reason( - "message-debug-1", - "handler_error", - "proof RPC request timed out", - ) - .await - .unwrap(); - - let messages = s.find_message_debug_overviews(&business_id).await.unwrap(); - assert_eq!(messages.len(), 1); - assert_eq!(messages[0].reason_count, 2); - - let reasons = s.find_message_debug_reasons("message-debug-1").await.unwrap(); - assert_eq!(reasons.len(), 2); - assert!(reasons.iter().any(|reason| { - reason.reason_code == "operator_proof_pending" && reason.occurrences == 2 - })); - assert!( - reasons - .iter() - .any(|reason| reason.reason_code == "handler_error" && reason.occurrences == 1) - ); - } - - #[tokio::test] - async fn test_proof_metrics_state_counts() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - - sqlx::query("DELETE FROM long_running_task_proof").execute(s.conn()).await.unwrap(); - - sqlx::query( - "INSERT INTO long_running_task_proof (block_start, block_end, chain_name, proof_state, created_at, updated_at) VALUES (0, 1, 'header-chain', 2, 40, 90), (1, 2, 'header-chain', 2, 20, 100), (2, 3, 'header-chain', 0, 10, 10)", - ) - .execute(s.conn()) - .await - .unwrap(); - sqlx::query( - "INSERT INTO operator_proof (instance_id, graph_id, execution_layer_block_number, operator_committed_blockhash, proof_state, created_at, updated_at) VALUES ('instance', 'graph', 0, 'hash', 2, 50, 80)", - ) - .execute(s.conn()) - .await - .unwrap(); - sqlx::query( - "INSERT INTO watchtower_proof (instance_id, graph_id, public_key, challenge_txid, challenge_init_txid, execution_layer_block_number, proof_state, created_at, updated_at) VALUES ('instance', 'graph', 'key', 'challenge', 'init', 0, 1, 60, 70)", - ) - .execute(s.conn()) - .await - .unwrap(); - - let counts = s.proof_metrics_state_counts().await.unwrap(); - assert_eq!( - counts.iter().find(|count| count.category == "header-chain" && count.state == "2"), - Some(&MetricsStateCount { - category: "header-chain".to_string(), - state: "2".to_string(), - count: 2, - oldest_created_at: None, - last_success_at: Some(100), - }) - ); - assert_eq!( - counts - .iter() - .find(|count| count.category == "operator" && count.state == "2") - .unwrap() - .last_success_at, - Some(80) - ); - assert_eq!( - counts - .iter() - .find(|count| count.category == "watchtower" && count.state == "1") - .unwrap() - .last_success_at, - None - ); - } - - #[tokio::test] - async fn test_upsert_and_find_latest_hash_change() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - - s.upsert_sequencer_set_hash_change(100, 1000, "aabbcc").await.unwrap(); - s.upsert_sequencer_set_hash_change(200, 2000, "ddeeff").await.unwrap(); - s.upsert_sequencer_set_hash_change(150, 1500, "112233").await.unwrap(); - - let latest = s.find_latest_sequencer_set_hash_change().await.unwrap().unwrap(); - assert_eq!(latest.cosmos_block_height, 200); - assert_eq!(latest.goat_block_height, 2000); - assert_eq!(latest.validators_hash, "ddeeff"); - } - - #[tokio::test] - async fn test_upsert_hash_change_conflict() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - - s.upsert_sequencer_set_hash_change(100, 1000, "aabbcc").await.unwrap(); - // Same cosmos_block_height, different data — should overwrite - s.upsert_sequencer_set_hash_change(100, 1001, "ddeeff").await.unwrap(); - - let latest = s.find_latest_sequencer_set_hash_change().await.unwrap().unwrap(); - assert_eq!(latest.cosmos_block_height, 100); - assert_eq!(latest.goat_block_height, 1001); - assert_eq!(latest.validators_hash, "ddeeff"); - } - - #[tokio::test] - async fn test_find_by_goat_block_at_or_before() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - - s.upsert_sequencer_set_hash_change(100, 1000, "aa").await.unwrap(); - s.upsert_sequencer_set_hash_change(200, 2000, "bb").await.unwrap(); - s.upsert_sequencer_set_hash_change(300, 3000, "cc").await.unwrap(); - - // Exact match - let r = s - .find_first_sequencer_set_hash_change_by_goat_block_at_or_before(2000) - .await - .unwrap() - .unwrap(); - assert_eq!(r.goat_block_height, 2000); - assert_eq!(r.validators_hash, "bb"); - - // Between records — should return previous one - let r = s - .find_first_sequencer_set_hash_change_by_goat_block_at_or_before(2500) - .await - .unwrap() - .unwrap(); - assert_eq!(r.goat_block_height, 2000); - assert_eq!(r.validators_hash, "bb"); - - // Before all records — should return None - let r = - s.find_first_sequencer_set_hash_change_by_goat_block_at_or_before(500).await.unwrap(); - assert!(r.is_none()); - } - - #[tokio::test] - async fn test_scan_state_upsert_and_get() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - - assert!(s.get_sequencer_set_scan_state().await.unwrap().is_none()); - - s.upsert_sequencer_set_scan_state(100, 1000, "aabbcc").await.unwrap(); - let state = s.get_sequencer_set_scan_state().await.unwrap().unwrap(); - assert_eq!(state.next_cosmos_block_height, 100); - assert_eq!(state.latest_goat_block_height, 1000); - assert_eq!(state.latest_validators_hash, "aabbcc"); - - // Update - s.upsert_sequencer_set_scan_state(200, 2000, "ddeeff").await.unwrap(); - let state = s.get_sequencer_set_scan_state().await.unwrap().unwrap(); - assert_eq!(state.next_cosmos_block_height, 200); - assert_eq!(state.latest_validators_hash, "ddeeff"); - } - - fn on_demand_completion() -> OnDemandProofCompletion { - OnDemandProofCompletion { - path_to_proof: "proofs/1.bin".to_string(), - public_value_hex: "00".to_string(), - proof_size: 1, - cycles: 2, - proof_state: crate::ProofState::Proven.to_i64(), - total_time_to_proof: 3, - proving_time: 4, - zkm_version: "v2".to_string(), - } - } - - async fn new_operator_proof(s: &mut StorageProcessor<'_>) -> OperatorProof { - let proof = OperatorProof { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - proof_state: crate::ProofState::New.to_i64(), - ..Default::default() - }; - assert_eq!(s.create_operator_proof(&proof).await.unwrap(), 1); - s.find_next_operator_proof().await.unwrap().unwrap() - } - - #[tokio::test] - async fn test_complete_on_demand_proof_only_while_new() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let task = new_operator_proof(&mut s).await; - let table = OnDemandProofTable::Operator; - - let completion = on_demand_completion(); - assert_eq!( - s.complete_on_demand_proof_if_new(table, task.id, &completion).await.unwrap(), - 1 - ); - assert_eq!( - s.get_on_demand_proof_state(table, task.id).await.unwrap(), - Some(crate::ProofState::Proven.to_i64()) - ); - // A finished task is never rewritten, including by a later timeout. - assert_eq!( - s.complete_on_demand_proof_if_new(table, task.id, &completion).await.unwrap(), - 0 - ); - let timed_out = s - .update_operator_proof_state_with_instance_graph( - &task.instance_id, - &task.graph_id, - crate::ProofState::New.to_i64(), - crate::ProofState::Failed.to_i64(), - ) - .await - .unwrap(); - assert_eq!(timed_out, 0); - assert_eq!(s.get_on_demand_proof_state(table, i64::MAX).await.unwrap(), None); - } - - #[tokio::test] - async fn test_timed_out_on_demand_proof_is_not_completed() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let task = new_operator_proof(&mut s).await; - s.update_operator_proof_state_with_instance_graph( - &task.instance_id, - &task.graph_id, - crate::ProofState::New.to_i64(), - crate::ProofState::Failed.to_i64(), - ) - .await - .unwrap(); - - let table = OnDemandProofTable::Operator; - let completion = on_demand_completion(); - assert_eq!( - s.complete_on_demand_proof_if_new(table, task.id, &completion).await.unwrap(), - 0 - ); - assert_eq!( - s.get_on_demand_proof_state(table, task.id).await.unwrap(), - Some(crate::ProofState::Failed.to_i64()) - ); - - let watchtower = WatchtowerProof { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - public_key: "pk".to_string(), - proof_state: crate::ProofState::New.to_i64(), - ..Default::default() - }; - s.create_watchtower_proof(&watchtower).await.unwrap(); - let watchtower_task = s.find_next_watchtower_proof().await.unwrap().unwrap(); - s.update_watchtower_proof_state_with_instance_graph_pubkey( - &watchtower.instance_id, - &watchtower.graph_id, - "pk", - crate::ProofState::New.to_i64(), - crate::ProofState::Failed.to_i64(), - ) - .await - .unwrap(); - let table = OnDemandProofTable::Watchtower; - assert_eq!( - s.complete_on_demand_proof_if_new(table, watchtower_task.id, &completion) - .await - .unwrap(), - 0 - ); - } - - #[tokio::test] - async fn test_count_pending_long_running_task_proofs() { - let db = setup_db().await; - let mut s = db.acquire().await.unwrap(); - let chain = "state-chain".to_string(); - for (start, state) in [ - (5, crate::ProofState::New), - (10, crate::ProofState::Proven), - (20, crate::ProofState::New), - (30, crate::ProofState::Proving), - (40, crate::ProofState::Failed), - ] { - s.create_long_running_task_proof(&LongRunningTaskProof { - block_start: start, - block_end: start + 10, - chain_name: chain.clone(), - proof_state: state.to_i64(), - ..Default::default() - }) - .await - .unwrap(); - } - assert_eq!(s.count_pending_long_running_task_proofs(&chain, 20).await.unwrap(), 2); - assert_eq!(s.count_pending_long_running_task_proofs(&chain, 0).await.unwrap(), 3); - assert_eq!(s.count_pending_long_running_task_proofs("header-chain", 0).await.unwrap(), 0); - } -} diff --git a/crates/store/src/localdb/goat.rs b/crates/store/src/localdb/goat.rs new file mode 100644 index 000000000..14b16d83d --- /dev/null +++ b/crates/store/src/localdb/goat.rs @@ -0,0 +1,288 @@ +//! Goat-side records: watched contracts, Goat transactions and the sequencer +//! set hash history. + +use super::{StorageProcessor, get_current_timestamp_secs}; +use crate::{GoatTxRecord, SequencerSetHashChange, SequencerSetScanState, WatchContract}; + +use sqlx::types::Uuid; + +impl StorageProcessor<'_> { + /// Check graph data exists under both IDs without reading the blob. + pub async fn find_watch_contract( + &mut self, + addr: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_as!( + WatchContract, + "SELECT * + FROM watch_contract + WHERE contract_addr = ?", + addr + ) + .fetch_optional(self.conn()) + .await?) + } + + pub async fn upsert_watch_contract( + &mut self, + watch_contract: &WatchContract, + ) -> anyhow::Result<()> { + let _ = sqlx::query!( + "INSERT INTO watch_contract (contract_addr, the_graph_url, gap, from_height, status, extra, updated_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (contract_addr) DO UPDATE SET the_graph_url = excluded.the_graph_url, + gap = excluded.gap, + from_height = excluded.from_height, + status = excluded.status, + extra = excluded.extra, + updated_at = excluded.updated_at", + watch_contract.contract_addr, + watch_contract.the_graph_url, + watch_contract.gap, + watch_contract.from_height, + watch_contract.status, + watch_contract.extra, + watch_contract.updated_at, + watch_contract.created_at + ).execute(self.conn()).await; + Ok(()) + } + + pub async fn update_watch_contract_status( + &mut self, + contract_addr: &str, + status: &str, + updated_at: i64, + ) -> anyhow::Result<()> { + let _ = sqlx::query!( + "UPDATE watch_contract SET status = ?, updated_at = ? WHERE contract_addr = ?", + status, + updated_at, + contract_addr, + ) + .execute(self.conn()) + .await; + Ok(()) + } + + pub async fn upsert_goat_tx_record( + &mut self, + goat_tx_record: &GoatTxRecord, + ) -> anyhow::Result<()> { + let mut update_goat_tx_record = goat_tx_record.clone(); + if let Some(goat_tx_record_store) = self + .find_graph_goat_tx_record( + &goat_tx_record.instance_id, + &goat_tx_record.graph_id, + &goat_tx_record.tx_type, + ) + .await? + { + update_goat_tx_record.created_at = goat_tx_record_store.created_at; + update_goat_tx_record.is_local = goat_tx_record_store.is_local; + if goat_tx_record_store.is_processed() { + update_goat_tx_record.processing_status = goat_tx_record_store.processing_status; + } + if update_goat_tx_record.height < goat_tx_record_store.height { + update_goat_tx_record.height = goat_tx_record_store.height; + } + if goat_tx_record_store.extra.is_some() && update_goat_tx_record.extra.is_none() { + update_goat_tx_record.extra = goat_tx_record_store.extra.clone(); + } + + if !goat_tx_record_store.tx_hash.is_empty() { + update_goat_tx_record.tx_hash = goat_tx_record_store.tx_hash.clone(); + } + } + sqlx::query!( + "INSERT OR + REPLACE INTO goat_tx_record (instance_id, + graph_id, + tx_type, + tx_hash, + height, + is_local, + processing_status, + extra, + created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + update_goat_tx_record.instance_id, + update_goat_tx_record.graph_id, + update_goat_tx_record.tx_type, + update_goat_tx_record.tx_hash, + update_goat_tx_record.height, + update_goat_tx_record.is_local, + update_goat_tx_record.processing_status, + update_goat_tx_record.extra, + update_goat_tx_record.created_at + ) + .execute(self.conn()) + .await?; + + Ok(()) + } + + pub async fn find_graph_goat_tx_record( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + tx_type: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_as!( + GoatTxRecord, + "SELECT instance_id AS \"instance_id:Uuid\", + graph_id AS \"graph_id:Uuid\", + tx_type, + tx_hash, + height, + is_local, + processing_status, + extra, + created_at + FROM goat_tx_record + WHERE instance_id = ? + AND graph_id = ? + AND tx_type = ?", + instance_id, + graph_id, + tx_type + ) + .fetch_optional(self.conn()) + .await?) + } + + pub async fn get_goat_tx_record_by_processing_status( + &mut self, + tx_type: &str, + processing_status: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_as!( + GoatTxRecord, + "SELECT instance_id AS \"instance_id:Uuid\", + graph_id AS \"graph_id:Uuid\", + tx_type, tx_hash, + height, + is_local, + processing_status, + extra, + created_at + FROM goat_tx_record + WHERE tx_type = ? + AND processing_status = ? + ORDER BY height ASC", + tx_type, + processing_status + ) + .fetch_all(self.conn()) + .await?) + } + + pub async fn update_goat_tx_record_processing_status( + &mut self, + graph_id: &Uuid, + instance_id: &Uuid, + tx_type: &str, + status: &str, + ) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE goat_tx_record + SET processing_status = ? + where instance_id = ? + AND graph_id = ? + AND tx_type = ?", + status, + instance_id, + graph_id, + tx_type + ) + .execute(self.conn()) + .await?; + Ok(()) + } + + pub async fn upsert_sequencer_set_hash_change( + &mut self, + cosmos_block_height: i64, + goat_block_height: i64, + validators_hash: &str, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query( + r#"INSERT INTO sequencer_set_hash_changes (cosmos_block_height, goat_block_height, validators_hash, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (cosmos_block_height) DO UPDATE + SET goat_block_height = excluded.goat_block_height, + validators_hash = excluded.validators_hash, + updated_at = excluded.updated_at"#, + ) + .bind(cosmos_block_height) + .bind(goat_block_height) + .bind(validators_hash) + .bind(current_time) + .bind(current_time) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn find_latest_sequencer_set_hash_change( + &mut self, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, SequencerSetHashChange>( + "SELECT * FROM sequencer_set_hash_changes ORDER BY cosmos_block_height DESC LIMIT 1", + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_first_sequencer_set_hash_change_by_goat_block_at_or_before( + &mut self, + goat_block_height: i64, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, SequencerSetHashChange>( + "SELECT * FROM sequencer_set_hash_changes WHERE goat_block_height <= ? ORDER BY goat_block_height DESC LIMIT 1", + ) + .bind(goat_block_height) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn get_sequencer_set_scan_state( + &mut self, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, SequencerSetScanState>( + "SELECT * FROM sequencer_set_scan_state WHERE id = 1 LIMIT 1", + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn upsert_sequencer_set_scan_state( + &mut self, + next_cosmos_block_height: i64, + latest_goat_block_height: i64, + latest_validators_hash: &str, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query( + r#"INSERT INTO sequencer_set_scan_state (id, next_cosmos_block_height, latest_goat_block_height, latest_validators_hash, created_at, updated_at) + VALUES (1, ?, ?, ?, ?, ?) + ON CONFLICT (id) DO UPDATE + SET next_cosmos_block_height = excluded.next_cosmos_block_height, + latest_goat_block_height = excluded.latest_goat_block_height, + latest_validators_hash = excluded.latest_validators_hash, + updated_at = excluded.updated_at"#, + ) + .bind(next_cosmos_block_height) + .bind(latest_goat_block_height) + .bind(latest_validators_hash) + .bind(current_time) + .bind(current_time) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } +} diff --git a/crates/store/src/localdb/graph.rs b/crates/store/src/localdb/graph.rs new file mode 100644 index 000000000..b81fbed7f --- /dev/null +++ b/crates/store/src/localdb/graph.rs @@ -0,0 +1,874 @@ +//! Graph rows: pending inits, definitions, runtime columns, status transitions, +//! raw data and the monitored outpoints. + +use super::{StorageProcessor, get_current_timestamp_secs}; +use crate::utils::{QueryBuilder, QueryParam}; +use crate::{ + Graph, GraphBtcTxVoutMonitor, GraphRawData, GraphStatus, GraphStatusSource, + GraphStatusTransitionOutcome, PeginGraphProcessData, PendingGraphInit, SerializableTxid, +}; + +use sqlx::Row; +use sqlx::types::Uuid; +use std::str::FromStr; + +#[derive(Clone, Debug, Default)] +pub struct GraphQuery { + pub statuses: Vec, + pub operator_pubkey: Option, + pub kickoff_index: Option, + pub from_addr: Option, + pub graph_id: Option, + pub pegin_txid: Option, + pub raw_conditions: Vec, + pub order: Option, + pub offset: Option, + pub limit: Option, +} + +impl GraphQuery { + pub fn with_status(mut self, status: String) -> Self { + self.statuses.push(status); + self + } + + pub fn with_statuses(mut self, statuses: Vec) -> Self { + self.statuses = statuses; + self + } + pub fn with_raw_condition(mut self, raw_condition: String) -> Self { + self.raw_conditions.push(raw_condition); + self + } + + pub fn with_operator_pubkey(mut self, operator_pubkey: String) -> Self { + self.operator_pubkey = Some(operator_pubkey); + self + } + + pub fn with_order(mut self, order: String) -> Self { + self.order = Some(order); + self + } + + pub fn with_from_addr(mut self, from_addr: String) -> Self { + self.from_addr = Some(from_addr); + self + } + + pub fn with_kickoff_index(mut self, kickoff_index: i64) -> Self { + self.kickoff_index = Some(kickoff_index); + self + } + + pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { + self.offset = Some(offset); + self.limit = Some(limit); + self + } + + pub fn with_limit(mut self, limit: u32) -> Self { + self.limit = Some(limit); + self + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::new(base_sql); + // Add WHERE conditions + if !self.statuses.is_empty() { + query_builder.and_where_in("status", &self.statuses, false); + } + + if let Some(from_addr) = &self.from_addr { + query_builder.and_where("from_addr = ?", Some(QueryParam::Text(from_addr.clone()))); + } + + if let Some(operator) = &self.operator_pubkey { + query_builder + .and_where("operator_pubkey = ?", Some(QueryParam::Text(operator.clone()))); + } + + if let Some(kickoff_index) = self.kickoff_index { + query_builder.and_where("kickoff_index = ?", Some(QueryParam::Int(kickoff_index))); + } + + if let Some(pegin_txid) = &self.pegin_txid { + query_builder + .and_where("pegin_txid = ?", Some(QueryParam::BTCTxid(pegin_txid.clone()))); + } + + if let Some(graph_id) = &self.graph_id { + query_builder.and_where( + "hex(graph_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(graph_id.clone())), + ); + } + for raw_condition in &self.raw_conditions { + query_builder.add_raw_condition(raw_condition); + } + if let Some(order) = &self.order { + query_builder.apply_order(order) + } + // Add pagination + query_builder.apply_pagination(self.limit, self.offset); + query_builder + } +} + +/// Runtime graph fields that are not part of the signed graph definition. +/// +/// Status is intentionally absent. All graph status changes must go through +/// `StorageProcessor::transition_graph_status` so a stale event cannot replace +/// a later chain-observed state. +#[derive(Clone, Debug)] +pub struct GraphRuntimeUpdate { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub challenge_txid: Option, + pub bridge_out_start_at: Option, + pub init_withdraw_tx_hash: Option, + pub proceed_withdraw_height: Option, +} + +impl GraphRuntimeUpdate { + /// Create new update parameters + pub fn new(instance_id: Uuid, graph_id: Uuid) -> Self { + Self { + instance_id, + graph_id, + challenge_txid: None, + bridge_out_start_at: None, + init_withdraw_tx_hash: None, + proceed_withdraw_height: None, + } + } + + /// Set challenge transaction ID + pub fn with_challenge_txid(mut self, challenge_txid: SerializableTxid) -> Self { + self.challenge_txid = Some(challenge_txid); + self + } + + /// Set bridge out start time + pub fn with_bridge_out_start_at(mut self, bridge_out_start_at: i64) -> Self { + self.bridge_out_start_at = Some(bridge_out_start_at); + self + } + + /// Set init withdraw transaction ID + pub fn with_init_withdraw_tx_hash(mut self, init_withdraw_tx_hash: String) -> Self { + self.init_withdraw_tx_hash = Some(init_withdraw_tx_hash); + self + } + + /// Set proceed withdraw tx height at goat chain + pub fn with_proceed_withdraw_height(mut self, proceed_withdraw_height: i64) -> Self { + self.proceed_withdraw_height = Some(proceed_withdraw_height); + self + } + + /// Check if any fields need to be updated + pub fn has_updates(&self) -> bool { + self.challenge_txid.is_some() + || self.bridge_out_start_at.is_some() + || self.init_withdraw_tx_hash.is_some() + || self.proceed_withdraw_height.is_some() + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::update(base_sql); + // Add SET fields + if let Some(ref challenge_txid) = self.challenge_txid { + query_builder.set_field("challenge_txid", QueryParam::BTCTxid(challenge_txid.clone())); + } + if let Some(bridge_out_start_at) = self.bridge_out_start_at { + query_builder.set_field("bridge_out_start_at", QueryParam::Int(bridge_out_start_at)); + } + if let Some(proceed_withdraw_height) = self.proceed_withdraw_height { + query_builder + .set_field("proceed_withdraw_height", QueryParam::Int(proceed_withdraw_height)); + } + if let Some(ref init_withdraw_tx_hash) = self.init_withdraw_tx_hash { + if init_withdraw_tx_hash.is_empty() { + // Set NULL value + query_builder.set_field_null("init_withdraw_tx_hash"); + } else { + query_builder.set_field( + "init_withdraw_tx_hash", + QueryParam::Text(init_withdraw_tx_hash.clone()), + ); + } + } + // Add update time + let current_time = get_current_timestamp_secs(); + query_builder.set_field("updated_at", QueryParam::Int(current_time)); + + // Add WHERE clause + query_builder.and_where( + "hex(graph_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(self.graph_id))), + ); + query_builder.and_where( + "hex(instance_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(self.instance_id))), + ); + + query_builder + } +} + +impl StorageProcessor<'_> { + pub async fn upsert_pending_graph_init( + &mut self, + instance_id: &Uuid, + operator_pubkey: &str, + graph_id: &Uuid, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let result = sqlx::query( + "INSERT INTO pending_graph_init + (instance_id, operator_pubkey, graph_id, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(instance_id, operator_pubkey) DO UPDATE SET + graph_id = excluded.graph_id, + updated_at = excluded.updated_at", + ) + .bind(instance_id) + .bind(operator_pubkey) + .bind(graph_id) + .bind(current_time) + .bind(current_time) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + + pub async fn find_pending_graph_init_by_instance_and_operator_pubkey( + &mut self, + instance_id: &Uuid, + operator_pubkey: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_as::<_, PendingGraphInit>( + "SELECT instance_id, operator_pubkey, graph_id, created_at, updated_at + FROM pending_graph_init + WHERE instance_id = ? AND operator_pubkey = ?", + ) + .bind(instance_id) + .bind(operator_pubkey) + .fetch_optional(self.conn()) + .await?) + } + + /// Every open setup session of an instance (one per local operator). + pub async fn list_pending_graph_inits_by_instance( + &mut self, + instance_id: &Uuid, + ) -> anyhow::Result> { + Ok(sqlx::query_as::<_, PendingGraphInit>( + "SELECT instance_id, operator_pubkey, graph_id, created_at, updated_at + FROM pending_graph_init + WHERE instance_id = ?", + ) + .bind(instance_id) + .fetch_all(self.conn()) + .await?) + } + + /// Every open setup session of an operator. + pub async fn list_pending_graph_inits_by_operator( + &mut self, + operator_pubkey: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_as::<_, PendingGraphInit>( + "SELECT instance_id, operator_pubkey, graph_id, created_at, updated_at + FROM pending_graph_init + WHERE operator_pubkey = ? + ORDER BY created_at ASC, graph_id ASC", + ) + .bind(operator_pubkey) + .fetch_all(self.conn()) + .await?) + } + + pub async fn find_pending_graph_init_by_graph_id( + &mut self, + graph_id: &Uuid, + ) -> anyhow::Result> { + Ok(sqlx::query_as::<_, PendingGraphInit>( + "SELECT instance_id, operator_pubkey, graph_id, created_at, updated_at + FROM pending_graph_init + WHERE graph_id = ?", + ) + .bind(graph_id) + .fetch_optional(self.conn()) + .await?) + } + + pub async fn delete_pending_graph_init( + &mut self, + instance_id: &Uuid, + operator_pubkey: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "DELETE FROM pending_graph_init WHERE instance_id = ? AND operator_pubkey = ?", + ) + .bind(instance_id) + .bind(operator_pubkey) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } + + /// Insert a graph definition or verify a compatible replay. + /// + /// The canonical definition hash is an identity fence: a new graph may + /// never reuse an existing graph id and inherit its runtime projection. + /// Compatible replays may only fill missing non-identity metadata; they + /// never replace status, observed transaction ids, or withdraw metadata. + pub async fn upsert_graph_definition(&mut self, graph: &Graph) -> anyhow::Result { + if graph.definition_hash.is_empty() { + anyhow::bail!("graph {} is missing its definition hash", graph.graph_id); + } + if graph.status != GraphStatus::OperatorPresigned.to_string() + || !graph.sub_status.is_empty() + || graph.challenge_txid.is_some() + || graph.init_withdraw_tx_hash.is_some() + || graph.bridge_out_start_at != 0 + || graph.proceed_withdraw_height != 0 + { + anyhow::bail!( + "graph {} definition writes must use the OperatorPresigned baseline without runtime data", + graph.graph_id + ); + } + let verifier_assert_txids_json = serde_json::to_string(&graph.verifier_assert_txids)?; + let disprove_txids_json = serde_json::to_string(&graph.disprove_txids)?; + let watchtower_challenge_timeout_txids_json = + serde_json::to_string(&graph.watchtower_challenge_timeout_txids)?; + let operator_challenge_nack_txids_json = + serde_json::to_string(&graph.operator_challenge_nack_txids)?; + let res = sqlx::query( + "INSERT INTO graph (graph_id, instance_id, kickoff_index, from_addr, to_addr, amount, challenge_amount, + status, sub_status, operator_pubkey, definition_hash, cur_prekickoff_txid, next_prekickoff, force_skip_kickoff_txid, + quick_challenge_txid, challenge_incomplete_kickoff_txid, pegin_txid, kickoff_txid, take1_txid, + challenge_txid, take2_txid, watchtower_challenge_init_txid, operator_assert_txid, verifier_assert_txids, disprove_txids, + watchtower_challenge_timeout_txids, operator_challenge_nack_txids, operator_commit_timeout_txid, + init_withdraw_tx_hash, bridge_out_start_at, status_updated_at, proceed_withdraw_height, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(graph_id) DO UPDATE SET + from_addr = CASE + WHEN graph.from_addr = '' AND excluded.from_addr <> '' THEN excluded.from_addr + ELSE graph.from_addr + END, + to_addr = CASE + WHEN graph.to_addr = '' AND excluded.to_addr <> '' THEN excluded.to_addr + ELSE graph.to_addr + END + WHERE graph.definition_hash = excluded.definition_hash + AND graph.instance_id = excluded.instance_id", + ) + .bind(graph.graph_id) + .bind(graph.instance_id) + .bind(graph.kickoff_index) + .bind(&graph.from_addr) + .bind(&graph.to_addr) + .bind(graph.amount) + .bind(graph.challenge_amount) + .bind(GraphStatus::OperatorPresigned.to_string()) + .bind("") + .bind(&graph.operator_pubkey) + .bind(&graph.definition_hash) + .bind(graph.cur_prekickoff_txid.clone()) + .bind(graph.next_prekickoff.clone()) + .bind(graph.force_skip_kickoff_txid.clone()) + .bind(graph.quick_challenge_txid.clone()) + .bind(graph.challenge_incomplete_kickoff_txid.clone()) + .bind(graph.pegin_txid.clone()) + .bind(graph.kickoff_txid.clone()) + .bind(graph.take1_txid.clone()) + .bind(Option::::None) + .bind(graph.take2_txid.clone()) + .bind(graph.watchtower_challenge_init_txid.clone()) + .bind(graph.operator_assert_txid.clone()) + .bind(verifier_assert_txids_json) + .bind(disprove_txids_json) + .bind(watchtower_challenge_timeout_txids_json) + .bind(operator_challenge_nack_txids_json) + .bind(graph.operator_commit_timeout_txid.clone()) + .bind(Option::::None) + .bind(0_i64) + .bind(graph.status_updated_at) + .bind(0_i64) + .bind(graph.created_at) + .bind(graph.updated_at) + .execute(self.conn()) + .await?; + if res.rows_affected() == 0 { + let Some(existing) = self.find_graph(&graph.graph_id).await? else { + anyhow::bail!("graph {} disappeared while storing its definition", graph.graph_id); + }; + if existing.definition_hash != graph.definition_hash { + anyhow::bail!( + "conflicting graph definition for graph_id {}: existing={}, incoming={}", + graph.graph_id, + existing.definition_hash, + graph.definition_hash + ); + } + if existing.instance_id != graph.instance_id { + anyhow::bail!( + "graph definition instance mismatch for graph_id {}: existing={}, incoming={}", + graph.graph_id, + existing.instance_id, + graph.instance_id + ); + } + } + Ok(res.rows_affected()) + } + + pub async fn update_graph_runtime( + &mut self, + params: &GraphRuntimeUpdate, + ) -> anyhow::Result { + if !params.has_updates() { + return Ok(false); + } + let query_builder = params.get_query_builder("graph"); + let update_sql = query_builder.get_sql(); + let query = sqlx::query(&update_sql); + let query = query_builder.query(query); + let result = query.execute(self.conn()).await?; + Ok(result.rows_affected() > 0) + } + + /// Atomically advance a graph status according to its evidence source. + /// + /// The conditional UPDATE is the authority check. The follow-up read only + /// distinguishes an idempotent replay from a stale event; it never decides + /// whether a write is permitted. + pub async fn transition_graph_status( + &mut self, + instance_id: Uuid, + graph_id: Uuid, + target: GraphStatus, + source: GraphStatusSource, + sub_status: Option, + ) -> anyhow::Result { + if !target.is_protocol_status() { + anyhow::bail!("frontend-only graph status {target} cannot be persisted"); + } + + let allowed_from = target.allowed_transition_from(source); + if allowed_from.is_empty() { + // Some verified scans merely observe a baseline state (for + // example, an operator-pre-signed graph). There is no authorized + // predecessor edge to write in that case; return the current + // projection without mutating it. + return self.graph_status_transition_outcome(instance_id, graph_id, target).await; + } + + let allowed_from: Vec = allowed_from.iter().map(ToString::to_string).collect(); + let current_time = get_current_timestamp_secs(); + let mut query_builder = QueryBuilder::update("graph"); + query_builder.set_field("status", QueryParam::Text(target.to_string())); + if let Some(sub_status) = sub_status.as_ref() { + query_builder.set_field("sub_status", QueryParam::Text(sub_status.clone())); + } + query_builder.set_field("status_updated_at", QueryParam::Int(current_time)); + query_builder.set_field("updated_at", QueryParam::Int(current_time)); + query_builder.and_where( + "hex(graph_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(graph_id))), + ); + query_builder.and_where( + "hex(instance_id) = ? COLLATE NOCASE", + Some(QueryParam::Text(hex::encode(instance_id))), + ); + query_builder.and_where_in("status", &allowed_from, false); + + let update_sql = query_builder.get_sql(); + let query = query_builder.query(sqlx::query(&update_sql)); + if query.execute(self.conn()).await?.rows_affected() > 0 { + return Ok(GraphStatusTransitionOutcome::Applied); + } + + // A zero-row conditional update has two distinct meanings: this can + // be an idempotent replay, or another writer may have moved the row to + // a state which rejects this transition. The follow-up read reports + // that distinction; it never authorizes a write. + let outcome = self.graph_status_transition_outcome(instance_id, graph_id, target).await?; + if !matches!(outcome, GraphStatusTransitionOutcome::AlreadyCurrent) { + return Ok(outcome); + } + + let Some(sub_status) = sub_status else { + return Ok(GraphStatusTransitionOutcome::AlreadyCurrent); + }; + + let current_time = get_current_timestamp_secs(); + let result = sqlx::query( + "UPDATE graph SET sub_status = ?, updated_at = ? \ + WHERE graph_id = ? AND instance_id = ? AND status = ?", + ) + .bind(sub_status) + .bind(current_time) + .bind(graph_id) + .bind(instance_id) + .bind(target.to_string()) + .execute(self.conn()) + .await?; + if result.rows_affected() > 0 { + return Ok(GraphStatusTransitionOutcome::AlreadyCurrent); + } + + // The row may have changed between the outcome read and the optional + // sub-status update. Re-read so a concurrent transition is never + // misreported as an idempotent replay or a missing graph. + self.graph_status_transition_outcome(instance_id, graph_id, target).await + } + + async fn graph_status_transition_outcome( + &mut self, + instance_id: Uuid, + graph_id: Uuid, + target: GraphStatus, + ) -> anyhow::Result { + let Some(current_graph) = self.find_graph(&graph_id).await? else { + return Ok(GraphStatusTransitionOutcome::NotFound); + }; + if current_graph.instance_id != instance_id { + return Ok(GraphStatusTransitionOutcome::NotFound); + } + let current = GraphStatus::from_str(¤t_graph.status).map_err(|_| { + anyhow::anyhow!( + "graph {graph_id} has invalid persisted status {}", + current_graph.status + ) + })?; + Ok(if current == target { + GraphStatusTransitionOutcome::AlreadyCurrent + } else { + GraphStatusTransitionOutcome::Rejected { current } + }) + } + + /// The one graph an operator may hold in a slot. + pub async fn find_graph_by_operator_slot( + &mut self, + operator_pubkey: &str, + graph_nonce: i64, + ) -> anyhow::Result> { + let row = sqlx::query_as::<_, Graph>( + "SELECT * FROM graph WHERE operator_pubkey = ? AND kickoff_index = ?", + ) + .bind(operator_pubkey) + .bind(graph_nonce) + .fetch_optional(self.conn()) + .await?; + Ok(row) + } + + pub async fn find_graph(&mut self, graph_id: &Uuid) -> anyhow::Result> { + let row = sqlx::query_as::<_, Graph>( + "SELECT * + FROM graph + WHERE graph_id = ?", + ) + .bind(graph_id) + .fetch_optional(self.conn()) + .await?; + Ok(row) + } + + pub async fn find_graphs(&mut self, params: GraphQuery) -> anyhow::Result<(Vec, i64)> { + // Build base query + let mut count_params = params.clone(); + (count_params.offset, count_params.limit) = (None, None); + let query_builder = params.get_query_builder( + "SELECT graph_id, + instance_id, + kickoff_index, + from_addr, + to_addr, + amount, + challenge_amount, + status, + sub_status, + operator_pubkey, + definition_hash, + cur_prekickoff_txid, + next_prekickoff, + force_skip_kickoff_txid, + quick_challenge_txid, + challenge_incomplete_kickoff_txid, + pegin_txid, + kickoff_txid, + take1_txid, + challenge_txid, + take2_txid, + watchtower_challenge_init_txid, + operator_assert_txid, + verifier_assert_txids, + disprove_txids, + watchtower_challenge_timeout_txids, + operator_challenge_nack_txids, + operator_commit_timeout_txid, + init_withdraw_tx_hash, + bridge_out_start_at, + status_updated_at, + proceed_withdraw_height, + CASE + WHEN bridge_out_start_at > 0 + THEN bridge_out_start_at + ELSE created_at + END AS created_at, + updated_at + FROM graph", + ); + + let count_query_builder = + count_params.get_query_builder("SELECT count(graph_id) as total_graphs FROM graph"); + let sql = query_builder.get_sql(); + let mut graphs_query = sqlx::query_as::<_, Graph>(&sql); + graphs_query = query_builder.query_as(graphs_query); + let count_sql = count_query_builder.get_sql(); + let mut count_query = sqlx::query(&count_sql); + count_query = count_query_builder.query(count_query); + + Ok(( + graphs_query.fetch_all(self.conn()).await?, + count_query.fetch_one(self.conn()).await?.get::("total_graphs"), + )) + } + + pub async fn get_graphs_by_instance_id( + &mut self, + instance_id: &Uuid, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, Graph>( + "SELECT * + FROM graph + WHERE instance_id = ?", + ) + .bind(instance_id) + .fetch_all(self.conn()) + .await?; + Ok(res) + } + + pub async fn get_graph_id_by_instance_id_and_operator_pubkey( + &mut self, + instance_id: &Uuid, + operator_pubkey: &str, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, Graph>( + "SELECT * + FROM graph + WHERE instance_id = ? AND operator_pubkey = ?", + ) + .bind(instance_id) + .bind(operator_pubkey) + .fetch_optional(self.conn()) + .await?; + Ok(res.map(|graph| graph.graph_id)) + } + + pub async fn get_operator_graphs(&mut self, params: GraphQuery) -> anyhow::Result> { + let graph_query_builder = params.get_query_builder("SELECT * FROM graph"); + let operator_graph_sql = graph_query_builder.get_sql(); + let mut operator_graphs_query = sqlx::query_as::<_, Graph>(&operator_graph_sql); + operator_graphs_query = graph_query_builder.query_as(operator_graphs_query); + Ok(operator_graphs_query.fetch_all(self.conn()).await?) + } + + pub async fn upsert_pegin_graph_process_data( + &mut self, + pegin_graph_process_data: &PeginGraphProcessData, + ) -> anyhow::Result { + let res = sqlx::query!( + "INSERT INTO pegin_graph_process_data + (graph_id, instance_id, process_data, is_endorsed, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(graph_id) DO UPDATE + SET process_data = excluded.process_data, + instance_id = excluded.instance_id, + is_endorsed = excluded.is_endorsed, + updated_at = excluded.updated_at", + pegin_graph_process_data.graph_id, + pegin_graph_process_data.instance_id, + pegin_graph_process_data.process_data, + pegin_graph_process_data.is_endorsed, + pegin_graph_process_data.created_at, + pegin_graph_process_data.updated_at + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected() > 0) + } + + pub async fn find_pegin_graph_process_data( + &mut self, + graph_id: &Uuid, + ) -> anyhow::Result> { + let row = sqlx::query_as!( + PeginGraphProcessData, + "SELECT + graph_id AS \"graph_id:Uuid\", + instance_id AS \"instance_id:Uuid\", + process_data, + is_endorsed, + created_at, + updated_at + FROM pegin_graph_process_data + WHERE graph_id = ?", + graph_id + ) + .fetch_optional(self.conn()) + .await?; + Ok(row) + } + + pub async fn get_pegin_graph_endorsed_len_by_instance_id( + &mut self, + instance_id: &Uuid, + is_endorsed: bool, + ) -> anyhow::Result { + let record = sqlx::query!( + r#"SELECT count(*) AS length + FROM pegin_graph_process_data + WHERE instance_id = ? AND is_endorsed = ?"#, + instance_id, + is_endorsed + ) + .fetch_one(self.conn()) + .await?; + Ok(record.length) + } + + pub async fn upsert_graph_raw_data( + &mut self, + instance_id: Uuid, + graph_raw_data: GraphRawData, + definition_hash: &str, + ) -> anyhow::Result { + if definition_hash.is_empty() { + anyhow::bail!( + "graph {} raw definition is missing its canonical definition hash", + graph_raw_data.graph_id + ); + } + let Some(graph) = self.find_graph(&graph_raw_data.graph_id).await? else { + anyhow::bail!( + "cannot store raw definition for missing graph {}", + graph_raw_data.graph_id + ); + }; + if graph.instance_id != instance_id { + anyhow::bail!( + "raw definition instance does not match graph {}: stored={}, incoming={instance_id}", + graph_raw_data.graph_id, + graph.instance_id + ); + } + if graph.definition_hash != definition_hash { + anyhow::bail!( + "raw definition hash does not match graph {}: stored={}, incoming={definition_hash}", + graph_raw_data.graph_id, + graph.definition_hash + ); + } + + let result = sqlx::query( + "INSERT INTO graph_raw_data (graph_id, raw_data, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(graph_id) DO UPDATE SET + raw_data = excluded.raw_data, + updated_at = excluded.updated_at", + ) + .bind(graph_raw_data.graph_id) + .bind(graph_raw_data.raw_data) + .bind(graph_raw_data.created_at) + .bind(graph_raw_data.updated_at) + .execute(self.conn()) + .await?; + + Ok(result.rows_affected()) + } + + pub async fn find_graph_raw_data( + &mut self, + graph_id: &Uuid, + ) -> anyhow::Result> { + let row = sqlx::query_as!( + GraphRawData, + "SELECT + graph_id AS \"graph_id:Uuid\", + raw_data, + created_at, + updated_at + FROM graph_raw_data WHERE graph_id = ?", + graph_id + ) + .fetch_optional(self.conn()) + .await?; + + Ok(row) + } + + pub async fn upsert_graph_btc_tx_vout_monitor( + &mut self, + monitor: &GraphBtcTxVoutMonitor, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + + let res = sqlx::query!( + r#" + INSERT OR REPLACE INTO graph_btc_tx_vout_monitor + (graph_id, tx_name, txid, height, vout_len, monitor_data, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + "#, + monitor.graph_id, + monitor.tx_name, + monitor.txid, + monitor.height, + monitor.vout_len, + monitor.monitor_data, + monitor.created_at, + current_time + ) + .execute(self.conn()) + .await?; + + Ok(res.rows_affected()) + } + + pub async fn find_graph_btc_tx_vout_monitor( + &mut self, + graph_id: &Uuid, + txid: &SerializableTxid, + ) -> anyhow::Result> { + let row = sqlx::query_as!( + GraphBtcTxVoutMonitor, + r#" + SELECT + graph_id AS "graph_id: Uuid", + tx_name, + txid AS "txid: SerializableTxid", + height, + vout_len, + monitor_data, + created_at, + updated_at + FROM graph_btc_tx_vout_monitor + WHERE graph_id = ? AND txid = ? + "#, + graph_id, + txid + ) + .fetch_optional(self.conn()) + .await?; + + Ok(row) + } +} diff --git a/crates/store/src/localdb/instance.rs b/crates/store/src/localdb/instance.rs new file mode 100644 index 000000000..088017e8d --- /dev/null +++ b/crates/store/src/localdb/instance.rs @@ -0,0 +1,574 @@ +//! Instance rows: peg-in requests, their answers, parameters and process data. + +use super::{StorageProcessor, get_current_timestamp_secs}; +use crate::utils::{QueryBuilder, QueryParam}; +use crate::{Instance, PeginInstanceProcessData, SerializableTxid}; + +use indexmap::IndexMap; +use sqlx::Row; +use sqlx::types::Uuid; + +#[derive(Clone, Debug, Default)] +pub struct InstanceQuery { + pub from_addr: Option, + pub statuses: Vec, + pub earliest_updated: Option, + pub pegin_request_height_threshold: Option, + pub order: Option, + pub raw_conditions: Vec, + pub offset: Option, + pub limit: Option, +} + +impl InstanceQuery { + pub fn with_from_addr(mut self, from_addr: String) -> Self { + self.from_addr = Some(from_addr); + self + } + + pub fn with_status(mut self, status: String) -> Self { + self.statuses.push(status); + self + } + + pub fn with_statuses(mut self, statuses: Vec) -> Self { + self.statuses = statuses; + self + } + + pub fn with_order(mut self, order: String) -> Self { + self.order = Some(order); + self + } + + pub fn with_pegin_request_height_threshold(mut self, threshold: i64) -> Self { + self.pegin_request_height_threshold = Some(threshold); + self + } + + pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { + self.offset = Some(offset); + self.limit = Some(limit); + self + } + + pub fn with_limit(mut self, limit: u32) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_raw_condition(mut self, raw_condition: String) -> Self { + self.raw_conditions.push(raw_condition); + self + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::new(base_sql); + if let Some(from_addr) = &self.from_addr { + query_builder.and_where("from_addr = ?", Some(QueryParam::Text(from_addr.clone()))); + } + + if !self.statuses.is_empty() { + query_builder.and_where_in("status", &self.statuses, false); + } + if let Some(earliest_updated) = self.earliest_updated { + query_builder.and_where("updated_at >= ?", Some(QueryParam::Int(earliest_updated))); + } + if let Some(pegin_request_height_threshold) = self.pegin_request_height_threshold { + query_builder.and_where( + "goat_tx_height < ?", + Some(QueryParam::Int(pegin_request_height_threshold)), + ); + } + for raw_condition in &self.raw_conditions { + query_builder.add_raw_condition(raw_condition); + } + + if let Some(order) = &self.order { + query_builder.apply_order(order); + } + query_builder.apply_pagination(self.limit, self.offset); + query_builder + } +} + +/// Instance field update parameters +/// +/// Provides a more elegant way to specify which instance fields to update +#[derive(Debug, Clone)] +pub struct InstanceUpdate { + pub instance_id: Uuid, + pub from_addr: Option, + pub to_addr: Option, + pub btc_txid: Option, + pub status: Option, + pub pegin_confirm_txid: Option, + pub pegin_cancel_txid: Option, + pub post_pegin_txhash: Option, + pub btc_height: Option, + pub committees_answers: Option>>, + pub key_deleted: Option, + pub only_if_status_in: Option>, +} + +impl InstanceUpdate { + /// Create new update parameters + pub fn new_with_instance_id(instance_id: Uuid) -> Self { + Self { + instance_id, + from_addr: None, + to_addr: None, + btc_txid: None, + status: None, + pegin_confirm_txid: None, + pegin_cancel_txid: None, + post_pegin_txhash: None, + btc_height: None, + committees_answers: None, + key_deleted: None, + only_if_status_in: None, + } + } + + /// Set from_addr + pub fn with_from_addr(mut self, from_addr: String) -> Self { + self.from_addr = Some(from_addr); + self + } + + /// Set btc txid + pub fn with_btc_txid(mut self, btc_txid: SerializableTxid) -> Self { + self.btc_txid = Some(btc_txid); + self + } + /// Set status + pub fn with_status(mut self, status: String) -> Self { + self.status = Some(status); + self + } + + /// Set pegin confirmation transaction ID. + pub fn with_pegin_confirm_txid(mut self, txid: SerializableTxid) -> Self { + self.pegin_confirm_txid = Some(txid); + self + } + + /// Set pegin cancellation transaction ID. + pub fn with_pegin_cancel_txid(mut self, txid: SerializableTxid) -> Self { + self.pegin_cancel_txid = Some(txid); + self + } + + /// Set post pegin information + pub fn with_post_pegin(mut self, txid: String) -> Self { + self.post_pegin_txhash = Some(txid); + self + } + + /// Set committees answers + pub fn with_committees_answers( + mut self, + committees_answers: IndexMap>, + ) -> Self { + self.committees_answers = Some(committees_answers); + self + } + + /// Set btc height + pub fn with_btc_height(mut self, btc_height: i64) -> Self { + self.btc_height = Some(btc_height); + self + } + + /// Record whether the committee instance key envelope has been deleted. + pub fn with_key_deleted(mut self, key_deleted: bool) -> Self { + self.key_deleted = Some(key_deleted); + self + } + + /// Apply this update only while the instance is still in one of the + /// expected states. The condition is folded into the UPDATE statement. + pub fn with_only_if_status_in(mut self, statuses: Vec) -> Self { + self.only_if_status_in = Some(statuses); + self + } + + /// Check if any fields need to be updated + pub fn has_updates(&self) -> bool { + self.from_addr.is_some() + || self.to_addr.is_some() + || self.btc_txid.is_some() + || self.status.is_some() + || self.pegin_confirm_txid.is_some() + || self.pegin_cancel_txid.is_some() + || self.post_pegin_txhash.is_some() + || self.btc_height.is_some() + || self.committees_answers.is_some() + || self.key_deleted.is_some() + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::update(base_sql); + // Set field + if let Some(ref status) = self.status { + query_builder.set_field("status", QueryParam::Text(status.clone())); + query_builder + .set_field("status_updated_at", QueryParam::Int(get_current_timestamp_secs())); + } + + if let Some(ref txid) = self.pegin_confirm_txid { + query_builder.set_field("pegin_confirm_txid", QueryParam::BTCTxid(txid.clone())); + } + + if let Some(ref txid) = self.pegin_cancel_txid { + query_builder.set_field("pegin_cancel_txid", QueryParam::BTCTxid(txid.clone())); + } + + if let Some(ref txid) = self.post_pegin_txhash { + query_builder.set_field("post_pegin_txhash", QueryParam::Text(txid.clone())); + } + + if let Some(pegin_prepare_height) = self.btc_height { + query_builder.set_field("btc_height", QueryParam::Int(pegin_prepare_height)); + } + + if let Some(ref to_addr) = self.to_addr { + query_builder.set_field("to_addr", QueryParam::Text(to_addr.clone())); + } + + if let Some(ref from_addr) = self.from_addr { + query_builder.set_field("from_addr", QueryParam::Text(from_addr.clone())); + } + + if let Some(ref btc_txid) = self.btc_txid { + query_builder.set_field("btc_txid", QueryParam::BTCTxid(btc_txid.clone())); + } + + if let Some(ref committees_answers) = self.committees_answers { + let committees_answers = serde_json::to_string(committees_answers) + .expect("IndexMap> serialization is infallible"); + query_builder.set_field("committees_answers", QueryParam::Text(committees_answers)); + } + + if let Some(key_deleted) = self.key_deleted { + query_builder.set_field("key_deleted", QueryParam::Int(i64::from(key_deleted))); + } + + // Add update time + let current_time = get_current_timestamp_secs(); + query_builder.set_field("updated_at", QueryParam::Int(current_time)); + + // Add WHERE clause + query_builder.and_where( + "hex(instance_id) = ? COLLATE NOCASE ", + Some(QueryParam::Text(hex::encode(self.instance_id))), + ); + + if let Some(ref statuses) = self.only_if_status_in { + if statuses.is_empty() { + // An empty allow-list must reject the update rather than + // silently dropping the compare-and-swap guard. + query_builder.and_where("1 = 0", None); + } else { + query_builder.and_where_in("status", statuses, false); + } + } + + query_builder + } +} + +impl StorageProcessor<'_> { + /// Insert an instance only when its ID is not already present. + /// + /// Parameters: + /// - instance: The complete instance data to insert + /// + /// Returns: + /// - Ok(true) if the instance was inserted + /// - Ok(false) if an instance with the same ID already exists + /// - Err if the operation failed + pub async fn insert_instance_if_absent(&mut self, instance: &Instance) -> anyhow::Result { + let committees_answers_json = serde_json::to_string(&instance.committees_answers)?; + let res = sqlx::query( + "INSERT INTO instance (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, status, goat_tx_hash, goat_tx_height, + user_xonly_pubkey, user_change_addr, user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers, + pegin_data_tx_hash, btc_height, parameters, status_updated_at, post_pegin_txhash, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(instance_id) DO NOTHING", + ) + .bind(instance.instance_id) + .bind(&instance.network) + .bind(&instance.from_addr) + .bind(&instance.to_addr) + .bind(instance.amount) + .bind(instance.fees) + .bind(&instance.input_utxos) + .bind(&instance.status) + .bind(&instance.goat_tx_hash) + .bind(instance.goat_tx_height) + .bind(instance.user_xonly_pubkey) + .bind(&instance.user_change_addr) + .bind(&instance.user_refund_addr) + .bind(&instance.btc_txid) + .bind(&instance.pegin_confirm_txid) + .bind(&instance.pegin_cancel_txid) + .bind(committees_answers_json) + .bind(&instance.pegin_data_tx_hash) + .bind(instance.btc_height) + .bind(&instance.parameters) + .bind(instance.status_updated_at) + .bind(&instance.post_pegin_txhash) + .bind(instance.created_at) + .bind(instance.updated_at) + .execute(self.conn()) + .await?; + Ok(res.rows_affected() > 0) + } + + /// Create a bridge-in instance from a pegin request, or refresh one that + /// has not moved past the pegin-request stage yet. + /// + /// The request monitor writes the row from the indexed Goat event and may + /// see the same request again, so a full-row replacement would let a late + /// or repeated request roll a live instance back to its initial row and + /// clear everything the later stages wrote. The write is therefore a + /// compare-and-swap over the current status, and it only touches + /// the columns a pegin request owns: + /// committee answers, instance parameters and the BTC-side fields are never + /// overwritten. The caller is responsible for passing canonical request + /// metadata - `goat_tx_hash`/`goat_tx_height` are refreshed from it, so that + /// a re-indexed event can still correct them. + /// + /// Returns: + /// - Ok(true) if the row was inserted or updated + /// - Ok(false) if an existing row was left untouched because its status + /// is outside `allowed_statuses` + pub async fn upsert_pegin_request_instance( + &mut self, + instance: &Instance, + ) -> anyhow::Result { + let committees_answers_json = serde_json::to_string(&instance.committees_answers)?; + // The row is refreshed only while it still waits for the committee's + // answers; a row that moved on keeps what it accrued. + let sql = "INSERT INTO instance \ + (instance_id, network, from_addr, to_addr, amount, fees, input_utxos, \ + status, goat_tx_hash, goat_tx_height, user_xonly_pubkey, user_change_addr, \ + user_refund_addr, btc_txid, pegin_confirm_txid, pegin_cancel_txid, committees_answers, \ + pegin_data_tx_hash, btc_height, parameters, status_updated_at, \ + post_pegin_txhash, created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(instance_id) DO UPDATE SET \ + network = excluded.network, \ + from_addr = CASE WHEN excluded.from_addr = '' \ + THEN instance.from_addr ELSE excluded.from_addr END, \ + to_addr = excluded.to_addr, \ + amount = excluded.amount, \ + fees = excluded.fees, \ + input_utxos = excluded.input_utxos, \ + status = excluded.status, \ + goat_tx_hash = excluded.goat_tx_hash, \ + goat_tx_height = excluded.goat_tx_height, \ + user_xonly_pubkey = excluded.user_xonly_pubkey, \ + user_change_addr = excluded.user_change_addr, \ + user_refund_addr = excluded.user_refund_addr, \ + status_updated_at = excluded.status_updated_at, \ + updated_at = excluded.updated_at \ + WHERE instance.status = ?"; + let query = sqlx::query(sql) + .bind(instance.instance_id) + .bind(&instance.network) + .bind(&instance.from_addr) + .bind(&instance.to_addr) + .bind(instance.amount) + .bind(instance.fees) + .bind(&instance.input_utxos) + .bind(&instance.status) + .bind(&instance.goat_tx_hash) + .bind(instance.goat_tx_height) + .bind(instance.user_xonly_pubkey) + .bind(&instance.user_change_addr) + .bind(&instance.user_refund_addr) + .bind(&instance.btc_txid) + .bind(&instance.pegin_confirm_txid) + .bind(&instance.pegin_cancel_txid) + .bind(committees_answers_json) + .bind(&instance.pegin_data_tx_hash) + .bind(instance.btc_height) + .bind(&instance.parameters) + .bind(instance.status_updated_at) + .bind(&instance.post_pegin_txhash) + .bind(instance.created_at) + .bind(instance.updated_at) + .bind(crate::InstanceBridgeInStatus::UserInited.to_string()); + let res = query.execute(self.conn()).await?; + Ok(res.rows_affected() > 0) + } + + /// Find a single instance by its ID + /// + /// Retrieves an instance from the database using its unique instance_id. + /// + /// Parameters: + /// - instance_id: The UUID of the instance to find + /// + /// Returns: + /// - Ok(Some(instance)) if the instance was found + /// - Ok(None) if no instance with the given ID exists + /// - Err if the query failed + pub async fn find_instance(&mut self, instance_id: &Uuid) -> anyhow::Result> { + let row = sqlx::query_as::<_, Instance>("SELECT * FROM instance WHERE instance_id = ?") + .bind(instance_id) + .fetch_optional(self.conn()) + .await?; + Ok(row) + } + + /// Find multiple instances with filtering and pagination + /// + /// Retrieves instances from the database with optional filtering criteria + /// and pagination support. Uses QueryBuilder for dynamic query construction. + /// + /// Parameters: + /// - params: InstanceQuery containing all filter criteria and pagination options + /// + /// Returns: + /// - Ok((instances, total_count)) where instances is a vector of matching instances + /// and total_count is the total number of instances matching the criteria + /// - Err if the query failed + pub async fn find_instances( + &mut self, + params: InstanceQuery, + ) -> anyhow::Result<(Vec, i64)> { + let mut count_params = params.clone(); + (count_params.order, count_params.offset) = (None, None); + let instances_query_builder = params.get_query_builder("SELECT * FROM instance"); + let count_query_builder = + count_params.get_query_builder("SELECT count(*) as total_instances FROM instance"); + // Execute data query + let sql = instances_query_builder.get_sql(); + let mut data_query = sqlx::query_as::<_, Instance>(&sql); + data_query = instances_query_builder.query_as(data_query); + // Execute count query + let count_sql = count_query_builder.get_sql(); + let mut count_query = sqlx::query(&count_sql); + count_query = count_query_builder.query(count_query); + Ok(( + data_query.fetch_all(self.conn()).await?, + count_query.fetch_one(self.conn()).await?.get::("total_instances"), + )) + } + + /// Update instance fields using builder pattern + /// + /// This is the most elegant update method, using the InstanceUpdate builder pattern + /// Provides type safety and clear API + pub async fn update_instance(&mut self, params: &InstanceUpdate) -> anyhow::Result { + if !params.has_updates() { + return Ok(false); + } + let query_builder = params.get_query_builder("instance"); + // Get SQL and parameters + let update_sql = query_builder.get_sql(); + // Execute query + let mut query = sqlx::query(&update_sql); + query = query_builder.query(query); + let result = query.execute(self.conn()).await?; + Ok(result.rows_affected() > 0) + } + + /// Add or update a single committee answer for an instance + /// + /// This method merges one committee answer atomically so concurrent event + /// handlers cannot overwrite each other's answers with stale full maps. + pub async fn update_instance_committee_answer( + &mut self, + instance_id: &Uuid, + committee_addr: &str, + pubkey: Vec, + ) -> anyhow::Result { + let committee_patch = serde_json::json!({ (committee_addr): pubkey }).to_string(); + let current_time = get_current_timestamp_secs(); + let result = sqlx::query( + "UPDATE instance \ + SET committees_answers = json_patch(COALESCE(committees_answers, '{}'), json(?)), \ + updated_at = ? \ + WHERE instance_id = ?", + ) + .bind(committee_patch) + .bind(current_time) + .bind(instance_id) + .execute(self.conn()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn update_instance_parameters( + &mut self, + instance_id: &Uuid, + parameters: &str, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE instance SET parameters = ?, updated_at = ? WHERE instance_id = ?", + parameters, + current_time, + instance_id + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected() > 0) + } + + pub async fn get_instance_parameters_by_id( + &mut self, + instance_id: &Uuid, + ) -> anyhow::Result> { + let res = + sqlx::query!("SELECT parameters FROM instance WHERE instance_id = ?", instance_id) + .fetch_optional(self.conn()) + .await?; + Ok(res.and_then(|record| record.parameters)) + } + + pub async fn upsert_pegin_instance_process_data( + &mut self, + pegin_instance_process_data: &PeginInstanceProcessData, + ) -> anyhow::Result { + let res = sqlx::query!( + "INSERT INTO pegin_instance_process_data + (instance_id, process_data, created_at, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(instance_id) DO UPDATE + SET process_data = excluded.process_data, + updated_at = excluded.updated_at", + pegin_instance_process_data.instance_id, + pegin_instance_process_data.process_data, + pegin_instance_process_data.created_at, + pegin_instance_process_data.updated_at + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected() > 0) + } + + pub async fn find_pegin_instance_process_data( + &mut self, + instance_id: &Uuid, + ) -> anyhow::Result> { + let row = sqlx::query_as!( + PeginInstanceProcessData, + "SELECT + instance_id AS \"instance_id:Uuid\", + process_data, + created_at, + updated_at + FROM pegin_instance_process_data + WHERE instance_id = ?", + instance_id + ) + .fetch_optional(self.conn()) + .await?; + Ok(row) + } +} diff --git a/crates/store/src/localdb/mod.rs b/crates/store/src/localdb/mod.rs new file mode 100644 index 000000000..5a217207d --- /dev/null +++ b/crates/store/src/localdb/mod.rs @@ -0,0 +1,156 @@ +mod goat; +mod graph; +mod instance; +mod node; +mod proof; +mod stats; +mod swap; +#[cfg(test)] +mod tests; + +use sqlx::migrate::Migrator; +use sqlx::pool::PoolConnection; +use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode}; +use sqlx::{Sqlite, SqliteConnection, SqlitePool, Transaction, migrate::MigrateDatabase}; +use std::str::FromStr; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub use graph::{GraphQuery, GraphRuntimeUpdate}; +pub use instance::{InstanceQuery, InstanceUpdate}; +pub use node::NodeQuery; +pub use swap::{SwapEscrowQuery, SwapEscrowUpdate}; + +pub(super) fn get_current_timestamp_secs() -> i64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 +} + +#[derive(Clone, Debug)] +pub struct LocalDB { + pub path: String, + pub is_mem: bool, + pub conn: SqlitePool, +} + +#[derive(Debug)] +pub enum ConnectionHolder<'a> { + Pooled(PoolConnection), + Direct(SqliteConnection), + Transaction(Transaction<'a, Sqlite>), +} + +#[derive(Debug)] +pub struct StorageProcessor<'a> { + pub conn: ConnectionHolder<'a>, + pub in_transaction: bool, +} + +static MIGRATOR: Migrator = sqlx::migrate!("./migrations"); +/// sqlx's own default, made explicit: how long a statement waits for SQLite's +/// write lock before failing with "database is locked". +pub const DEFAULT_BUSY_TIMEOUT: Duration = Duration::from_secs(5); + +impl LocalDB { + pub async fn new(path: &str, is_mem: bool) -> LocalDB { + Self::new_with_busy_timeout(path, is_mem, DEFAULT_BUSY_TIMEOUT).await + } + + /// `busy_timeout` must cover the longest write transaction the process + /// holds (the engine's apply), or every other writer fails while it runs. + pub async fn new_with_busy_timeout( + path: &str, + is_mem: bool, + busy_timeout: Duration, + ) -> LocalDB { + if !Sqlite::database_exists(path).await.unwrap_or(false) { + tracing::info!("Creating database {}", path); + match Sqlite::create_database(path).await { + Ok(_) => tracing::info!( + event = "database_lifecycle", + outcome = "created", + "local database created" + ), + Err(error) => panic!("error: {error}"), + } + } else { + tracing::info!("Database already exists"); + } + + let mut options = SqliteConnectOptions::from_str(path) + .unwrap() + .create_if_missing(true) + .busy_timeout(busy_timeout); + if !is_mem { + // File-backed nodes run event watchers, P2P handlers, and maintenance tasks + // concurrently. WAL allows their readers to proceed while a short write commits. + options = options.journal_mode(SqliteJournalMode::Wal); + } + let conn = SqlitePool::connect_with(options).await.unwrap(); + Self { path: path.to_string(), is_mem, conn } + } + + pub async fn migrate(&self) { + match MIGRATOR.run(&self.conn).await { + Ok(_) => tracing::info!("Migration success"), + Err(error) => { + panic!("error: {error:?}"); + } + } + } + + pub async fn acquire<'a>(&self) -> anyhow::Result> { + Ok(StorageProcessor { + conn: ConnectionHolder::Pooled(self.conn.acquire().await?), + in_transaction: false, + }) + } + pub async fn start_transaction<'a>(&self) -> anyhow::Result> { + Ok(StorageProcessor { + conn: ConnectionHolder::Transaction(self.conn.begin().await?), + in_transaction: true, + }) + } + + /// Start a short write transaction before reading state that will be + /// immediately reconciled. This prevents a stale snapshot from dropping + /// a concurrent update between the read and the conditional write. + pub async fn start_immediate_transaction<'a>(&self) -> anyhow::Result> { + Ok(StorageProcessor { + conn: ConnectionHolder::Transaction(self.conn.begin_with("BEGIN IMMEDIATE").await?), + in_transaction: true, + }) + } +} + +impl StorageProcessor<'_> { + pub fn conn(&mut self) -> &mut SqliteConnection { + match &mut self.conn { + ConnectionHolder::Pooled(conn) => conn, + ConnectionHolder::Direct(conn) => conn, + ConnectionHolder::Transaction(conn) => conn, + } + } + + pub async fn commit(self) -> anyhow::Result<()> { + if let ConnectionHolder::Transaction(transaction) = self.conn { + transaction.commit().await?; + Ok(()) + } else { + panic!( + "StorageProcessor::commit can only be invoked after calling StorageProcessor::begin_transaction" + ); + } + } +} + +pub async fn create_local_db(db_path: &str) -> LocalDB { + create_local_db_with_busy_timeout(db_path, DEFAULT_BUSY_TIMEOUT).await +} + +pub async fn create_local_db_with_busy_timeout(db_path: &str, busy_timeout: Duration) -> LocalDB { + // Only in-memory databases skip WAL; a file-backed node needs its readers + // to proceed while the engine holds the write lock. + let is_mem = db_path.contains(":memory:"); + let local_db = LocalDB::new_with_busy_timeout(db_path, is_mem, busy_timeout).await; + local_db.migrate().await; + local_db +} diff --git a/crates/store/src/localdb/node.rs b/crates/store/src/localdb/node.rs new file mode 100644 index 000000000..5fad0a517 --- /dev/null +++ b/crates/store/src/localdb/node.rs @@ -0,0 +1,402 @@ +//! Node rows and the persisted peer registrations. + +use super::{StorageProcessor, get_current_timestamp_secs}; +use crate::utils::{QueryBuilder, QueryParam}; +use crate::{Node, NodesOverview}; + +use sqlx::Row; +use sqlx::types::Uuid; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Clone, Debug, Default)] +pub struct NodeQuery { + pub actor: Option, + pub goat_addr: Option, + pub time_threshold: Option, + pub is_in_time_threshold: bool, + pub order: Option, + pub offset: Option, + pub limit: Option, +} +impl NodeQuery { + pub fn with_actor(mut self, actor: String) -> Self { + self.actor = Some(actor); + self + } + + pub fn with_goat_addr(mut self, goat_addr: String) -> Self { + self.goat_addr = Some(goat_addr); + self + } + + pub fn with_order(mut self, order: String) -> Self { + self.order = Some(order); + self + } + + pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { + self.offset = Some(offset); + self.limit = Some(limit); + self + } + + pub fn with_limit(mut self, limit: u32) -> Self { + self.limit = Some(limit); + self + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::new(base_sql); + + // Add WHERE conditions + if let Some(actor) = &self.actor { + query_builder.and_where("actor = ?", Some(QueryParam::Text(actor.clone()))); + } + + if let Some(goat_addr) = &self.goat_addr { + query_builder.and_where("goat_addr = ?", Some(QueryParam::Text(goat_addr.clone()))); + } + + if let Some(time_threshold) = &self.time_threshold { + if self.is_in_time_threshold { + query_builder.and_where("updated_at > ?", Some(QueryParam::Int(*time_threshold))); + } else { + query_builder.and_where("updated_at <= ?", Some(QueryParam::Int(*time_threshold))); + } + } + + if let Some(order) = &self.order { + query_builder.apply_order(order) + } + + // Add pagination + query_builder.apply_pagination(self.limit, self.offset); + query_builder + } +} + +impl StorageProcessor<'_> { + pub async fn update_node_timestamp( + &mut self, + peer_id: &str, + timestamp: i64, + ) -> anyhow::Result<()> { + let result = + sqlx::query!(r#"UPDATE node SET updated_at = ? WHERE peer_id = ?"#, timestamp, peer_id) + .execute(self.conn()) + .await?; + + if result.rows_affected() == 0 { + // Any peer that never announced itself lands here, once per message + // burst, so this must not be louder than debug. + tracing::debug!("Node {peer_id} not found in DB, no rows updated"); + } + + Ok(()) + } + + pub async fn find_graph_neighbor_ids( + &mut self, + graph_id: Uuid, + range: i64, + ) -> anyhow::Result> { + #[derive(sqlx::FromRow)] + struct GraphIds { + pub graph_id: Uuid, + pub kickoff_index: i64, + } + if let Some(graph) = self.find_graph(&graph_id).await? { + let start = 0.max(graph.kickoff_index - range); + let end = graph.kickoff_index + range; + let res = sqlx::query_as!( + GraphIds, + "SELECT graph_id AS \"graph_id:Uuid\", kickoff_index + FROM graph + WHERE operator_pubkey = ? + AND kickoff_index >= ? + AND kickoff_index <= ?", + graph.operator_pubkey, + start, + end + ) + .fetch_all(self.conn()) + .await?; + + Ok(res.into_iter().map(|g| (g.kickoff_index, g.graph_id)).collect()) + } else { + Ok(vec![]) + } + } + + /// Insert or update node without reward field + pub async fn upsert_node(&mut self, node: &Node) -> anyhow::Result { + let res = sqlx::query!( + r#" + INSERT INTO node (peer_id, node_name, actor, goat_addr, btc_pub_key, socket_addr, service_fee_rate, available_peg_btc, + binding_sig, binding_issued_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (peer_id) DO UPDATE SET actor = excluded.actor, + node_name = excluded.node_name, + goat_addr = excluded.goat_addr, + btc_pub_key = excluded.btc_pub_key, + service_fee_rate = excluded.service_fee_rate, + available_peg_btc = excluded.available_peg_btc, + socket_addr = excluded.socket_addr, + binding_sig = excluded.binding_sig, + binding_issued_at = excluded.binding_issued_at, + updated_at = excluded.updated_at + "#, + node.peer_id, + node.node_name, + node.actor, + node.goat_addr, + node.btc_pub_key, + node.socket_addr, + node.service_fee_rate, + node.available_peg_btc, + node.binding_sig, + node.binding_issued_at, + node.created_at, + node.updated_at, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + // Do not update the `updated_at` field; this field is updated based on heartbeat messages + // and is used to determine whether a node is alive. + pub async fn update_node_reward_by_peer_id( + &mut self, + peer_id: &str, + reward: &str, + ) -> anyhow::Result<()> { + sqlx::query!(r#"UPDATE node SET reward = reward + ? WHERE peer_id = ?"#, reward, peer_id) + .execute(self.conn()) + .await?; + Ok(()) + } + + pub async fn get_node_by_btc_pub_key( + &mut self, + btc_pub_key: &str, + ) -> anyhow::Result> { + Ok(sqlx::query_as!(Node, r#"SELECT * FROM node WHERE btc_pub_key = ?"#, btc_pub_key) + .fetch_optional(self.conn()) + .await?) + } + + pub async fn find_nodes(&mut self, params: &NodeQuery) -> anyhow::Result<(Vec, i64)> { + let data_query_builder = params.get_query_builder("SELECT * FROM node"); + let mut count_params = params.clone(); + (count_params.offset, count_params.limit) = (None, None); + let count_query_builder = + count_params.get_query_builder("SELECT count(peer_id) as total_nodes FROM node"); + + let data_sql = data_query_builder.get_sql(); + let mut nodes_query = sqlx::query_as::<_, Node>(&data_sql); + nodes_query = data_query_builder.query_as(nodes_query); + + let count_sql = count_query_builder.get_sql(); + let mut count_query = sqlx::query(&count_sql); + count_query = count_query_builder.query(count_query); + + Ok(( + nodes_query.fetch_all(self.conn()).await?, + count_query.fetch_one(self.conn()).await?.get::("total_nodes"), + )) + } + + pub async fn node_overview(&mut self, time_threshold: i64) -> anyhow::Result { + let records = sqlx::query!( + r#"SELECT count(*) AS total, + actor, + SUM(CASE WHEN updated_at >= ? THEN 1 ELSE 0 END) AS online, + SUM(CASE WHEN updated_at < ? THEN 1 ELSE 0 END) AS offline + FROM node + GROUP BY actor"#, + time_threshold, + time_threshold + ) + .fetch_all(self.conn()) + .await?; + + let mut res = NodesOverview::default(); + for record in records { + res.total += record.total; + match record.actor.as_str() { + "Verifier" => { + res.offline_verifiers += record.offline; + res.online_verifiers += record.online; + } + "Operator" => { + (res.offline_operators, res.online_operators) = (record.offline, record.online); + } + "Committee" => { + (res.offline_committees, res.online_committees) = + (record.offline, record.online); + } + "Watchtower" => { + (res.offline_watchtowers, res.online_watchtowers) = + (record.offline, record.online); + } + _ => {} + }; + } + Ok(res) + } + + pub async fn node_by_id(&mut self, peer_id: &str) -> anyhow::Result> { + let res = sqlx::query_as!(Node, r#"SELECT * FROM node WHERE peer_id = ?"#, peer_id) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn get_nodes_info(&mut self, time_threshold: i64) -> anyhow::Result<(i64, i64)> { + let total = sqlx::query!(r#"SELECT COUNT(peer_id) AS total FROM node"#) + .fetch_one(self.conn()) + .await? + .total; + let time_pri = + SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() as i64 - time_threshold; + tracing::info!("{time_pri}"); + let alive = sqlx::query!( + r#"SELECT COUNT(peer_id) AS alive FROM node WHERE updated_at >= ?"#, + time_pri + ) + .fetch_one(self.conn()) + .await? + .alive; + Ok((total, alive)) + } + + /// Load up to `limit` operator bindings, confirmed operators first. + /// The caller re-verifies signatures. + pub async fn load_p2p_peer_bindings(&mut self, limit: i64) -> anyhow::Result> { + let rows = sqlx::query_as::<_, Node>( + "SELECT * FROM node \ + WHERE binding_sig != '' AND btc_pub_key != '' AND actor = 'Operator' \ + ORDER BY peer_id IN (SELECT peer_id FROM p2p_registered_peer \ + WHERE kind = 'Operator') DESC, \ + updated_at DESC \ + LIMIT ?", + ) + .bind(limit) + .fetch_all(self.conn()) + .await?; + Ok(rows) + } + + /// Persist confirmed registration; an operator key may belong to only one peer. + pub async fn upsert_p2p_registered_peer( + &mut self, + peer_id: &str, + kind: &str, + pubkey: &str, + ) -> anyhow::Result<()> { + if !pubkey.is_empty() { + sqlx::query( + "DELETE FROM p2p_registered_peer WHERE kind = ? AND pubkey = ? AND peer_id != ?", + ) + .bind(kind) + .bind(pubkey) + .bind(peer_id) + .execute(self.conn()) + .await?; + } + sqlx::query( + "INSERT INTO p2p_registered_peer (peer_id, kind, pubkey, verified_at) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (peer_id, kind) DO UPDATE SET pubkey = excluded.pubkey, \ + verified_at = excluded.verified_at", + ) + .bind(peer_id) + .bind(kind) + .bind(pubkey) + .bind(get_current_timestamp_secs()) + .execute(self.conn()) + .await?; + Ok(()) + } + + pub async fn delete_p2p_registered_peer( + &mut self, + peer_id: &str, + kind: &str, + ) -> anyhow::Result<()> { + sqlx::query("DELETE FROM p2p_registered_peer WHERE peer_id = ? AND kind = ?") + .bind(peer_id) + .bind(kind) + .execute(self.conn()) + .await?; + Ok(()) + } + + /// `(peer_id, kind, pubkey)` of every peer confirmed in an earlier session. + pub async fn load_p2p_registered_peers( + &mut self, + ) -> anyhow::Result> { + let rows = sqlx::query("SELECT peer_id, kind, pubkey FROM p2p_registered_peer") + .fetch_all(self.conn()) + .await?; + rows.iter() + .map(|row| Ok((row.try_get("peer_id")?, row.try_get("kind")?, row.try_get("pubkey")?))) + .collect::, sqlx::Error>>() + .map_err(Into::into) + } + + /// Allow existing peers or a new unregistered row below `max_rows`. + /// At capacity, evict the oldest unregistered row older than `evict_stale_before`. + pub async fn node_row_admissible( + &mut self, + peer_id: &str, + max_rows: i64, + evict_stale_before: i64, + local_peer_id: &str, + ) -> anyhow::Result { + let row = sqlx::query( + "SELECT EXISTS(SELECT 1 FROM node WHERE peer_id = ?) AS known, \ + (SELECT COUNT(*) FROM node \ + WHERE peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer)) AS total", + ) + .bind(peer_id) + .fetch_one(self.conn()) + .await?; + let known: i64 = row.try_get("known")?; + let total: i64 = row.try_get("total")?; + if known != 0 || total < max_rows { + return Ok(true); + } + let evicted = sqlx::query( + "DELETE FROM node WHERE peer_id = ( \ + SELECT peer_id FROM node \ + WHERE updated_at < ? AND peer_id != ? \ + AND peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer) \ + ORDER BY updated_at ASC LIMIT 1)", + ) + .bind(evict_stale_before) + .bind(local_peer_id) + .execute(self.conn()) + .await?; + Ok(evicted.rows_affected() > 0) + } + + /// Delete stale node rows except this node and confirmed registered peers. + pub async fn purge_stale_unregistered_nodes( + &mut self, + updated_before: i64, + local_peer_id: &str, + ) -> anyhow::Result { + let result = sqlx::query( + "DELETE FROM node \ + WHERE updated_at < ? AND peer_id != ? \ + AND peer_id NOT IN (SELECT peer_id FROM p2p_registered_peer)", + ) + .bind(updated_before) + .bind(local_peer_id) + .execute(self.conn()) + .await?; + Ok(result.rows_affected()) + } +} diff --git a/crates/store/src/localdb/proof.rs b/crates/store/src/localdb/proof.rs new file mode 100644 index 000000000..edc692fd8 --- /dev/null +++ b/crates/store/src/localdb/proof.rs @@ -0,0 +1,678 @@ +//! Proof-builder storage: the operator, watchtower and long-running chain proof rows and their +//! metrics counts. Used by proof-builder-rpc (the node itself does not read these tables). + +use super::{StorageProcessor, get_current_timestamp_secs}; +use crate::{ + LongRunningTaskProof, MetricsStateCount, OnDemandProofCompletion, OnDemandProofTable, + OperatorProof, WatchtowerProof, +}; +use sqlx::types::Uuid; + +impl StorageProcessor<'_> { + /// Returns grouped proof task counts, ages, and latest success times for Proof Builder metrics. + pub async fn proof_metrics_state_counts(&mut self) -> anyhow::Result> { + let counts = sqlx::query_as::<_, MetricsStateCount>( + r#" + SELECT + chain_name AS category, + CAST(proof_state AS TEXT) AS state, + COUNT(*) AS count, + MIN(CASE WHEN proof_state IN (0, 1) THEN created_at END) AS oldest_created_at, + MAX(CASE WHEN proof_state = 2 THEN updated_at END) AS last_success_at + FROM long_running_task_proof + GROUP BY chain_name, proof_state + UNION ALL + SELECT + 'operator' AS category, + CAST(proof_state AS TEXT) AS state, + COUNT(*) AS count, + MIN(CASE WHEN proof_state IN (0, 1) THEN created_at END) AS oldest_created_at, + MAX(CASE WHEN proof_state = 2 THEN updated_at END) AS last_success_at + FROM operator_proof + GROUP BY proof_state + UNION ALL + SELECT + 'watchtower' AS category, + CAST(proof_state AS TEXT) AS state, + COUNT(*) AS count, + MIN(CASE WHEN proof_state IN (0, 1) THEN created_at END) AS oldest_created_at, + MAX(CASE WHEN proof_state = 2 THEN updated_at END) AS last_success_at + FROM watchtower_proof + GROUP BY proof_state + ORDER BY category, state + "#, + ) + .fetch_all(self.conn()) + .await?; + Ok(counts) + } + + pub async fn create_long_running_task_proof( + &mut self, + long_running_task_proof: &LongRunningTaskProof, + ) -> anyhow::Result { + let res = sqlx::query!( + "INSERT + INTO long_running_task_proof (block_start, block_end, chain_name, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, + zkm_version, extra, updated_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + long_running_task_proof.block_start, + long_running_task_proof.block_end, + long_running_task_proof.chain_name, + long_running_task_proof.path_to_proof, + long_running_task_proof.public_value_hex, + long_running_task_proof.proof_size, + long_running_task_proof.cycles, + long_running_task_proof.proof_state, + long_running_task_proof.total_time_to_proof, + long_running_task_proof.proving_time, + long_running_task_proof.zkm_version, + long_running_task_proof.extra, + long_running_task_proof.updated_at, + long_running_task_proof.created_at, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + /// Deletes all persisted proof tasks for one proof chain. + pub async fn delete_long_running_task_proofs_by_name( + &mut self, + chain_name: &str, + ) -> anyhow::Result { + let res = sqlx::query("DELETE FROM long_running_task_proof WHERE chain_name = ?") + .bind(chain_name) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + #[allow(clippy::too_many_arguments)] + pub async fn update_long_running_task_proof_success( + &mut self, + block_start: i64, + chain_name: &str, + batch_size: i64, + path_to_proof: String, + public_value_hex: String, + proof_size: i64, + cycles: i64, + proof_state: i64, + total_time_to_proof: i64, + proving_time: i64, + zkm_version: &str, + ) -> anyhow::Result { + let block_end = block_start + batch_size; + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE long_running_task_proof + SET path_to_proof = ?, + cycles = ?, + proof_state = ?, + total_time_to_proof = ?, + proving_time = ?, + zkm_version = ?, + block_end = ?, + public_value_hex = ?, + proof_size = ?, + updated_at = ? + WHERE block_start = ? AND chain_name = ?", + path_to_proof, + cycles, + proof_state, + total_time_to_proof, + proving_time, + zkm_version, + block_end, + public_value_hex, + proof_size, + current_time, + block_start, + chain_name, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn update_long_running_task_proof_state( + &mut self, + block_start: i64, + chain_name: &str, + batch_size: i64, + proof_state: i64, + ) -> anyhow::Result { + let block_end = block_start + batch_size; + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE long_running_task_proof + SET proof_state = ?, + block_end = ?, + updated_at = ? + WHERE block_start = ? AND chain_name = ?", + proof_state, + block_end, + current_time, + block_start, + chain_name, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn find_all_running_task_proofs_by_name( + &mut self, + chain_name: String, + ) -> anyhow::Result> { + let res = sqlx::query_as!( + LongRunningTaskProof, + "SELECT block_start, block_end, chain_name, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, + zkm_version, extra, updated_at, created_at FROM long_running_task_proof + WHERE chain_name = ?", + chain_name, + ) + .fetch_all(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_long_running_task_proof_including_block_number( + &mut self, + block_number: i64, + chain_name: String, + ) -> anyhow::Result> { + let res = sqlx::query_as!( + LongRunningTaskProof, + "SELECT block_start, block_end, chain_name, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, + zkm_version, extra, updated_at, created_at FROM long_running_task_proof + + WHERE block_end > ? and block_start <= ? AND chain_name = ? LIMIT 1", + block_number, + block_number, + chain_name, + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_latest_long_running_task_proof_by_name_and_state( + &mut self, + chain_name: String, + proof_state: i64, + ) -> anyhow::Result> { + let res = sqlx::query_as!( + LongRunningTaskProof, + "SELECT + block_start, + block_end, + chain_name, + path_to_proof, + public_value_hex, + proof_size, + cycles, + proof_state, + total_time_to_proof, + proving_time, + zkm_version, + extra, + updated_at, + created_at + FROM long_running_task_proof + WHERE chain_name = ? + AND proof_state = ? + ORDER BY block_start DESC + LIMIT 1", + chain_name, + proof_state, + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_latest_long_running_task_proof_by_name( + &mut self, + chain_name: String, + ) -> anyhow::Result> { + let res = sqlx::query_as!( + LongRunningTaskProof, + "SELECT + block_start, + block_end, + chain_name, + path_to_proof, + public_value_hex, + proof_size, + cycles, + proof_state, + total_time_to_proof, + proving_time, + zkm_version, + extra, + updated_at, + created_at + FROM long_running_task_proof + WHERE chain_name = ? + ORDER BY block_start DESC + LIMIT 1", + chain_name, + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn create_operator_proof( + &mut self, + operator_proof: &OperatorProof, + ) -> anyhow::Result { + let res = sqlx::query!( + "INSERT + INTO operator_proof (instance_id, graph_id, execution_layer_block_number, path_to_proof, public_value_hex, proof_size, cycles, proof_state, total_time_to_proof, proving_time, + zkm_version, extra, updated_at, created_at, operator_committed_blockhash) + VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + operator_proof.instance_id, + operator_proof.graph_id, + operator_proof.execution_layer_block_number, + operator_proof.path_to_proof, + operator_proof.public_value_hex, + operator_proof.proof_size, + operator_proof.cycles, + operator_proof.proof_state, + operator_proof.total_time_to_proof, + operator_proof.proving_time, + operator_proof.zkm_version, + operator_proof.extra, + operator_proof.updated_at, + operator_proof.created_at, + operator_proof.operator_committed_blockhash, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + #[allow(clippy::too_many_arguments)] + pub async fn update_operator_proof( + &mut self, + id: i64, + path_to_proof: String, + public_value_hex: String, + proof_size: i64, + cycles: i64, + proof_state: i64, + total_time_to_proof: i64, + proving_time: i64, + zkm_version: &str, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE operator_proof + SET path_to_proof = ?, + public_value_hex = ?, + proof_size = ?, + cycles = ?, + proof_state = ?, + total_time_to_proof = ?, + proving_time = ?, + zkm_version = ?, + updated_at = ? + WHERE id = ?", + path_to_proof, + public_value_hex, + proof_size, + cycles, + proof_state, + total_time_to_proof, + proving_time, + zkm_version, + current_time, + id, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn update_operator_proof_state_with_instance_graph( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + old_proof_state: i64, + new_proof_state: i64, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE operator_proof + SET proof_state = ?, + updated_at = ? + WHERE instance_id = ? AND graph_id = ? AND proof_state = ?", + new_proof_state, + current_time, + instance_id, + graph_id, + old_proof_state + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn find_operator_proof_by_instance_and_graph( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, OperatorProof>( + "SELECT * FROM operator_proof + WHERE instance_id = ? + AND graph_id = ?", + ) + .bind(instance_id) + .bind(graph_id) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_next_operator_proof(&mut self) -> anyhow::Result> { + let res = sqlx::query_as::<_, OperatorProof>( + "SELECT * FROM operator_proof + WHERE proof_state == 0 + ORDER BY id ASC + LIMIT 1", + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + #[allow(clippy::too_many_arguments)] + pub async fn update_watchtower_proof( + &mut self, + id: i64, + path_to_proof: String, + public_value_hex: String, + proof_size: i64, + cycles: i64, + proof_state: i64, + total_time_to_proof: i64, + proving_time: i64, + zkm_version: &str, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE watchtower_proof + SET path_to_proof = ?, + public_value_hex = ?, + proof_size = ?, + cycles = ?, + proof_state = ?, + total_time_to_proof = ?, + proving_time = ?, + zkm_version = ?, + updated_at = ? + WHERE id = ?", + path_to_proof, + public_value_hex, + proof_size, + cycles, + proof_state, + total_time_to_proof, + proving_time, + zkm_version, + current_time, + id, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + pub async fn update_watchtower_proof_state_with_instance_graph_pubkey( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + public_key: &str, + old_proof_state: i64, + new_proof_state: i64, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE watchtower_proof + SET proof_state = ?, + updated_at = ? + WHERE instance_id = ? + AND graph_id = ? + AND public_key = ? AND proof_state = ?", + new_proof_state, + current_time, + instance_id, + graph_id, + public_key, + old_proof_state + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn update_watchtower_proof_challenge_txid( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + node_index: i32, + challenge_txid: &str, + included: bool, + ) -> anyhow::Result { + let current_time = get_current_timestamp_secs(); + let res = sqlx::query!( + "UPDATE watchtower_proof + SET challenge_txid = ?, + included =?, + updated_at = ? + WHERE instance_id = ? + AND graph_id = ? + AND node_index = ?", + challenge_txid, + included, + current_time, + instance_id, + graph_id, + node_index, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn update_watchtower_proof_node_index( + &mut self, + id: i64, + instance_id: &Uuid, + graph_id: &Uuid, + node_index: i32, + ) -> anyhow::Result { + let res = sqlx::query( + "UPDATE watchtower_proof + SET node_index = ? + WHERE id = ? + AND instance_id = ? + AND graph_id = ?", + ) + .bind(node_index) + .bind(id) + .bind(instance_id) + .bind(graph_id) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + pub async fn find_watchtower_proof_by_instance_and_graph( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, WatchtowerProof>( + "SELECT * + FROM watchtower_proof + WHERE instance_id = ? + AND graph_id = ? + ORDER BY node_index ASC, id ASC", + ) + .bind(instance_id) + .bind(graph_id) + .fetch_all(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_watchtower_proof_by_instance_and_graph_and_pubkey( + &mut self, + instance_id: &Uuid, + graph_id: &Uuid, + public_key: &str, + ) -> anyhow::Result> { + let res = sqlx::query_as::<_, WatchtowerProof>( + "SELECT * + FROM watchtower_proof + WHERE instance_id = ? + AND graph_id = ? + AND public_key = ?", + ) + .bind(instance_id) + .bind(graph_id) + .bind(public_key) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn find_next_watchtower_proof(&mut self) -> anyhow::Result> { + let res = sqlx::query_as::<_, WatchtowerProof>( + "SELECT * + FROM watchtower_proof + WHERE proof_state == 0 + ORDER BY id ASC + lIMIT 1", + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } + + pub async fn create_watchtower_proof( + &mut self, + watchtower_proof: &WatchtowerProof, + ) -> anyhow::Result { + let res = sqlx::query!( + "INSERT + INTO watchtower_proof (instance_id, graph_id, public_key, challenge_txid, challenge_init_txid, execution_layer_block_number, + path_to_proof, + public_value_hex, proof_size, + cycles, proof_state, total_time_to_proof, proving_time, + zkm_version, node_index, included, + extra, updated_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + watchtower_proof.instance_id, + watchtower_proof.graph_id, + watchtower_proof.public_key, + watchtower_proof.challenge_txid, + watchtower_proof.challenge_init_txid, + watchtower_proof.execution_layer_block_number, + watchtower_proof.path_to_proof, + watchtower_proof.public_value_hex, + watchtower_proof.proof_size, + watchtower_proof.cycles, + watchtower_proof.proof_state, + watchtower_proof.total_time_to_proof, + watchtower_proof.proving_time, + watchtower_proof.zkm_version, + watchtower_proof.node_index, + watchtower_proof.included, + watchtower_proof.extra, + watchtower_proof.updated_at, + watchtower_proof.created_at, + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } +} + +impl StorageProcessor<'_> { + /// Finishes an on-demand proof task only while it is still `New`. + /// + /// A timeout request may mark the task `Failed` while its proof is queued or + /// running; that state wins and this update affects no row. + pub async fn complete_on_demand_proof_if_new( + &mut self, + table: OnDemandProofTable, + id: i64, + completion: &OnDemandProofCompletion, + ) -> anyhow::Result { + let sql = format!( + "UPDATE {} + SET path_to_proof = ?, + public_value_hex = ?, + proof_size = ?, + cycles = ?, + proof_state = ?, + total_time_to_proof = ?, + proving_time = ?, + zkm_version = ?, + updated_at = ? + WHERE id = ? AND proof_state = ?", + table.table_name() + ); + let res = sqlx::query(&sql) + .bind(&completion.path_to_proof) + .bind(&completion.public_value_hex) + .bind(completion.proof_size) + .bind(completion.cycles) + .bind(completion.proof_state) + .bind(completion.total_time_to_proof) + .bind(completion.proving_time) + .bind(&completion.zkm_version) + .bind(get_current_timestamp_secs()) + .bind(id) + .bind(crate::ProofState::New.to_i64()) + .execute(self.conn()) + .await?; + Ok(res.rows_affected()) + } + + /// Returns the current proof state of an on-demand proof task. + pub async fn get_on_demand_proof_state( + &mut self, + table: OnDemandProofTable, + id: i64, + ) -> anyhow::Result> { + let sql = format!("SELECT proof_state FROM {} WHERE id = ?", table.table_name()); + let state = sqlx::query_scalar::<_, i64>(&sql).bind(id).fetch_optional(self.conn()).await?; + Ok(state) + } + + /// Counts unfinished (`New` or `Proving`) tasks of one proof chain starting at or after + /// `from_block`. + pub async fn count_pending_long_running_task_proofs( + &mut self, + chain_name: &str, + from_block: i64, + ) -> anyhow::Result { + let count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM long_running_task_proof + WHERE chain_name = ? AND block_start >= ? AND proof_state IN (?, ?)", + ) + .bind(chain_name) + .bind(from_block) + .bind(crate::ProofState::New.to_i64()) + .bind(crate::ProofState::Proving.to_i64()) + .fetch_one(self.conn()) + .await?; + Ok(count) + } +} diff --git a/crates/store/src/localdb/stats.rs b/crates/store/src/localdb/stats.rs new file mode 100644 index 000000000..24a80052c --- /dev/null +++ b/crates/store/src/localdb/stats.rs @@ -0,0 +1,200 @@ +//! Counts and sums for metrics, the overview and the bridge-out statistics. + +use super::StorageProcessor; +use crate::utils::create_place_holders; +use crate::{ + BridgeOutGlobalStats, EventWatchMetricsSnapshot, MetricsStateCount, NodeAlertMetricsSnapshot, +}; + +impl StorageProcessor<'_> { + /// Returns grouped instance, graph, and queue state counts for Node metrics. + pub async fn node_metrics_state_counts(&mut self) -> anyhow::Result> { + let counts = sqlx::query_as::<_, MetricsStateCount>( + r#" + SELECT + 'instance' AS category, + status AS state, + COUNT(*) AS count, + MIN(created_at) AS oldest_created_at, + NULL AS last_success_at + FROM instance + GROUP BY status + UNION ALL + SELECT + 'swap_escrow' AS category, + status AS state, + COUNT(*) AS count, + MIN(created_at) AS oldest_created_at, + NULL AS last_success_at + FROM swap_escrow + GROUP BY status + UNION ALL + SELECT + 'graph' AS category, + status AS state, + COUNT(*) AS count, + MIN(created_at) AS oldest_created_at, + NULL AS last_success_at + FROM graph + GROUP BY status + ORDER BY category, state + "#, + ) + .fetch_all(self.conn()) + .await?; + Ok(counts) + } + + /// Returns local aggregate values for flow-stall and operator-liquidity alerts. + pub async fn node_alert_metrics_snapshot( + &mut self, + local_peer_id: &str, + ) -> anyhow::Result { + let snapshot = sqlx::query_as::<_, NodeAlertMetricsSnapshot>( + r#" + SELECT + ( + SELECT MIN(status_updated_at) + FROM instance + WHERE status NOT IN ( + 'RelayerL2Minted', 'PresignedFailed', 'RelayerL2MintedFailed', + 'Timeout', 'UserCanceled', 'NoEnoughCommitteesAnswered', 'UserDiscarded', + 'Failed', 'Success', 'Canceled' + ) + ) AS pegin_oldest_active_status_updated_at, + ( + SELECT MIN(status_updated_at) + FROM instance + WHERE status = 'UserInited' + ) AS pegin_oldest_committee_wait_status_updated_at, + ( + SELECT MIN(status_updated_at) + FROM swap_escrow + WHERE status NOT IN ('Claim', 'Timeout', 'Refund') + ) AS pegout_oldest_active_status_updated_at, + ( + SELECT available_peg_btc + FROM node + WHERE peer_id = ? AND actor = 'Operator' + LIMIT 1 + ) AS operator_available_pegbtc + "#, + ) + .bind(local_peer_id) + .fetch_one(self.conn()) + .await?; + Ok(snapshot) + } + + /// Returns aggregate local progress for the configured event watchers. + pub async fn event_watch_metrics_snapshot( + &mut self, + finalized_height: i64, + ) -> anyhow::Result { + Ok(sqlx::query_as::<_, EventWatchMetricsSnapshot>( + r#" + SELECT + COALESCE(MAX(MAX(? - from_height + 1, 0)), 0) AS lag_blocks, + COUNT(*) AS watcher_count, + COALESCE(SUM(CASE WHEN status IN ('UnSync', 'Syncing') THEN 1 ELSE 0 END), 0) + AS syncing_count, + COALESCE(SUM(CASE WHEN status = 'Failed' THEN 1 ELSE 0 END), 0) + AS failed_count + FROM watch_contract + WHERE from_height != 0 + "#, + ) + .bind(finalized_height) + .fetch_one(self.conn()) + .await?) + } + + pub async fn get_sum_bridge_in_txn( + &mut self, + statuses: &[String], + ) -> anyhow::Result<(i64, i64)> { + #[derive(sqlx::FromRow)] + struct BridgeInRow { + pub total: i64, + pub tx_count: i64, + } + + let query_str = format!( + "SELECT SUM(amount) AS total, COUNT(*) AS tx_count + FROM instance + WHERE status IN ({})", + create_place_holders(statuses) + ); + let mut query = sqlx::query_as::<_, BridgeInRow>(&query_str); + for status in statuses { + query = query.bind(status); + } + let record = query.fetch_one(self.conn()).await?; + Ok((record.total, record.tx_count)) + } + + pub async fn get_sum_peg_out(&mut self, statuses: &[String]) -> anyhow::Result<(i64, i64)> { + #[derive(sqlx::FromRow)] + struct BridgeOutRow { + pub total: i64, + pub tx_count: i64, + } + + let query_str = format!( + "SELECT SUM(amount) AS total, COUNT(*) AS tx_count + FROM graph + WHERE status IN + ({})", + create_place_holders(statuses) + ); + let mut query = sqlx::query_as::<_, BridgeOutRow>(&query_str); + for status in statuses { + query = query.bind(status); + } + let record = query.fetch_one(self.conn()).await?; + Ok((record.total, record.tx_count)) + } + + pub async fn upsert_bridge_out_global_stats( + &mut self, + bridge_out_stats: &BridgeOutGlobalStats, + ) -> anyhow::Result<()> { + sqlx::query!( + r#"INSERT INTO bridge_out_global_stats (id, initial_txn, initial_amount, claim_txn, claim_amount, refund_txn, + refund_amount,updated_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (id) DO UPDATE SET initial_txn = excluded.initial_txn, + initial_amount = excluded.initial_amount, + claim_txn = excluded.claim_txn, + claim_amount = excluded.claim_amount, + refund_txn = excluded.refund_txn, + refund_amount = excluded.refund_amount, + updated_at = excluded.updated_at"#, + bridge_out_stats.id, + bridge_out_stats.initial_txn, + bridge_out_stats.initial_amount, + bridge_out_stats.claim_txn, + bridge_out_stats.claim_amount, + bridge_out_stats.refund_txn, + bridge_out_stats.refund_amount, + bridge_out_stats.updated_at, + bridge_out_stats.created_at, + ).execute(self.conn()) + .await?; + Ok(()) + } + + pub async fn find_bridge_out_global_stats_by_id( + &mut self, + id: i64, + ) -> anyhow::Result> { + let res = sqlx::query_as!( + BridgeOutGlobalStats, + r#"SELECT * from bridge_out_global_stats WHERE id = ?"#, + id + ) + .fetch_optional(self.conn()) + .await?; + Ok(res) + } +} diff --git a/crates/store/src/localdb/swap.rs b/crates/store/src/localdb/swap.rs new file mode 100644 index 000000000..167d148dd --- /dev/null +++ b/crates/store/src/localdb/swap.rs @@ -0,0 +1,277 @@ +//! Swap escrow rows. + +use super::{StorageProcessor, get_current_timestamp_secs}; +use crate::utils::{QueryBuilder, QueryParam}; +use crate::{SerializableTxid, SwapEscrow, SwapEscrowStatus}; + +use sqlx::Row; + +/// Filter parameters for listing swap escrows. +#[derive(Clone, Debug, Default)] +pub struct SwapEscrowQuery { + pub offerer_addr: Option, + pub statuses: Vec, + pub order: Option, + pub offset: Option, + pub limit: Option, +} + +impl SwapEscrowQuery { + pub fn with_offerer_addr(mut self, offerer_addr: String) -> Self { + self.offerer_addr = Some(offerer_addr); + self + } + + pub fn with_status(mut self, status: String) -> Self { + self.statuses.push(status); + self + } + + pub fn with_order(mut self, order: String) -> Self { + self.order = Some(order); + self + } + + pub fn with_pagination(mut self, offset: u32, limit: u32) -> Self { + self.offset = Some(offset); + self.limit = Some(limit); + self + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::new(base_sql); + if let Some(offerer_addr) = &self.offerer_addr { + query_builder + .and_where("offerer_addr = ?", Some(QueryParam::Text(offerer_addr.clone()))); + } + if !self.statuses.is_empty() { + query_builder.and_where_in("status", &self.statuses, false); + } + if let Some(order) = &self.order { + query_builder.apply_order(order); + } + query_builder.apply_pagination(self.limit, self.offset); + query_builder + } +} + +/// Swap escrow field update parameters, keyed by escrow hash. +#[derive(Debug, Clone, Default)] +pub struct SwapEscrowUpdate { + pub escrow_hash: String, + pub status: Option, + pub claimer_addr: Option, + pub btc_addr: Option, + pub token: Option, + pub amount: Option, + pub refund_deadline: Option, + pub escrow_data: Option, + pub init_tx_hash: Option, + pub init_tx_height: Option, + pub claim_tx_hash: Option, + pub claim_btc_txid: Option, + pub refund_tx_hash: Option, + pub only_if_status_in: Option>, +} + +impl SwapEscrowUpdate { + pub fn new(escrow_hash: String) -> Self { + Self { escrow_hash, ..Self::default() } + } + + pub fn with_status(mut self, status: String) -> Self { + self.status = Some(status); + self + } + + pub fn with_btc_addr(mut self, btc_addr: String) -> Self { + self.btc_addr = Some(btc_addr); + self + } + + pub fn with_claim_tx_hash(mut self, tx_hash: String) -> Self { + self.claim_tx_hash = Some(tx_hash); + self + } + + pub fn with_claim_btc_txid(mut self, txid: SerializableTxid) -> Self { + self.claim_btc_txid = Some(txid); + self + } + + pub fn with_refund_tx_hash(mut self, tx_hash: String) -> Self { + self.refund_tx_hash = Some(tx_hash); + self + } + + /// Apply this update only while the escrow is still in one of the + /// expected states. The condition is folded into the UPDATE statement. + pub fn with_only_if_status_in(mut self, statuses: Vec) -> Self { + self.only_if_status_in = Some(statuses); + self + } + + pub fn get_query_builder(&self, base_sql: &str) -> QueryBuilder { + let mut query_builder = QueryBuilder::update(base_sql); + if let Some(ref status) = self.status { + query_builder.set_field("status", QueryParam::Text(status.clone())); + query_builder + .set_field("status_updated_at", QueryParam::Int(get_current_timestamp_secs())); + } + if let Some(ref claimer_addr) = self.claimer_addr { + query_builder.set_field("claimer_addr", QueryParam::Text(claimer_addr.clone())); + } + if let Some(ref btc_addr) = self.btc_addr { + query_builder.set_field("btc_addr", QueryParam::Text(btc_addr.clone())); + } + if let Some(ref token) = self.token { + query_builder.set_field("token", QueryParam::Text(token.clone())); + } + if let Some(ref amount) = self.amount { + query_builder.set_field("amount", QueryParam::Text(amount.clone())); + } + if let Some(refund_deadline) = self.refund_deadline { + query_builder.set_field("refund_deadline", QueryParam::Int(refund_deadline)); + } + if let Some(ref escrow_data) = self.escrow_data { + query_builder.set_field("escrow_data", QueryParam::Text(escrow_data.clone())); + } + if let Some(ref init_tx_hash) = self.init_tx_hash { + query_builder.set_field("init_tx_hash", QueryParam::Text(init_tx_hash.clone())); + } + if let Some(init_tx_height) = self.init_tx_height { + query_builder.set_field("init_tx_height", QueryParam::Int(init_tx_height)); + } + if let Some(ref claim_tx_hash) = self.claim_tx_hash { + query_builder.set_field("claim_tx_hash", QueryParam::Text(claim_tx_hash.clone())); + } + if let Some(ref claim_btc_txid) = self.claim_btc_txid { + query_builder.set_field("claim_btc_txid", QueryParam::BTCTxid(claim_btc_txid.clone())); + } + if let Some(ref refund_tx_hash) = self.refund_tx_hash { + query_builder.set_field("refund_tx_hash", QueryParam::Text(refund_tx_hash.clone())); + } + + query_builder.set_field("updated_at", QueryParam::Int(get_current_timestamp_secs())); + + query_builder + .and_where("escrow_hash = ?", Some(QueryParam::Text(self.escrow_hash.clone()))); + if let Some(ref statuses) = self.only_if_status_in { + if statuses.is_empty() { + // An empty allow-list must reject the update rather than + // silently dropping the compare-and-swap guard. + query_builder.and_where("1 = 0", None); + } else { + query_builder.and_where_in("status", statuses, false); + } + } + query_builder + } +} + +impl StorageProcessor<'_> { + /// Insert a swap escrow only when its escrow hash is not already present. + /// + /// The chain-event watcher is the sole writer for Initialize records; + /// the escrow-hash primary key makes event replays idempotent. + pub async fn insert_swap_escrow_if_absent( + &mut self, + escrow: &SwapEscrow, + ) -> anyhow::Result { + let res = sqlx::query!( + "INSERT INTO swap_escrow \ + (escrow_hash, network, status, offerer_addr, claimer_addr, btc_addr, token, amount, \ + refund_deadline, escrow_data, init_tx_hash, init_tx_height, claim_tx_hash, \ + claim_btc_txid, refund_tx_hash, status_updated_at, created_at, updated_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(escrow_hash) DO NOTHING", + escrow.escrow_hash, + escrow.network, + escrow.status, + escrow.offerer_addr, + escrow.claimer_addr, + escrow.btc_addr, + escrow.token, + escrow.amount, + escrow.refund_deadline, + escrow.escrow_data, + escrow.init_tx_hash, + escrow.init_tx_height, + escrow.claim_tx_hash, + escrow.claim_btc_txid, + escrow.refund_tx_hash, + escrow.status_updated_at, + escrow.created_at, + escrow.updated_at + ) + .execute(self.conn()) + .await?; + Ok(res.rows_affected() > 0) + } + + /// Find a swap escrow by its (normalized) escrow hash. + pub async fn find_swap_escrow( + &mut self, + escrow_hash: &str, + ) -> anyhow::Result> { + let row = + sqlx::query_as::<_, SwapEscrow>("SELECT * FROM swap_escrow WHERE escrow_hash = ?") + .bind(escrow_hash) + .fetch_optional(self.conn()) + .await?; + Ok(row) + } + + /// Find swap escrows with filtering and pagination. + /// + /// Returns the matching page together with the total match count. + pub async fn find_swap_escrows( + &mut self, + params: SwapEscrowQuery, + ) -> anyhow::Result<(Vec, i64)> { + let mut count_params = params.clone(); + (count_params.order, count_params.offset) = (None, None); + let escrows_query_builder = params.get_query_builder("SELECT * FROM swap_escrow"); + let count_query_builder = + count_params.get_query_builder("SELECT count(*) as total_escrows FROM swap_escrow"); + let sql = escrows_query_builder.get_sql(); + let mut data_query = sqlx::query_as::<_, SwapEscrow>(&sql); + data_query = escrows_query_builder.query_as(data_query); + let count_sql = count_query_builder.get_sql(); + let mut count_query = sqlx::query(&count_sql); + count_query = count_query_builder.query(count_query); + Ok(( + data_query.fetch_all(self.conn()).await?, + count_query.fetch_one(self.conn()).await?.get::("total_escrows"), + )) + } + + /// Update swap escrow fields using the builder pattern. + pub async fn update_swap_escrow(&mut self, params: &SwapEscrowUpdate) -> anyhow::Result { + let query_builder = params.get_query_builder("swap_escrow"); + let update_sql = query_builder.get_sql(); + let mut query = sqlx::query(&update_sql); + query = query_builder.query(query); + let result = query.execute(self.conn()).await?; + Ok(result.rows_affected() > 0) + } + + /// Mark still-initializing swap escrows whose refund deadline has passed + /// as timed out. Returns the number of escrows transitioned. + pub async fn timeout_expired_swap_escrows(&mut self, current_time: i64) -> anyhow::Result { + let initialize_status = SwapEscrowStatus::Initialize.to_string(); + let timeout_status = SwapEscrowStatus::Timeout.to_string(); + let row = sqlx::query!( + "UPDATE swap_escrow SET status = ?, status_updated_at = ?, updated_at = ? \ + WHERE status = ? AND refund_deadline > 0 AND refund_deadline < ?", + timeout_status, + current_time, + current_time, + initialize_status, + current_time + ) + .execute(self.conn()) + .await?; + Ok(row.rows_affected()) + } +} diff --git a/crates/store/src/localdb/tests.rs b/crates/store/src/localdb/tests.rs new file mode 100644 index 000000000..00d4c85dd --- /dev/null +++ b/crates/store/src/localdb/tests.rs @@ -0,0 +1,558 @@ +use super::*; +use crate::{Instance, MetricsStateCount, Node, OperatorProof}; +use indexmap::IndexMap; +use sqlx::types::Uuid; + +async fn setup_db() -> LocalDB { + create_local_db("sqlite::memory:").await +} + +fn pegin_instance(instance_id: Uuid, status: &str) -> Instance { + Instance { + instance_id, + network: "testnet".to_string(), + input_utxos: "[]".to_string(), + status: status.to_string(), + created_at: 100, + updated_at: 100, + ..Default::default() + } +} + +#[tokio::test] +async fn test_upsert_pegin_request_instance_refreshes_a_waiting_row() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let instance_id = Uuid::new_v4(); + + let mut initing = pegin_instance(instance_id, "UserInited"); + initing.to_addr = "0xuser".to_string(); + initing.from_addr = "bcrt1quser".to_string(); + assert!(s.insert_instance_if_absent(&initing).await.unwrap()); + + let mut request = pegin_instance(instance_id, "UserInited"); + request.to_addr = "0xuser".to_string(); + request.amount = 10_000; + request.goat_tx_hash = "0xrequest".to_string(); + request.goat_tx_height = 500; + request.status_updated_at = 200; + request.created_at = 999; + request.updated_at = 999; + assert!(s.upsert_pegin_request_instance(&request).await.unwrap()); + + let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); + assert_eq!(stored.status, "UserInited"); + assert_eq!(stored.amount, 10_000); + assert_eq!(stored.goat_tx_hash, "0xrequest"); + assert_eq!(stored.goat_tx_height, 500); + assert_eq!(stored.status_updated_at, 200); + // An undecodable from_addr must not erase the one the request tag stored. + assert_eq!(stored.from_addr, "bcrt1quser"); + // created_at belongs to whoever created the row. + assert_eq!(stored.created_at, 100); +} + +#[tokio::test] +async fn test_upsert_pegin_request_instance_replay_keeps_accrued_state() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let instance_id = Uuid::new_v4(); + + let mut inited = pegin_instance(instance_id, "UserInited"); + inited.goat_tx_hash = "0xrequest".to_string(); + inited.goat_tx_height = 500; + inited.committees_answers = IndexMap::from([("0xcommittee".to_string(), vec![1u8, 2, 3])]); + inited.parameters = Some("{}".to_string()); + assert!(s.insert_instance_if_absent(&inited).await.unwrap()); + + // A re-delivered request refreshes the row in place; everything the + // instance accrued after the request must survive it. + let mut replay = pegin_instance(instance_id, "UserInited"); + replay.goat_tx_hash = "0xrequest".to_string(); + replay.goat_tx_height = 500; + assert!(s.upsert_pegin_request_instance(&replay).await.unwrap()); + let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); + assert_eq!(stored.committees_answers.get("0xcommittee"), Some(&vec![1u8, 2, 3])); + assert_eq!(stored.parameters, Some("{}".to_string())); + assert_eq!(stored.goat_tx_height, 500); +} + +#[tokio::test] +async fn test_upsert_pegin_request_instance_rejects_regression() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let instance_id = Uuid::new_v4(); + + let mut minted = pegin_instance(instance_id, "RelayerL2Minted"); + minted.goat_tx_hash = "0xrequest".to_string(); + minted.goat_tx_height = 500; + minted.parameters = Some("{}".to_string()); + minted.post_pegin_txhash = Some("0xmint".to_string()); + assert!(s.insert_instance_if_absent(&minted).await.unwrap()); + + // Once the instance moves on, a re-delivered request may not pull it back. + let replay = pegin_instance(instance_id, "UserInited"); + assert!(!s.upsert_pegin_request_instance(&replay).await.unwrap()); + let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); + assert_eq!(stored.status, "RelayerL2Minted"); + assert_eq!(stored.parameters, Some("{}".to_string())); + assert_eq!(stored.post_pegin_txhash, Some("0xmint".to_string())); + assert_eq!(stored.goat_tx_height, 500); +} + +#[tokio::test] +async fn test_upsert_pegin_request_instance_rejects_progressed_instance() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let instance_id = Uuid::new_v4(); + + let progressed = pegin_instance(instance_id, "CommitteesAnswered"); + assert!(s.insert_instance_if_absent(&progressed).await.unwrap()); + + let request = pegin_instance(instance_id, "UserInited"); + assert!(!s.upsert_pegin_request_instance(&request).await.unwrap()); + let stored = s.find_instance(&instance_id).await.unwrap().unwrap(); + assert_eq!(stored.status, "CommitteesAnswered"); +} + +#[tokio::test] +async fn test_instance_update_rejects_stale_status_transition() { + let db = setup_db().await; + let instance_id = Uuid::new_v4(); + let mut storage = db.acquire().await.unwrap(); + assert!( + storage + .insert_instance_if_absent(&pegin_instance(instance_id, "RelayerL2Minted")) + .await + .unwrap() + ); + + let updated = storage + .update_instance( + &InstanceUpdate::new_with_instance_id(instance_id) + .with_status("Timeout".to_string()) + .with_only_if_status_in(vec!["Presigned".to_string()]), + ) + .await + .unwrap(); + assert!(!updated); + assert_eq!( + storage.find_instance(&instance_id).await.unwrap().unwrap().status, + "RelayerL2Minted" + ); +} + +#[tokio::test] +async fn test_node_metrics_state_counts() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + + sqlx::query( + "INSERT INTO instance (instance_id, status, created_at, updated_at) VALUES ('in-1', 'Pending', 20, 20), ('in-2', 'Pending', 10, 10)", + ) + .execute(s.conn()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO swap_escrow (escrow_hash, status, created_at, updated_at) VALUES ('0xaa', 'Initialize', 30, 30)", + ) + .execute(s.conn()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO graph (graph_id, instance_id, status, kickoff_index, created_at, updated_at) VALUES ('graph-1', 'in-1', 'Created', 0, 40, 40), ('graph-2', 'in-2', 'Created', 1, 25, 25)", + ) + .execute(s.conn()) + .await + .unwrap(); + + let counts = s.node_metrics_state_counts().await.unwrap(); + assert_eq!( + counts.iter().find(|count| count.category == "instance" && count.state == "Pending"), + Some(&MetricsStateCount { + category: "instance".to_string(), + state: "Pending".to_string(), + count: 2, + oldest_created_at: Some(10), + last_success_at: None, + }) + ); + assert_eq!( + counts + .iter() + .find(|count| count.category == "swap_escrow" && count.state == "Initialize") + .unwrap() + .count, + 1 + ); + assert_eq!( + counts + .iter() + .find(|count| count.category == "graph" && count.state == "Created") + .unwrap() + .oldest_created_at, + Some(25) + ); + assert!( + counts.iter().all(|count| { + matches!(count.category.as_str(), "instance" | "swap_escrow" | "graph") + }) + ); +} + +/// Verify confirmed operator bindings are loaded before unconfirmed identities. +#[tokio::test] +async fn confirmed_operator_bindings_are_loaded_first() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let operator = |peer_id: &str, updated_at: i64| Node { + peer_id: peer_id.to_string(), + actor: "Operator".to_string(), + btc_pub_key: format!("key-of-{peer_id}"), + binding_sig: "sig".to_string(), + binding_issued_at: 1, + updated_at, + created_at: updated_at, + ..Default::default() + }; + s.upsert_node(&operator("confirmed", 10)).await.unwrap(); + for index in 0..4 { + s.upsert_node(&operator(&format!("recent-{index}"), 900 + index)).await.unwrap(); + } + // Not an operator, and an operator without a binding: never loaded. + s.upsert_node(&Node { actor: "Committee".to_string(), ..operator("member", 999) }) + .await + .unwrap(); + s.upsert_node(&Node { binding_sig: String::new(), ..operator("unbound", 999) }).await.unwrap(); + s.upsert_p2p_registered_peer("confirmed", "Operator", "aa").await.unwrap(); + + let loaded: Vec = + s.load_p2p_peer_bindings(2).await.unwrap().into_iter().map(|node| node.peer_id).collect(); + assert_eq!(loaded, ["confirmed", "recent-3"]); +} + +#[tokio::test] +async fn registered_peers_persist_and_shield_their_node_rows() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + s.upsert_p2p_registered_peer("member", "Committee", "").await.unwrap(); + s.upsert_p2p_registered_peer("member", "Committee", "").await.unwrap(); + s.upsert_p2p_registered_peer("other-member", "Committee", "").await.unwrap(); + s.upsert_p2p_registered_peer("operator", "Operator", "aa").await.unwrap(); + let load = async |s: &mut StorageProcessor<'_>| { + let mut peers = s.load_p2p_registered_peers().await.unwrap(); + peers.sort(); + peers + }; + let row = + |peer: &str, kind: &str, key: &str| (peer.to_string(), kind.to_string(), key.to_string()); + assert_eq!( + load(&mut s).await, + [ + row("member", "Committee", ""), + row("operator", "Operator", "aa"), + row("other-member", "Committee", "") + ], + "keyless kinds never displace each other" + ); + // Move the key's registration to the new peer. + s.upsert_p2p_registered_peer("operator-moved", "Operator", "aa").await.unwrap(); + // A peer confirmed for another key keeps one row, for the new key. + s.upsert_p2p_registered_peer("operator-moved", "Operator", "bb").await.unwrap(); + assert_eq!( + load(&mut s).await, + [ + row("member", "Committee", ""), + row("operator-moved", "Operator", "bb"), + row("other-member", "Committee", "") + ] + ); + s.delete_p2p_registered_peer("operator-moved", "Operator").await.unwrap(); + s.delete_p2p_registered_peer("other-member", "Committee").await.unwrap(); + assert_eq!(load(&mut s).await.len(), 1); + + for (peer_id, updated_at) in [("member", 10), ("sybil", 10), ("local", 10), ("live", 900)] { + s.upsert_node(&Node { + peer_id: peer_id.to_string(), + actor: "Operator".to_string(), + updated_at, + created_at: updated_at, + ..Default::default() + }) + .await + .unwrap(); + } + // Only unregistered rows count toward capacity; stale rows may be evicted. + assert!(s.node_row_admissible("sybil", 3, 0, "local").await.unwrap()); + assert!(s.node_row_admissible("newcomer", 4, 0, "local").await.unwrap()); + assert!( + !s.node_row_admissible("newcomer", 3, 5, "local").await.unwrap(), + "full, and every row was refreshed after the eviction horizon" + ); + assert!( + s.node_row_admissible("newcomer", 3, 500, "local").await.unwrap(), + "the stalest unregistered row makes room" + ); + let known = async |s: &mut StorageProcessor<'_>, peer_id: &str| { + s.node_row_admissible(peer_id, 0, 0, "local").await.unwrap() + }; + assert!(!known(&mut s, "sybil").await, "evicted: stale, unregistered, not this node"); + for kept in ["member", "local", "live"] { + assert!(known(&mut s, kept).await, "{kept}"); + } + + s.upsert_node(&Node { + peer_id: "left".to_string(), + actor: "Watchtower".to_string(), + updated_at: 20, + created_at: 20, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(s.purge_stale_unregistered_nodes(500, "local").await.unwrap(), 1); + assert!(!known(&mut s, "left").await); + assert!(known(&mut s, "member").await && known(&mut s, "local").await); +} + +#[tokio::test] +async fn test_upsert_and_find_latest_hash_change() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + + s.upsert_sequencer_set_hash_change(100, 1000, "aabbcc").await.unwrap(); + s.upsert_sequencer_set_hash_change(200, 2000, "ddeeff").await.unwrap(); + s.upsert_sequencer_set_hash_change(150, 1500, "112233").await.unwrap(); + + let latest = s.find_latest_sequencer_set_hash_change().await.unwrap().unwrap(); + assert_eq!(latest.cosmos_block_height, 200); + assert_eq!(latest.goat_block_height, 2000); + assert_eq!(latest.validators_hash, "ddeeff"); +} + +#[tokio::test] +async fn test_upsert_hash_change_conflict() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + + s.upsert_sequencer_set_hash_change(100, 1000, "aabbcc").await.unwrap(); + // Same cosmos_block_height, different data — should overwrite + s.upsert_sequencer_set_hash_change(100, 1001, "ddeeff").await.unwrap(); + + let latest = s.find_latest_sequencer_set_hash_change().await.unwrap().unwrap(); + assert_eq!(latest.cosmos_block_height, 100); + assert_eq!(latest.goat_block_height, 1001); + assert_eq!(latest.validators_hash, "ddeeff"); +} + +#[tokio::test] +async fn test_find_by_goat_block_at_or_before() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + + s.upsert_sequencer_set_hash_change(100, 1000, "aa").await.unwrap(); + s.upsert_sequencer_set_hash_change(200, 2000, "bb").await.unwrap(); + s.upsert_sequencer_set_hash_change(300, 3000, "cc").await.unwrap(); + + // Exact match + let r = s + .find_first_sequencer_set_hash_change_by_goat_block_at_or_before(2000) + .await + .unwrap() + .unwrap(); + assert_eq!(r.goat_block_height, 2000); + assert_eq!(r.validators_hash, "bb"); + + // Between records — should return previous one + let r = s + .find_first_sequencer_set_hash_change_by_goat_block_at_or_before(2500) + .await + .unwrap() + .unwrap(); + assert_eq!(r.goat_block_height, 2000); + assert_eq!(r.validators_hash, "bb"); + + // Before all records — should return None + let r = s.find_first_sequencer_set_hash_change_by_goat_block_at_or_before(500).await.unwrap(); + assert!(r.is_none()); +} + +#[tokio::test] +async fn test_scan_state_upsert_and_get() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + + assert!(s.get_sequencer_set_scan_state().await.unwrap().is_none()); + + s.upsert_sequencer_set_scan_state(100, 1000, "aabbcc").await.unwrap(); + let state = s.get_sequencer_set_scan_state().await.unwrap().unwrap(); + assert_eq!(state.next_cosmos_block_height, 100); + assert_eq!(state.latest_goat_block_height, 1000); + assert_eq!(state.latest_validators_hash, "aabbcc"); + + // Update + s.upsert_sequencer_set_scan_state(200, 2000, "ddeeff").await.unwrap(); + let state = s.get_sequencer_set_scan_state().await.unwrap().unwrap(); + assert_eq!(state.next_cosmos_block_height, 200); + assert_eq!(state.latest_validators_hash, "ddeeff"); +} + +// proof-builder-rpc keeps its proofs in these tables through the shared store migrations; offline sqlx +// checks would not notice the tables disappearing from the migrations, this does. +#[tokio::test] +async fn test_proof_builder_tables_are_migrated() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + + // Ziren 2.0 removes the bundled v1 header proof. The table must still exist. + assert!( + s.find_latest_long_running_task_proof_by_name("header-chain".to_string()) + .await + .unwrap() + .is_none() + ); + + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let proof = OperatorProof { + instance_id, + graph_id, + operator_committed_blockhash: "00".repeat(32), + execution_layer_block_number: 7, + ..Default::default() + }; + assert_eq!(s.create_operator_proof(&proof).await.unwrap(), 1); + let stored = s + .find_operator_proof_by_instance_and_graph(&instance_id, &graph_id) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.execution_layer_block_number, 7); + assert!(s.proof_metrics_state_counts().await.is_ok()); +} + +use crate::{LongRunningTaskProof, OnDemandProofCompletion, OnDemandProofTable, WatchtowerProof}; +fn on_demand_completion() -> OnDemandProofCompletion { + OnDemandProofCompletion { + path_to_proof: "proofs/1.bin".to_string(), + public_value_hex: "00".to_string(), + proof_size: 1, + cycles: 2, + proof_state: crate::ProofState::Proven.to_i64(), + total_time_to_proof: 3, + proving_time: 4, + zkm_version: "v2".to_string(), + } +} + +async fn new_operator_proof(s: &mut StorageProcessor<'_>) -> OperatorProof { + let proof = OperatorProof { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + proof_state: crate::ProofState::New.to_i64(), + ..Default::default() + }; + assert_eq!(s.create_operator_proof(&proof).await.unwrap(), 1); + s.find_next_operator_proof().await.unwrap().unwrap() +} + +#[tokio::test] +async fn test_complete_on_demand_proof_only_while_new() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let task = new_operator_proof(&mut s).await; + let table = OnDemandProofTable::Operator; + + let completion = on_demand_completion(); + assert_eq!(s.complete_on_demand_proof_if_new(table, task.id, &completion).await.unwrap(), 1); + assert_eq!( + s.get_on_demand_proof_state(table, task.id).await.unwrap(), + Some(crate::ProofState::Proven.to_i64()) + ); + // A finished task is never rewritten, including by a later timeout. + assert_eq!(s.complete_on_demand_proof_if_new(table, task.id, &completion).await.unwrap(), 0); + let timed_out = s + .update_operator_proof_state_with_instance_graph( + &task.instance_id, + &task.graph_id, + crate::ProofState::New.to_i64(), + crate::ProofState::Failed.to_i64(), + ) + .await + .unwrap(); + assert_eq!(timed_out, 0); + assert_eq!(s.get_on_demand_proof_state(table, i64::MAX).await.unwrap(), None); +} + +#[tokio::test] +async fn test_timed_out_on_demand_proof_is_not_completed() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let task = new_operator_proof(&mut s).await; + s.update_operator_proof_state_with_instance_graph( + &task.instance_id, + &task.graph_id, + crate::ProofState::New.to_i64(), + crate::ProofState::Failed.to_i64(), + ) + .await + .unwrap(); + + let table = OnDemandProofTable::Operator; + let completion = on_demand_completion(); + assert_eq!(s.complete_on_demand_proof_if_new(table, task.id, &completion).await.unwrap(), 0); + assert_eq!( + s.get_on_demand_proof_state(table, task.id).await.unwrap(), + Some(crate::ProofState::Failed.to_i64()) + ); + + let watchtower = WatchtowerProof { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + public_key: "pk".to_string(), + proof_state: crate::ProofState::New.to_i64(), + ..Default::default() + }; + s.create_watchtower_proof(&watchtower).await.unwrap(); + let watchtower_task = s.find_next_watchtower_proof().await.unwrap().unwrap(); + s.update_watchtower_proof_state_with_instance_graph_pubkey( + &watchtower.instance_id, + &watchtower.graph_id, + "pk", + crate::ProofState::New.to_i64(), + crate::ProofState::Failed.to_i64(), + ) + .await + .unwrap(); + let table = OnDemandProofTable::Watchtower; + assert_eq!( + s.complete_on_demand_proof_if_new(table, watchtower_task.id, &completion).await.unwrap(), + 0 + ); +} + +#[tokio::test] +async fn test_count_pending_long_running_task_proofs() { + let db = setup_db().await; + let mut s = db.acquire().await.unwrap(); + let chain = "state-chain".to_string(); + for (start, state) in [ + (5, crate::ProofState::New), + (10, crate::ProofState::Proven), + (20, crate::ProofState::New), + (30, crate::ProofState::Proving), + (40, crate::ProofState::Failed), + ] { + s.create_long_running_task_proof(&LongRunningTaskProof { + block_start: start, + block_end: start + 10, + chain_name: chain.clone(), + proof_state: state.to_i64(), + ..Default::default() + }) + .await + .unwrap(); + } + assert_eq!(s.count_pending_long_running_task_proofs(&chain, 20).await.unwrap(), 2); + assert_eq!(s.count_pending_long_running_task_proofs(&chain, 0).await.unwrap(), 3); + assert_eq!(s.count_pending_long_running_task_proofs("header-chain", 0).await.unwrap(), 0); +} diff --git a/crates/store/src/schema.rs b/crates/store/src/schema.rs index df464f5e0..b5a9f87d5 100644 --- a/crates/store/src/schema.rs +++ b/crates/store/src/schema.rs @@ -196,25 +196,59 @@ pub struct CommitteeSignatures { pub l2_sig: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Display, EnumString)] +/// Bridge-in lifecycle status. +/// Failure states do not return to success; Timeout may transition to UserCanceled. +/// Transitions are based on chain facts and block heights. +#[derive( + Copy, Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq, Hash, Display, EnumString, +)] pub enum InstanceBridgeInStatus { #[default] - UserIniting, // front has call contract, but node not get event UserInited, // from contract event request // committee won't answer if userRequest is invalid(e.g. insufficient fee) CommitteesAnswered, // enough committee responsed & window expired UserBroadcastPeginPrepare, // user pegin prepare Presigned, // enough graphs completed committee pre-signing - PresignedFailed, // required graph pre-signing did not finish in time - RelayerL1Broadcasted, // PeginConfirm broadcast by relayer + RelayerL1Broadcasted, // PeginConfirm confirmed on Bitcoin RelayerL2Minted, // success - RelayerL2MintedFailed, - Timeout, // time to cancle bridgein - UserCanceled, // user broadcast Pegin-cancel tx + /// Abandoned: `h >= prepare height + T0 + buffer` and Peg-In-Confirm is + /// neither confirmed nor visible in the mempool. Failure terminal; the + /// committee key is kept for manual handling. + Timeout, + UserCanceled, // Peg-In-Cancel confirmed NoEnoughCommitteesAnswered, // no enough committee responsed & window expired UserDiscarded, // pegin prepare tx input uxto been spent in other tx } +impl InstanceBridgeInStatus { + /// States that stop normal bridge-in progression. + pub const FAILED: [InstanceBridgeInStatus; 4] = [ + InstanceBridgeInStatus::NoEnoughCommitteesAnswered, + InstanceBridgeInStatus::UserDiscarded, + InstanceBridgeInStatus::UserCanceled, + InstanceBridgeInStatus::Timeout, + ]; + + pub fn is_failed(self) -> bool { + Self::FAILED.contains(&self) + } + + pub fn is_success(self) -> bool { + self == InstanceBridgeInStatus::RelayerL2Minted + } + + /// Entering this failure state deletes the committee instance key in the + /// same transaction. `Timeout` keeps the key for manual handling. + pub fn deletes_key_on_entry(self) -> bool { + matches!( + self, + InstanceBridgeInStatus::NoEnoughCommitteesAnswered + | InstanceBridgeInStatus::UserDiscarded + | InstanceBridgeInStatus::UserCanceled + ) + } +} + /// Lifecycle of a swap-based bridge-out escrow. /// /// `Initialize`, `Claim`, and `Refund` mirror the swap contract events; @@ -287,6 +321,9 @@ pub struct Instance { pub pegin_data_tx_hash: String, pub parameters: Option, pub post_pegin_txhash: Option, + /// The committee instance key envelope was deleted; a rebuilt projection must not + /// regenerate it. + pub key_deleted: bool, pub status_updated_at: i64, pub created_at: i64, pub updated_at: i64, @@ -528,150 +565,6 @@ pub struct GraphBtcTxVoutMonitor { pub updated_at: i64, } -#[derive(Clone, Debug, Display, EnumString)] -pub enum MessageState { - Pending, - /// Claimed by a worker and currently being dispatched. A row left in this - /// state past its `lock_time_until` means the attempt never finished, which - /// is charged as an abandon rather than a retry. - Processing, - Processed, - Failed, - /// Repeated claims expired without reporting an outcome. Kept separate - /// from deterministic handler failures so operators can inspect/requeue it. - Quarantined, - Expired, - Cancelled, -} - -#[derive(Clone, FromRow, Debug, Serialize, Deserialize, Default)] -pub struct Message { - pub message_id: String, - pub business_id: Uuid, // graph_id or instance_id - pub actor: String, - pub from_peer: String, - pub msg_type: String, - pub content: Vec, - pub state: String, - pub message_version: i64, - pub weight: i64, - pub lock_time_until: i64, - /// Dispatch attempts that ended in a handler `Err` and were rescheduled. - /// Observability only; this counter never retires a message. - pub attempt_count: i64, - /// Claims whose previous attempt never reported an outcome, i.e. the worker - /// panicked or the process died mid-dispatch. Incremented when an expired - /// `Processing` lease is reclaimed. - pub abandon_count: i64, - pub last_error: Option, - pub created_at: i64, -} - -/// A durable copy of a message received from the P2P network. -/// -/// Unlike `Message`, which is used for locally generated compensation work, -/// this row retains the original sender and is consumed before dispatching the -/// external message. Processed/failed content is cleared; quarantined content is -/// retained temporarily so an operator can inspect or manually requeue it. -#[derive(Clone, FromRow, Debug, Serialize, Deserialize, Default)] -pub struct P2pInboxMessage { - pub message_id: String, - pub business_id: Option, - pub actor: String, - pub from_peer: String, - pub msg_type: String, - pub content: Vec, - pub content_size: i64, - pub state: String, - pub attempt_count: i64, - /// Claims whose attempt never reported an outcome. See `Message::abandon_count`. - pub abandon_count: i64, - pub next_retry_at: i64, - pub lease_until: i64, - pub lease_token: String, - pub last_error: Option, - /// How the sender was classified when the row was admitted. See - /// [`P2pInboxAdmissionClass`]; empty means the caller did not classify it. - pub admission_class: String, - /// Digest of `content`, used to collapse an identical payload re-published - /// by the same sender while the first copy is still queued. - pub content_hash: Option>, - pub created_at: i64, - pub updated_at: i64, -} - -/// Sender classification recorded on an inbox row for quota accounting. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Display, EnumString)] -pub enum P2pInboxAdmissionClass { - /// Committee member registered by peer ID on chain. - Committee, - /// A verifier (on-chain peer id registry) or an operator (staked master key - /// bound to the peer id by a NodeInfo binding proof). - Registered, - /// Anyone else, including anonymous peers. - Unregistered, -} - -impl P2pInboxAdmissionClass { - /// Whether the sender's identity is backed by an on-chain registration. - pub const fn is_registered(self) -> bool { - !matches!(self, Self::Unregistered) - } -} - -/// Queued (`Pending` + `Processing` + `Quarantined`) inbox usage for one -/// admission class, with the share attributable to the sender being admitted. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct P2pInboxClassUsage { - pub admission_class: String, - pub rows: i64, - pub bytes: i64, - pub peer_rows: i64, - pub peer_bytes: i64, -} - -#[derive(Clone, FromRow, Debug, Serialize, Deserialize, Default)] -pub struct P2pOutboxMessage { - pub message_id: String, - pub msg_type: String, - pub content: Vec, - pub state: String, - pub attempt_count: i64, - pub next_retry_at: i64, - pub lease_until: i64, - pub last_error: Option, - pub retry_until: i64, - pub retry_interval_secs: i64, - pub ack_peer_id: String, - /// Publishes that went through, unlike `attempt_count`, which counts claims. - pub publish_count: i64, - pub created_at: i64, -} - -#[derive(Clone, Debug, FromRow)] -pub struct MessageDebugOverview { - pub message_id: String, - pub actor: String, - pub msg_type: String, - pub state: String, - pub lock_time_until: i64, - pub created_at: i64, - pub updated_at: i64, - pub reason_count: i64, - pub last_reason_code: Option, - pub last_reason_detail: Option, - pub last_reason_seen_at: Option, -} - -#[derive(Clone, Debug, FromRow)] -pub struct MessageDebugReason { - pub reason_code: String, - pub reason_detail: String, - pub first_seen_at: i64, - pub last_seen_at: i64, - pub occurrences: i64, -} - #[derive(Clone, Debug, FromRow, PartialEq, Eq)] pub struct MetricsStateCount { pub category: String, @@ -1019,6 +912,31 @@ mod tests { InstanceBridgeInStatus::RelayerL2Minted ); assert!(InstanceBridgeInStatus::from_str("Invalid").is_err()); + assert!(InstanceBridgeInStatus::from_str("PresignedFailed").is_err()); + assert!(InstanceBridgeInStatus::from_str("RelayerL2MintedFailed").is_err()); + } + + #[test] + fn test_bridge_in_failure_set() { + use InstanceBridgeInStatus::*; + for status in InstanceBridgeInStatus::FAILED { + assert!(status.is_failed()); + assert!(!status.is_success()); + } + assert!(RelayerL2Minted.is_success() && !RelayerL2Minted.is_failed()); + for status in [ + UserInited, + CommitteesAnswered, + UserBroadcastPeginPrepare, + Presigned, + RelayerL1Broadcasted, + ] { + assert!(!status.is_failed() && !status.is_success()); + } + assert!(!Timeout.deletes_key_on_entry()); + assert!(UserCanceled.deletes_key_on_entry()); + assert!(UserDiscarded.deletes_key_on_entry()); + assert!(NoEnoughCommitteesAnswered.deletes_key_on_entry()); } #[test] diff --git a/crates/store/src/signing.rs b/crates/store/src/signing.rs new file mode 100644 index 000000000..11e75968b --- /dev/null +++ b/crates/store/src/signing.rs @@ -0,0 +1,78 @@ +//! Bind each signing nonce to one transcript before producing its signature. + +use crate::localdb::StorageProcessor; +use sqlx::Row; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SigningAuthorization { + /// First authorization of this nonce for this transcript. + Authorized, + /// The same nonce was already authorized for the same transcript; a retry + /// may sign it again. + Repeated, + /// The nonce is bound to another transcript: never sign. + Conflict { existing_digest: String }, +} + +impl<'a> StorageProcessor<'a> { + /// Authorize `nonce_id` for `transcript_digest` under `scope_kind:scope_id`. + pub async fn authorize_signing( + &mut self, + scope_kind: &str, + scope_id: &str, + nonce_id: &str, + transcript_digest: &str, + now: i64, + ) -> anyhow::Result { + let inserted = sqlx::query( + "INSERT OR IGNORE INTO signing_authorization \ + (nonce_id, scope_kind, scope_id, transcript_digest, created_at) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(nonce_id) + .bind(scope_kind) + .bind(scope_id) + .bind(transcript_digest) + .bind(now) + .execute(self.conn()) + .await? + .rows_affected() + > 0; + if inserted { + return Ok(SigningAuthorization::Authorized); + } + let existing: String = + sqlx::query("SELECT transcript_digest FROM signing_authorization WHERE nonce_id = ?") + .bind(nonce_id) + .fetch_one(self.conn()) + .await? + .try_get("transcript_digest")?; + if existing == transcript_digest { + Ok(SigningAuthorization::Repeated) + } else { + Ok(SigningAuthorization::Conflict { existing_digest: existing }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::localdb::create_local_db; + + #[tokio::test] + async fn a_nonce_binds_to_one_transcript() { + let db = create_local_db("sqlite::memory:").await; + let mut storage = db.acquire().await.unwrap(); + assert_eq!( + storage.authorize_signing("graph", "g1", "n1", "t1", 1).await.unwrap(), + SigningAuthorization::Authorized + ); + assert_eq!( + storage.authorize_signing("graph", "g1", "n1", "t1", 2).await.unwrap(), + SigningAuthorization::Repeated + ); + let conflict = storage.authorize_signing("graph", "g1", "n1", "t2", 3).await.unwrap(); + assert_eq!(conflict, SigningAuthorization::Conflict { existing_digest: "t1".into() }); + } +} diff --git a/deployment/README.md b/deployment/README.md index 62b453515..0f78c713a 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -81,6 +81,11 @@ on the same host. The business RPC listener does not serve metrics, so expose the dedicated listener through a private network, proxy, or access-control layer instead of binding it directly to a public interface. +For liveness and readiness probes use `GET /v1/health` on `--rpc-addr`: it only pings the local +database and otherwise answers from memory, `200` while the node is ready and `503` (with the same JSON body naming +the failing component or backend) otherwise. A node whose GOAT RPC is unreachable at start-up retries +for about a minute before exiting, so a supervisor restart during a short outage does not crash-loop. + The provided Docker Compose deployment enables the listener on port `9108` inside each container and publishes it only on the host loopback interface: diff --git a/localnet/README.md b/localnet/README.md new file mode 100644 index 000000000..edf622524 --- /dev/null +++ b/localnet/README.md @@ -0,0 +1,156 @@ +# localnet — a clone-and-run mock testnet for the node + +Five `mock-gc` nodes (committee_0, committee_1, operator_0, verifier_0, watchtower_0) on bitcoin +regtest + a local anvil + a local graph-node, every endpoint on 127.0.0.1. The garbled circuits, +soldering and proofs are deterministic mocks with configurable delays; everything on chain is +real, so peg-in, peg-out, challenge/assert/take-2 and disprove run the genuine transaction graph +(see `node/README.md`, "Mock build for local testnets"). The on-demand regression suite lives in +[e2e/](e2e/README.md). CI runs the whole suite after pushes to main / dev, see "CI" below. + +## Host requirements + +docker (with compose v2), jq, sqlite3, python3, curl, and the Rust/zkm toolchain to build the +node (`source ~/.zkm-toolchain/env`). forge runs in the pinned foundry container +(`tools/forge.sh`), graph-cli in the pinned node container (`subgraph/deploy-local.sh`); a host +`cast` is used when present and otherwise the container one (`tools/cast.sh`, slower per call). +The p2p-isolation e2e case additionally needs passwordless `sudo iptables`. + +## Quick start + +```bash +git clone --recurse-submodules && cd bitvm-node # or: git submodule update --init --recursive +make localnet-reset # builds the binaries if needed, then wipes+creates the whole localnet (2 min; 7 min when keys are generated; + the build) +make localnet-status +make e2e-smoke # pegin_happy pegout_happy challenge_take2 (~23 min) +make localnet-down # stop nodes/miner/proof builder; `localnet/scripts/down.sh infra` also stops the containers +``` + +`make localnet-up` adopts an already created localnet (containers, chains, contracts, node DBs) +without rebuilding anything; it refuses if the data dir has no keys/contracts yet (run the reset). + +## Layout: code vs state + +| | where | what | +|---|---|---| +| code | `localnet/` (this directory) | `scripts/` lifecycle + ops, `tools/` fault proxy / key generation / container wrappers, `e2e/` the suite, `infra/docker-compose.yml`, `contracts/` (git submodule, upstream contracts), `contracts-local/` (GatewayLocal + mocks + deploy), `subgraph/` (gateway subgraph sources + deploy) | +| state | `$LOCALNET_DATA` (default `localnet/.local`, git-ignored) | `bin/` installed binaries, `state/` keys, contracts.json, subgraph.json, cli envs, `nodes//` .env + sqlite DB, `data/` chain data of the containers, `logs/`, `run/` pids/overrides/tuning, `results/` e2e reports, `forge/` + `subgraph/` build artifacts | + +`scripts/localnet.env` is the only config file. Everything that must differ between two localnets +on one host derives from three variables, all taken from the environment: + +| variable | default | effect | +|---|---|---| +| `LOCALNET_DATA` | `localnet/.local` | state directory | +| `PORT_BASE` | `22000` | every port is `PORT_BASE + offset`: esplora +2, bitcoind RPC +443, ipfs +501, anvil +545, graph query/admin/index/metrics +800/+820/+830/+840, node rpc/p2p/metrics +900/+910/+920 (+ role index), proof builder +950, fault proxies +960..+979, chaos control +989 | +| `COMPOSE_PROJECT` | `bvlocal` | docker project, container names `-`, network `-net`, iptables chain `_CHAOS` | + +So a second localnet next to the first one is +`LOCALNET_DATA=/srv/bv2 PORT_BASE=23000 COMPOSE_PROJECT=bvlocal2 make localnet-reset`. +Per-data-dir overrides go in `$LOCALNET_DATA/localnet.override.env` (sourced last), e.g. +`export ANVIL_CHAIN_ID=48815` to adopt a localnet whose anvil was created before the default +moved to 48816. Runtime knobs the e2e suite flips (proof verdict, solder delay) are written to +`$LOCALNET_DATA/run/tuning.env` and picked up by `scripts/nodes.sh` at (re)start; the tracked +`localnet.env` is never edited by a script. + +## Scripts + +``` +scripts/reset.sh --yes DESTRUCTIVE: build if needed, wipe, recreate infra, keys, 101 blocks, contracts, subgraph, fund, start +scripts/up.sh adopt + start (idempotent); fails with a hint when a node DB does not fit the binary +scripts/down.sh [infra] stop nodes, proof builder, miner, fault proxy [+ docker compose stop] +scripts/status.sh containers, chain heights, subgraph sync, support processes, per-node readiness +scripts/nodes.sh start|stop|restart|status [role...] +scripts/miner.sh start|stop|status|once [n] regtest miner, 1 block / MINE_INTERVAL_SECS +scripts/proof-builder.sh start|stop|status mock-proof-builder +scripts/fund.sh BTC + gas top-ups (idempotent) +scripts/gen-node-envs.sh nodes//.env + state/cli/{user,operator,verifier}.env from state/*.json +scripts/install-bins.sh copy the release binaries into $LOCALNET_DATA/bin +scripts/chaos.sh fault injection (proxy faults, p2p cuts, freeze, container pause); `status`, `reset` +scripts/triage.sh [role] scripts/logq.sh ... scripts/trace.sh read-only diagnosis helpers (JSON logs under logs/nodes//) +build.sh cargo build -r -p bitvm-noded --features "rpc-debug-endpoints mock-gc" --bins (repo target) +``` + +Tuning: edit `scripts/localnet.env` (MOCK_GC_*_DELAY_SECS, MOCK_PROOF_*, MINE_INTERVAL_SECS, ...), +then `scripts/nodes.sh restart` (GC knobs) or `scripts/proof-builder.sh stop && start` (proof knobs). +The e2e runner applies its own faster tier (2 s GC / 5 s proof) through `run/tuning.env` for the duration of a run. +Keep challenge/eval and proof delays far below the block cadence or the honest operator loses the +WronglyChallenged race. + +## Contracts + +`contracts/` is a git submodule of the L2 contracts. It is pinned to PR #26 of +GOATNetwork/bitvm-L2-contracts ("fix GUARDIAN_CONNECTOR_VOUT & remove debug contracts", head +`3a679bf`, fetched from the fork `KSlashh/bitvm-L2-contracts`); once that PR is merged, point +`.gitmodules` back at `GOATNetwork/bitvm-L2-contracts` and bump the pin. The PR sets +`GUARDIAN_CONNECTOR_VOUT = 3`, which is what the node's kickoff topology uses (`goat` +`output_topology::kickoff::guardian_connector()`); older contract revisions parse peg-outs wrong. +No upstream source is patched. The localnet runs anvil at chain id 48816 (= the contracts' +testnet id, so `Constants.magicBytes()` accepts the regtest OP_RETURN magic the node emits). + +`contracts-local/` is a second foundry project that imports the submodule (`bitvm-l2/` remapping): + +- `GatewayLocal.sol` = upstream `GatewayUpgradeable` + two committee-only knobs: `debugMintPegBTC` + (pegBTC faucet for the peg-out cases) and `debugSetResponseWindowBlocks` (the initializer fixes + 200 goat blocks; the deploy lowers it to `RESPONSE_WINDOW_BLOCKS`, default 50). Runtime size + 23,930 B, ~650 B under the EIP-170 limit: keep additions tiny. +- `MockBitcoinSPV.sol`, `MockWGBTC.sol`: the SPV relay and the stake token that do not exist on anvil. +- `script/DeployLocal.sol`: one broadcast deploys mocks + GatewayLocal/CommitteeManagement/ + StakeManagement proxies (same order and console labels as upstream `DeployGateway.sol`). +- `deploy-local.sh`: gas funding, the script, the committee/verifier/operator registrations and + the operator stake, then writes `state/contracts.json`. + +## Subgraph + +`subgraph/gateway/` holds only sources (schema, manifest with a zero placeholder address, mapping, +ABI regenerated from the submodule build, `package-lock.json`). `subgraph/deploy-local.sh` stages +them under `$LOCALNET_DATA/subgraph/gateway`, patches address + start block from +`state/contracts.json`, and runs `npm ci` / `graph codegen` / `graph build` / `graph create` / +`graph deploy` inside the node container (node_modules is cached per data dir). + +## Images + +bitcoin/bitcoin:29, ipfs/kubo:v0.17.0, postgres:16 by tag; mempool/electrs, graph-node, foundry +(anvil + forge + cast) and node:22-alpine by digest, see `scripts/localnet.env`. To move a digest, +change it there and `scripts/reset.sh --yes`. + +## Observability + +- RPC per node (`rpc-debug-endpoints` on): `GET /v1/debug/status`, `/v1/debug/engine`, `/v1/instances`, `/v1/graphs`. +- Metrics `http://127.0.0.1:/metrics`: `bitvm_node_ready`, p2p peers/topics, `bitvm_node_engine_rows{table,state}`. +- Logs: one JSON file per boot under `logs/nodes//`, `current` points at the newest; `scripts/logq.sh`, `scripts/trace.sh`. +- DBs: `nodes//bitvm-node.db` (`sqlite3 -readonly`; UUIDs are 16-byte BLOBs, use `hex(col)`). + +## Measured (2026-10-16, fresh clone on a loaded 16-core host) + +| step | time | +|---|---| +| first `cargo build -r` of the mock-gc binaries (crates.io registry already cached) | 14 min | +| `make localnet-reset`, first time (includes gen-keys: ~1 min per role for the peer-id check) | 7 min | +| `make localnet-reset`, keys kept | 2 min | +| `make e2e-smoke` (pegin 6.5 min, pegout 3 min, challenge_take2 13 min incl. its own peg-in) | 23 min | +| `make e2e`, all 13 cases, fast delay tier (2 s GC / 5 s proof); three full passes on 2026-10-16 | 95–110 min | + +Nodes report `ready=1` two to three minutes after a reset; the suite's pre-check waits for it. + +## CI + +`.github/workflows/e2e_localnet.yml` runs on the self-hosted `bitvm2-node` runner on every push to +`main` / `dev` and on manual dispatch (Actions → "Localnet e2e" → Run workflow); the same triggers +as `ci_time_consuming.yml`. It is informational, not a required check. One run is build → +`make localnet-reset` → `make e2e` → stop, about two hours; runs queue (`concurrency: +localnet-e2e`) and a running one is never cancelled. + +The CI localnet is a third, private one: `LOCALNET_DATA=/data/actions-runner-bitvm2-node/cache/localnet-e2e/data`, +`PORT_BASE=25000`, `COMPOSE_PROJECT=bvci`, `CARGO_TARGET_DIR=.../localnet-e2e/target`. The data dir +is kept between runs (keys, solc and node_modules caches; the reset wipes chains and node DBs), and +the last 10 result dirs stay in it. The job summary shows `report.md`; the run's result dir +(`report.json`, per-case logs, `logs.tar.gz`) is uploaded as the `localnet-e2e-` artifact for +14 days. The runner user needs docker, passwordless `sudo` for the p2p case (otherwise it is +reported as SKIP), the zkm toolchain and a free 25000–25999 port range. To reproduce a CI failure +locally, run the same three make targets with the same three variables (any free `PORT_BASE`). + +## Known limits + +- Mocked verdicts: proof-validity branches test the mock policy (`MOCK_GC_PROOF_VERDICT`), not cryptography. +- A node whose DB predates a migration the binary cannot apply dies at start; `up.sh` reports it, the fix is a reset. +- All ports bind loopback; the p2p listener too (`P2P_LISTEN_IP=127.0.0.1`). diff --git a/localnet/build.sh b/localnet/build.sh new file mode 100755 index 000000000..361e0e633 --- /dev/null +++ b/localnet/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Build the mock-enabled node + helper binaries for the localnet from this checkout. +# localnet/build.sh # cargo build -r -p bitvm-noded --features "rpc-debug-endpoints mock-gc" --bins +# CARGO_TARGET_DIR defaults to the repo's own target/ (never share it with another checkout); +# scripts/install-bins.sh then copies the binaries into $LOCALNET_DATA/bin. +set -euo pipefail +source "$(cd "$(dirname "$0")" && pwd)/scripts/localnet.env" +# shellcheck disable=SC1090 +[[ -f ~/.zkm-toolchain/env ]] && source ~/.zkm-toolchain/env +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$REPO_ROOT/target}" +export SQLX_OFFLINE=true +cd "$REPO_ROOT" +echo "building in $REPO_ROOT (target $CARGO_TARGET_DIR, features ${NODE_FEATURES:-rpc-debug-endpoints mock-gc})" +exec nice -n "${BUILD_NICE:-15}" cargo build --release -j "${BUILD_JOBS:-8}" -p bitvm-noded \ + --features "${NODE_FEATURES:-rpc-debug-endpoints mock-gc}" --bins diff --git a/localnet/contracts b/localnet/contracts new file mode 160000 index 000000000..3a679bf96 --- /dev/null +++ b/localnet/contracts @@ -0,0 +1 @@ +Subproject commit 3a679bf963a6fdb11b034b87654205f34f94110b diff --git a/localnet/contracts-local/deploy-local.sh b/localnet/contracts-local/deploy-local.sh new file mode 100755 index 000000000..51ea78405 --- /dev/null +++ b/localnet/contracts-local/deploy-local.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# ============================================================================= +# deploy-local.sh -- deploy + configure the BitVM2 L2 contracts on the localnet anvil. +# +# Each run deploys a FRESH stack with one forge script (script/DeployLocal.sol, run inside the +# foundry container by tools/forge.sh): MockWGBTC, MockBitcoinSPV, GatewayLocal impl+proxy, PegBTC, +# CommitteeManagement impl+proxy, StakeManagement impl+proxy, then every registration the five +# node roles need, then writes $LOCALNET_DATA/state/contracts.json. Meant for a FRESH anvil, but a +# re-run on a live one simply deploys a second, independent stack (nonces never clash). +# +# Reads identities from state/keys.env. Faucet = anvil dev account 0 (gas only). +# Env overrides: RPC, OUT_FILE (dry-run target), STAKE_AMOUNT_WEI, LOCK_AMOUNT_WEI, GAS_TOPUP_ETHER, +# VERIFIER_NONCE, RESPONSE_WINDOW_BLOCKS +# ============================================================================= +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" + +RPC=${RPC:-$GOAT_CHAIN_URL} +STAKE_AMOUNT_WEI=${STAKE_AMOUNT_WEI:-100000000000000000} # 0.1 WGBTC (min on-chain stake is 0.06) +LOCK_AMOUNT_WEI=${LOCK_AMOUNT_WEI:-100000000000000000} # lock the whole stake +GAS_TOPUP_ETHER=${GAS_TOPUP_ETHER:-100} +VERIFIER_NONCE=${VERIFIER_NONCE:-1} +FORGE_DIR="$LOCALNET_DATA/forge" + +mkdir -p "$LOGS_DIR" "$STATE_DIR" "$FORGE_DIR" + +log(){ echo "[$(date +%H:%M:%S)] $*"; } +die(){ echo "ERROR: $*" >&2; exit 1; } +cast(){ "$CAST" "$@"; } + +[ -f "$STATE_DIR/keys.env" ] || die "missing $STATE_DIR/keys.env (run tools/gen-keys.sh first)" +set -a; source "$STATE_DIR/keys.env"; set +a + +CID=$(cast chain-id --rpc-url "$RPC") || die "cannot reach anvil at $RPC" +[ "$CID" = "$ANVIL_CHAIN_ID" ] || die "anvil chain id is $CID, expected ANVIL_CHAIN_ID=$ANVIL_CHAIN_ID" +log "anvil chain id $CID, block $(cast block-number --rpc-url "$RPC")" + +# ----------------------------------------------------------------------------- +# 1. Fund every GOAT (EVM) address with gas from the faucet (idempotent) +# ----------------------------------------------------------------------------- +fund_gas(){ # $1 = address + local addr=$1 bal; bal=$(cast balance "$addr" --rpc-url "$RPC") + if [ "$(python3 -c "print(1 if int('$bal') < 50000000000000000000 else 0)")" = "1" ]; then + cast send "$addr" --value "${GAS_TOPUP_ETHER}ether" --private-key "$ANVIL_FAUCET_KEY" --rpc-url "$RPC" >/dev/null + log "funded $addr with ${GAS_TOPUP_ETHER} ETH gas" + else + log "gas ok for $addr" + fi +} +log "== funding GOAT addresses with gas ==" +for A in "$DEPLOYER_GOAT_ADDRESS" "$C0_GOAT_ADDRESS" "$C1_GOAT_ADDRESS" "$OP0_GOAT_ADDRESS" \ + "$VF0_GOAT_ADDRESS" "$WT0_GOAT_ADDRESS" "$USER0_GOAT_ADDRESS"; do + fund_gas "$A" +done + +# ----------------------------------------------------------------------------- +# 2. Deploy the whole stack (mocks + GatewayLocal + CommitteeManagement + StakeManagement) +# ----------------------------------------------------------------------------- +PRE_DEPLOY_BLOCK=$(cast block-number --rpc-url "$RPC") +log "== forge script DeployLocal (container) ==" +DEPLOY_LOG=$LOGS_DIR/deploy-stack.log +PRIVATE_KEY=$DEPLOYER_GOAT_PRIVATE_KEY \ +COMMITTEE_0=$C0_GOAT_ADDRESS \ +COMMITTEE_1=$C1_GOAT_ADDRESS \ +WATCHTOWER_0=$WT0_BTC_PUBKEY_XONLY_0X \ + "$FORGE" script contracts-local/script/DeployLocal.sol:DeployLocal --rpc-url "$RPC" --broadcast --slow -vvv >"$DEPLOY_LOG" 2>&1 \ + || { tail -40 "$DEPLOY_LOG"; die "forge script deploy failed (log: $DEPLOY_LOG)"; } + +grep_addr(){ grep -F "$1" "$DEPLOY_LOG" | tail -1 | grep -oE '0x[0-9a-fA-F]{40}' | tail -1; } +WGBTC=$(grep_addr "MockWGBTC contract address:") +SPV=$(grep_addr "MockBitcoinSPV contract address:") +GATEWAY_IMPL=$(grep_addr "Gateway implementation contract address:") +GATEWAY=$(grep_addr "Gateway proxy contract address:") +PEGBTC=$(grep_addr "PegBTC contract address:") +COMMITTEE_IMPL=$(grep_addr "CommitteeManagement implementation contract address:") +COMMITTEE=$(grep_addr "CommitteeManagement proxy contract address:") +STAKE_IMPL=$(grep_addr "StakeManagement implementation contract address:") +STAKE=$(grep_addr "StakeManagement proxy contract address:") +for v in WGBTC SPV GATEWAY_IMPL GATEWAY PEGBTC COMMITTEE_IMPL COMMITTEE STAKE_IMPL STAKE; do + [ -n "${!v}" ] || { tail -60 "$DEPLOY_LOG"; die "could not parse $v from deploy log"; } +done +log "MockWGBTC = $WGBTC" +log "MockBitcoinSPV = $SPV (initialHeight=0, openPosting=true)" +log "GatewayLocal impl = $GATEWAY_IMPL" +log "Gateway proxy = $GATEWAY" +log "PegBTC = $PEGBTC" +log "CommitteeMgmt impl = $COMMITTEE_IMPL" +log "CommitteeManagement proxy= $COMMITTEE" +log "StakeMgmt impl = $STAKE_IMPL" +log "StakeManagement proxy = $STAKE" + +# Cross-check the gateway is fully wired (these are the exact getters the node calls at startup). +GW_CM=$(cast call "$GATEWAY" "committeeManagement()(address)" --rpc-url "$RPC") +GW_SM=$(cast call "$GATEWAY" "stakeManagement()(address)" --rpc-url "$RPC") +GW_SPV=$(cast call "$GATEWAY" "bitcoinSPV()(address)" --rpc-url "$RPC") +GW_PB=$(cast call "$GATEWAY" "pegBTC()(address)" --rpc-url "$RPC") +lc(){ echo "$1" | tr 'A-Z' 'a-z'; } +[ "$(lc "$GW_CM")" = "$(lc "$COMMITTEE")" ] || die "gateway.committeeManagement mismatch ($GW_CM)" +[ "$(lc "$GW_SM")" = "$(lc "$STAKE")" ] || die "gateway.stakeManagement mismatch ($GW_SM)" +[ "$(lc "$GW_SPV")" = "$(lc "$SPV")" ] || die "gateway.bitcoinSPV mismatch ($GW_SPV)" +[ "$(lc "$GW_PB")" = "$(lc "$PEGBTC")" ] || die "gateway.pegBTC mismatch ($GW_PB)" +log "gateway wiring verified (committee/stake/spv/pegBTC)" + +# ProxyAdmin of the gateway proxy (informational; ERC1967 admin slot) +ADMIN_SLOT=0xb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d6103 +PROXYADMIN=$(cast parse-bytes32-address "$(cast storage "$GATEWAY" "$ADMIN_SLOT" --rpc-url "$RPC")") + +# Gateway proxy creation block, from the broadcast receipts (canonical subgraph startBlock). +RUN_JSON=$FORGE_DIR/broadcast/DeployLocal.sol/$CID/run-latest.json +GATEWAY_DEPLOY_BLOCK=$(python3 - "$RUN_JSON" "$GATEWAY" <<'PY' +import sys,json +try: + run=json.load(open(sys.argv[1])) +except Exception: + sys.exit(0) +gw=sys.argv[2].lower(); tx_hash=None +for t in run.get("transactions",[]): + if (t.get("contractAddress") or "").lower()==gw: + tx_hash=(t.get("hash") or "").lower() +for r in run.get("receipts",[]): + if (r.get("transactionHash") or "").lower()==tx_hash: + print(int(r["blockNumber"],16)); break +PY +) +[ -n "$GATEWAY_DEPLOY_BLOCK" ] || GATEWAY_DEPLOY_BLOCK=$PRE_DEPLOY_BLOCK +log "gateway proxy creation block = $GATEWAY_DEPLOY_BLOCK (pre-deploy block was $PRE_DEPLOY_BLOCK)" + +# ----------------------------------------------------------------------------- +# 3. Local policy: shorter committee answer window (committee-only setter of GatewayLocal) +# ----------------------------------------------------------------------------- +cast send "$GATEWAY" "debugSetResponseWindowBlocks(uint256)" "$RESPONSE_WINDOW_BLOCKS" \ + --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +log "responseWindowBlocks = $(cast call "$GATEWAY" "responseWindowBlocks()(uint256)" --rpc-url "$RPC")" + +# ----------------------------------------------------------------------------- +# 4. Registrations +# ----------------------------------------------------------------------------- +log "== register committee peer ids ==" +cast send "$COMMITTEE" "registerPeerId(bytes)" "$C0_PEER_ID_BYTES" \ + --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +cast send "$COMMITTEE" "registerPeerId(bytes)" "$C1_PEER_ID_BYTES" \ + --private-key "$C1_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +log "registered committee_0 + committee_1 peer ids" + +log "== register verifier (2-of-2 committee signatures) ==" +VDIGEST=$(cast call "$COMMITTEE" "getAddVerifierDigestNonced(bytes,uint256)(bytes32)" \ + "$VF0_PEER_ID_BYTES" "$VERIFIER_NONCE" --rpc-url "$RPC") +VS0=$(cast wallet sign --no-hash --private-key "$C0_GOAT_PRIVATE_KEY" "$VDIGEST") +VS1=$(cast wallet sign --no-hash --private-key "$C1_GOAT_PRIVATE_KEY" "$VDIGEST") +cast send "$COMMITTEE" "addVerifier(bytes,uint256,bytes[])" \ + "$VF0_PEER_ID_BYTES" "$VERIFIER_NONCE" "[$VS0,$VS1]" \ + --private-key "$DEPLOYER_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +log "registered verifier_0 peer id (nonce $VERIFIER_NONCE)" + +log "== register operator pubkey ==" +# StakeManagement.getRegisterPubkeyDigest binds address(this)+chainid+operator; sign with the operator's +# BTC key (BITVM_SECRET). msg.sender must be OP0_GOAT_ADDRESS so pubkeyToAddress==the staker. +ODIGEST=$(cast call "$STAKE" "getRegisterPubkeyDigest(address)(bytes32)" "$OP0_GOAT_ADDRESS" --rpc-url "$RPC") +OSIG=$(cast wallet sign --no-hash --private-key "0x$OP0_BITVM_SECRET" "$ODIGEST") # 65 bytes r||s||v +OSIG64=${OSIG:0:130} # first 64 bytes r||s +cast send "$STAKE" "registerPubkey(bytes32,bytes1,bytes)" \ + "$OP0_BTC_PUBKEY_XONLY_0X" "$OP0_BTC_PUBKEY_PREFIX" "$OSIG64" \ + --private-key "$OP0_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +log "registered operator x-only pubkey -> $OP0_GOAT_ADDRESS" + +log "== operator stake + lock ==" +cast send "$WGBTC" "mint(address,uint256)" "$OP0_GOAT_ADDRESS" "$STAKE_AMOUNT_WEI" \ + --private-key "$DEPLOYER_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +cast send "$WGBTC" "approve(address,uint256)" "$STAKE" "$STAKE_AMOUNT_WEI" \ + --private-key "$OP0_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +cast send "$STAKE" "stake(uint256)" "$STAKE_AMOUNT_WEI" \ + --private-key "$OP0_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +cast send "$STAKE" "lockStake(address,uint256)" "$OP0_GOAT_ADDRESS" "$LOCK_AMOUNT_WEI" \ + --private-key "$OP0_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +log "operator staked $STAKE_AMOUNT_WEI, locked $LOCK_AMOUNT_WEI" + +log "== authorize committee_0 as SPV poster ==" +# openPosting is already true, but whitelist committee_0's GOAT address explicitly (it runs +# ENABLE_UPDATE_SPV_CONTRACT=true and posts block hashes from its GOAT_PRIVATE_KEY). +cast send "$SPV" "setPoster(address,bool)" "$C0_GOAT_ADDRESS" true \ + --private-key "$DEPLOYER_GOAT_PRIVATE_KEY" --rpc-url "$RPC" >/dev/null +log "committee_0 whitelisted on MockBitcoinSPV" + +# ----------------------------------------------------------------------------- +# 5. Write state/contracts.json +# ----------------------------------------------------------------------------- +OUT=${OUT_FILE:-$STATE_DIR/contracts.json} # OUT_FILE overrides for throwaway/dry-run deploys +python3 - "$OUT" <: committee member addresses (sequential from 0) + - WATCHTOWER_: watchtower x-only pubkeys (bytes32, sequential from 0) + The console lines are parsed by deploy-local.sh; keep their labels. +*/ +contract DeployLocal is Script { + address public deployer; + + function run() public { + uint256 deployerPrivateKey = vm.envUint("PRIVATE_KEY"); + deployer = vm.createWallet(deployerPrivateKey).addr; + console.log("deployer address: ", deployer); + vm.startBroadcast(deployerPrivateKey); + deploy(); + vm.stopBroadcast(); + } + + function deploy() public { + MockWGBTC wgbtc = new MockWGBTC(); + console.log("MockWGBTC contract address: ", address(wgbtc)); + MockBitcoinSPV spv = new MockBitcoinSPV(0, true); + console.log("MockBitcoinSPV contract address: ", address(spv)); + + GatewayLocal gatewayImpl = new GatewayLocal(); + console.log("Gateway implementation contract address: ", address(gatewayImpl)); + UpgradeableProxy gatewayProxy = new UpgradeableProxy(address(gatewayImpl), deployer, ""); + console.log("Gateway proxy contract address: ", address(gatewayProxy)); + GatewayLocal gateway = GatewayLocal(payable(gatewayProxy)); + + PegBTC pegBTC = new PegBTC(address(gateway)); + console.log("PegBTC contract address: ", address(pegBTC)); + + address[] memory initialMembers = _readSequentialAddresses("COMMITTEE"); + uint256 initialRequired = (initialMembers.length * 2 + 2) / 3; + bytes32[] memory initialWatchtowers = _readSequentialBytes32("WATCHTOWER"); + require(initialMembers.length > 0, "COMMITTEE list empty"); + require(initialWatchtowers.length > 0, "WATCHTOWER list empty"); + address[] memory initialAuthorizedCallers = new address[](1); + initialAuthorizedCallers[0] = address(gateway); + + CommitteeManagement committeeImpl = new CommitteeManagement(); + console.log("CommitteeManagement implementation contract address: ", address(committeeImpl)); + bytes memory committeeInitData = abi.encodeWithSelector( + CommitteeManagement.initialize.selector, + initialMembers, + initialRequired, + initialAuthorizedCallers, + initialWatchtowers + ); + UpgradeableProxy committeeProxy = new UpgradeableProxy(address(committeeImpl), deployer, committeeInitData); + console.log("CommitteeManagement proxy contract address: ", address(committeeProxy)); + + StakeManagement stakeImpl = new StakeManagement(); + console.log("StakeManagement implementation contract address: ", address(stakeImpl)); + bytes memory stakeInitData = + abi.encodeWithSelector(StakeManagement.initialize.selector, IERC20(address(wgbtc)), address(gateway)); + UpgradeableProxy stakeProxy = new UpgradeableProxy(address(stakeImpl), deployer, stakeInitData); + console.log("StakeManagement proxy contract address: ", address(stakeProxy)); + + gateway.initialize( + IPegBTC(address(pegBTC)), + IBitcoinSPV(address(spv)), + ICommitteeManagement(address(committeeProxy)), + IStakeManagement(address(stakeProxy)) + ); + } + + function _readSequentialAddresses(string memory baseKey) internal view returns (address[] memory out) { + uint256 count = 0; + while (vm.envOr(string(abi.encodePacked(baseKey, "_", vm.toString(count))), address(0)) != address(0)) { + count++; + } + out = new address[](count); + for (uint256 i = 0; i < count; i++) { + out[i] = vm.envAddress(string(abi.encodePacked(baseKey, "_", vm.toString(i)))); + } + } + + function _readSequentialBytes32(string memory baseKey) internal view returns (bytes32[] memory out) { + uint256 count = 0; + while (vm.envOr(string(abi.encodePacked(baseKey, "_", vm.toString(count))), bytes32(0)) != bytes32(0)) { + count++; + } + out = new bytes32[](count); + for (uint256 i = 0; i < count; i++) { + out[i] = vm.envBytes32(string(abi.encodePacked(baseKey, "_", vm.toString(i)))); + } + } +} diff --git a/localnet/contracts-local/src/GatewayLocal.sol b/localnet/contracts-local/src/GatewayLocal.sol new file mode 100644 index 000000000..4ec356204 --- /dev/null +++ b/localnet/contracts-local/src/GatewayLocal.sol @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.28; + +import {GatewayUpgradeable} from "bitvm-l2/Gateway.sol"; + +/// @title GatewayLocal +/// @notice The upstream Gateway plus the two committee-only knobs the localnet needs. Nothing else +/// differs, so the node talks to the genuine Gateway code paths. +/// - debugMintPegBTC: pegBTC faucet (the e2e peg-out cases need the operator to hold pegBTC +/// without running a peg-in first). +/// - debugSetResponseWindowBlocks: shrink the committee answer window the initializer fixes +/// at 200 goat blocks (~400 s on 2 s anvil blocks, during which peg-ins stall). +contract GatewayLocal is GatewayUpgradeable { + function debugMintPegBTC(address to, uint256 amount) external onlyCommittee { + pegBTC.mint(to, amount); + } + + function debugSetResponseWindowBlocks(uint256 blocks_) external onlyCommittee { + responseWindowBlocks = blocks_; + } +} diff --git a/localnet/contracts-local/src/MockBitcoinSPV.sol b/localnet/contracts-local/src/MockBitcoinSPV.sol new file mode 100644 index 000000000..f65607763 --- /dev/null +++ b/localnet/contracts-local/src/MockBitcoinSPV.sol @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.28; + +import {IBitcoinSPV} from "bitvm-l2/interfaces/IBitcoinSPV.sol"; + +/// @title MockBitcoinSPV +/// @notice Local-only stand-in for the GOAT BitcoinSPV relay used by the BitVM2 localnet. +/// Implements exactly the four functions the node calls +/// (goat_adaptor.rs): +/// - blockHash(uint256) view +/// - latestHeight() view +/// - postBlockHash(uint256,bytes32) +/// - postBlockHashBatch(uint256[],bytes32[]) +/// plus IBitcoinSPV (blockHash + latestHeight), so it can be wired into the Gateway as the +/// `bitcoinSPV` dependency and satisfy Gateway._verifyMerkleInclusion. +/// +/// Header hashes are stored exactly as posted. The node posts BlockHash::to_byte_array(), i.e. +/// internal (little-endian) byte order, which is the same order Gateway._verifyMerkleInclusion +/// compares against (sha256d(rawHeader)). Do NOT byte-reverse anything if the node is the only +/// poster (spv_maintenance_tasks.rs, ENABLE_UPDATE_SPV_CONTRACT). +/// +/// Posting model: `openPosting` (constructor arg) lets anybody post; otherwise only `owner` and +/// addresses whitelisted via setPoster() may post. The node's SPV poster is committee_0's +/// GOAT_PRIVATE_KEY address; deploy-local.sh both enables openPosting and whitelists it. +/// +/// batch/postBlockHash are permissive on ordering: the node always posts contiguously from +/// latestHeight()+1 via the batch call, but it also calls the single postBlockHash() to OVERWRITE +/// an already-posted height on a regtest rollback (goat_chain/mod.rs:456-464), so this mock must +/// accept arbitrary-height writes. latestHeight only ever advances to the highest height seen. +contract MockBitcoinSPV is IBitcoinSPV { + address public owner; + bool public openPosting; + uint256 public override latestHeight; + + mapping(address => bool) public posters; + mapping(uint256 => bytes32) private hashes; + + event BlockHashPosted(uint256 indexed height, bytes32 headerHash); + event PosterSet(address indexed poster, bool allowed); + event OpenPostingSet(bool open); + + error NotOwner(); + error NotAuthorizedPoster(); + error LengthMismatch(); + error EmptyBatch(); + + modifier onlyOwner() { + if (msg.sender != owner) revert NotOwner(); + _; + } + + modifier onlyPoster() { + if (!openPosting && msg.sender != owner && !posters[msg.sender]) revert NotAuthorizedPoster(); + _; + } + + /// @param _initialHeight starting latestHeight (0 lets the node backfill from height 1) + /// @param _openPosting true => anybody may post (convenient for a single-poster localnet) + constructor(uint256 _initialHeight, bool _openPosting) { + owner = msg.sender; + latestHeight = _initialHeight; + openPosting = _openPosting; + posters[msg.sender] = true; + } + + // ---- IBitcoinSPV views ---- + function blockHash(uint256 height) external view override returns (bytes32) { + return hashes[height]; + } + + // latestHeight() is the auto-generated getter of the `override` public state variable above, + // which satisfies IBitcoinSPV.latestHeight(). + + // ---- posting ---- + function postBlockHash(uint256 height, bytes32 headerHash) external onlyPoster { + _post(height, headerHash); + } + + function postBlockHashBatch(uint256[] calldata heights, bytes32[] calldata headerHashes) external onlyPoster { + if (heights.length != headerHashes.length) revert LengthMismatch(); + if (heights.length == 0) revert EmptyBatch(); + for (uint256 i = 0; i < heights.length; i++) { + _post(heights[i], headerHashes[i]); + } + } + + function _post(uint256 height, bytes32 headerHash) internal { + hashes[height] = headerHash; + if (height > latestHeight) { + latestHeight = height; + } + emit BlockHashPosted(height, headerHash); + } + + // ---- owner helpers (regtest reorgs / setup) ---- + function setPoster(address poster, bool allowed) external onlyOwner { + posters[poster] = allowed; + emit PosterSet(poster, allowed); + } + + function setOpenPosting(bool open) external onlyOwner { + openPosting = open; + emit OpenPostingSet(open); + } + + /// @notice Force-set a height's hash and/or latestHeight without ordering checks (setup/reorg escape hatch). + function forceSet(uint256 height, bytes32 headerHash) external onlyOwner { + hashes[height] = headerHash; + emit BlockHashPosted(height, headerHash); + } + + function setLatestHeight(uint256 height) external onlyOwner { + latestHeight = height; + } +} diff --git a/localnet/contracts-local/src/MockWGBTC.sol b/localnet/contracts-local/src/MockWGBTC.sol new file mode 100644 index 000000000..f69f9036b --- /dev/null +++ b/localnet/contracts-local/src/MockWGBTC.sol @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.28; + +import {ERC20} from "@openzeppelin/contracts/token/ERC20/ERC20.sol"; + +/// @title MockWGBTC +/// @notice Local-only WETH9-style ERC20 stake token used by StakeManagement on the localnet. +/// Stands in for GOAT's "Wrapped Goat Bitcoin" (WGBTC) predeploy +/// (0xbC10000000000000000000000000000000000000), which does not exist on anvil. +/// StakeManagement only uses ERC20 transfer/transferFrom/approve/balanceOf; the operator obtains +/// tokens via deposit()/mint() then approves + stakes + locks (see deploy-local.sh). +/// +/// 18 decimals (matches the on-chain policy amounts: minStakeAmount = 6e16 wei = 0.06 WGBTC). +contract MockWGBTC is ERC20 { + event Deposit(address indexed dst, uint256 wad); + event Withdrawal(address indexed src, uint256 wad); + + constructor() ERC20("Wrapped Goat Bitcoin (mock)", "WGBTC") {} + + /// @notice Wrap native anvil ETH 1:1 into WGBTC (WETH9-style). + function deposit() external payable { + _mint(msg.sender, msg.value); + emit Deposit(msg.sender, msg.value); + } + + /// @notice Unwrap WGBTC back to native ETH 1:1. + function withdraw(uint256 wad) external { + _burn(msg.sender, wad); + (bool ok,) = msg.sender.call{value: wad}(""); + require(ok, "WGBTC: ETH transfer failed"); + emit Withdrawal(msg.sender, wad); + } + + /// @notice Open faucet mint (no native ETH backing) — convenience for funding operator stake locally. + function mint(address to, uint256 amount) external { + _mint(to, amount); + } + + receive() external payable { + _mint(msg.sender, msg.value); + emit Deposit(msg.sender, msg.value); + } +} diff --git a/localnet/e2e/README.md b/localnet/e2e/README.md new file mode 100644 index 000000000..030b840f1 --- /dev/null +++ b/localnet/e2e/README.md @@ -0,0 +1,55 @@ +# e2e — on-demand end-to-end regression of the node + +Not CI: run it by hand after a change, on the localnet of [../README.md](../README.md) (five mocked nodes on +regtest + anvil + local graph). Honours `LOCALNET_DATA` / `PORT_BASE` / `COMPOSE_PROJECT` like every other script. + +``` +localnet/scripts/status.sh # infra, chains, nodes all up +localnet/e2e/run.sh --list # the cases, in run order +localnet/e2e/run.sh # all of them (95–110 min), report in $LOCALNET_DATA/results/latest/report.md +localnet/e2e/run.sh pegout_happy challenge_take2 # a subset; make e2e-smoke = pegin_happy pegout_happy challenge_take2 +localnet/e2e/run.sh --stop-on-fail # stop at the first failing case +``` + +Knobs: `E2E_CASE_TIMEOUT` (seconds per case, default 1200; an expired case is killed with its whole process tree, +chaos is reset and every node restarted before the post-check), `E2E_DELAY_TIER=fast|default` (default `fast`: the run +writes 2 s GC / 5 s proof delays to `$LOCALNET_DATA/run/tuning.env`, restarts nodes + proof builder, and restores the +`localnet.env` values (15/30 s, the manual-debugging tier) at the end), `E2E_POLL_SECS` (default 2). +Per run `$LOCALNET_DATA/results//` holds `.log`, `report.md`, `report.json` (same table as structured +data: result, duration, rc, post-check rc, notes) and `logs.tar.gz` (every node / miner / proof-builder / fault-proxy +log written during the run). Exit code 0 only when nothing failed; `p2p_isolate_operator` is reported as SKIP (not a +failure) when the host has no passwordless `sudo` for iptables. + +To test a new node commit: `localnet/build.sh`, `localnet/scripts/install-bins.sh`, `localnet/scripts/nodes.sh restart`, then `localnet/e2e/run.sh`. + +| case | what it drives | pass = | +|---|---|---| +| `pegin_happy` | request → answers → Prepare → ceremony → Peg-In-Confirm → L2 mint → graph posted | instance RelayerL2Minted, graph L2Recorded on all nodes | +| `pegout_happy` | initWithdraw → kickoff → proceedWithdraw → Take-1 → finishWithdrawHappyPath | OperatorTake1 on all nodes, finish sent | +| `challenge_take2` | peg-out challenged by the verifier, assert found valid → Take-2 | OperatorTake2, finishWithdrawUnhappyPath | +| `dispute_disprove` | same with `MOCK_GC_PROOF_VERDICT=always-invalid` on operator + verifier → ChallengeAssert → Disprove | Disprove on all nodes, finishWithdrawDisproved; verdict restored to `valid` afterwards | +| `backtoback` | two peg-ins with a 120 s verifier solder delay; B is prepared while A's ceremony runs, and the verifier queues B's setup behind A's solder (one GC job per node at a time, `ENGINE_CPU_POOL_SLOTS` default 1) | A still in its ceremony on every node when B's Prepare confirms; B's setup job created before A's solder finished; both L2Recorded; consecutive slots | +| `pegin_restart_*` | peg-in with committee_0 / operator_0 / verifier_0 restarted the moment one of the instance's jobs is Running on it (verifier: 20 s solder for the case) | a job was caught running (verifier: its solder); it ends Done, re-run by the new process or finished during the old one's graceful shutdown; peg-in completes; one graph and one Peg-In-Confirm txid | +| `goat_timeout_at_kickoff` | Goat RPC answers 25 s slower than the 20 s timeout for 60 s at the kickoff block | peg-out completes after the window | +| `subgraph_latency` | 8 s on every subgraph query for a whole peg-in | this case's peg-in completes | +| `proof_builder_blackhole` | proof-builder requests accepted but never answered for 240 s during Prover-Assert | at least one operator request hit the black hole; Assert then Take-2 | +| `p2p_isolate_operator` | operator_0's p2p links cut for 90 s during the ceremony (iptables, needs sudo; SKIP without) | 0 peers half way through; both committee peers back within 60 s of the heal; peg-in completes | +| `boot_with_goat_down` | watchtower_0 restarted with its Goat / esplora / subgraph proxy down | the same process survives each outage and is ready within 240 s after it; a death fails the case (the node is started again for the later cases) | + +Every case starts with an empty scratch dir (`$LOCALNET_DATA/run/e2e`, nothing carries over from an earlier case) +and a pre-check: a snapshot of every node's Failed job ids, nodes ready, no chaos injection left, and the operator's +locked stake at or above the gateway minimum (a Disprove slashes it; an operator under the minimum is silently +ignored by every peer, so the pre-check tops it up with the mock WGBTC faucet). Every case ends with a post-check: +`/v1/debug/engine` has no Failed / Poisoned inbox rows, no waiting / poisoned aggregates and no Running / Poisoned +jobs on any node, no Failed job exists that was not in the snapshot (and the debug endpoint reports no more Failed +jobs than the snapshot holds; a failed snapshot or DB query fails the check instead of counting as "none"), `scripts/chaos.sh status` is empty, the miner is +restarted. `run/tuning.env` is saved before each case and restored after it whatever happened (pass, fail, timeout, +Ctrl-C); when the restore had to change something the nodes are restarted and the report notes it. Scenario +scripts exit with their own verdict and the runner uses it; an unknown case name is an error (exit 2). The fault cases attach every node to `tools/faultproxy.py` the first time they run +(`scripts/chaos.sh attach all`, a restart of every node) and leave them attached; `scripts/chaos.sh detach all` undoes it. + +Layout: `lib.sh` helpers (status polling, log waits, DB queries, chaos helpers), `flows/` the protocol drives, +`scenarios/` the fault injections around a flow; `$LOCALNET_DATA/results//` one log per case plus `report.md`, +`results/latest`. CLI identities come from `$LOCALNET_DATA/state/cli/{user,operator,verifier}.env` (test-only keys, +generated by `scripts/gen-node-envs.sh`); peg-ins consume one wallet UTXO each. Cases that flip a mock knob +(`dispute_disprove`, `backtoback`) do it through `$LOCALNET_DATA/run/tuning.env` and restore it afterwards. diff --git a/localnet/e2e/flows/backtoback.sh b/localnet/e2e/flows/backtoback.sh new file mode 100755 index 000000000..4a74a2885 --- /dev/null +++ b/localnet/e2e/flows/backtoback.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Two peg-ins back to back with a slow verifier soldering (MOCK_GC_SOLDER_DELAY_SECS=120). B is prepared while A's +# ceremony still runs; the engine's cpu pool runs one GC job per node at a time (ENGINE_CPU_POOL_SLOTS, default 1), +# so the verifier queues B's setup behind A's solder. Asserted: A is still in its ceremony on every node when B's +# Prepare confirms, the verifier created B's setup job before A's solder finished, both end L2Minted / L2Recorded, +# and their graphs take consecutive slots. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +SOLDER_BEFORE=$(tuning_get MOCK_GC_SOLDER_DELAY_SECS); set_tuning MOCK_GC_SOLDER_DELAY_SECS 120 +NODES_SH restart verifier_0 >/dev/null 2>&1; log "verifier_0 restarted with a 120 s solder delay" +sleep 8 +req() { local out id; out=$( set -a; source "$CLI_ENV/user.env"; set +a; "$BIN_DIR/pegin-request" --network regtest --esplora-url "$BTC_CHAIN_URL" request --pegin-amount-sats 1000000 --receiver-evm-address "$OP0_GOAT_ADDRESS" 2>&1 ) || { echo "$out" >&2; return 1; } + grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' <<<"$out" | head -1; } +prep() { ( set -a; source "$CLI_ENV/user.env"; set +a; "$BIN_DIR/pegin-request" --network regtest --esplora-url "$BTC_CHAIN_URL" prepare --instance-id "$1" 2>&1 ) | tail -1; } +A=$(req) || exit 1; log "instance A $A" +wait_all inst "$A" CommitteesAnswered 600 || exit 1 +prep "$A"; wait_mempool 60 || exit 1; mine 1 +wait_all inst "$A" UserBroadcastPeginPrepare 240 || exit 1 +B=$(req) || exit 1; log "instance B $B (requested while A's setup runs)" +wait_all inst "$B" CommitteesAnswered 600 || exit 1 +prep "$B"; wait_mempool 60 || exit 1; mine 1 +wait_all inst "$B" UserBroadcastPeginPrepare 240 || exit 1 +sA=$(all_inst "$A"); log "A: $sA" +[[ $(grep -o "=UserBroadcastPeginPrepare " <<<"$sA " | wc -l) -eq ${#NODES[@]} ]] || { log "FAIL: A left its ceremony before B's Prepare confirmed, no overlap: $sA"; exit 1; } +log "operator sessions: $(db operator_0 "select substr(hex(instance_id),1,8)||':'||substr(hex(graph_id),1,8) from pending_graph_init" | tr '\n' ' ')" +log "operator build jobs: $(db operator_0 "select substr(aggregate_id,1,8)||':'||action||':'||state from engine_job where action in ('build_graph','ingest_soldering_proof','verifier_setup') and state in ('Pending','Running')" | tr '\n' ' ')" +# each instance: wait for the relayer's Peg-In-Confirm to be broadcast, then confirm it with a block (the fast +# poll otherwise mines before B's confirm reaches the mempool and B stays Presigned with the miner stopped) +confirm() { wait_log committee_0 "Peg-In-Confirm broadcast.*$1\|$1.*Peg-In-Confirm broadcast" 300 || return 1; wait_mempool 120 || return 1; mine 1; } +wait_all inst "$A" Presigned 1500 || { engine_summary; exit 1; } +confirm "$A" || exit 1 +wait_all inst "$B" Presigned 1500 || { engine_summary; exit 1; } +confirm "$B" || exit 1 +wait_all inst "$A" RelayerL1Broadcasted 240 || exit 1 +wait_all inst "$B" RelayerL1Broadcasted 240 || exit 1 +mine 1 +wait_all inst "$A" RelayerL2Minted 600 || exit 1 +wait_all inst "$B" RelayerL2Minted 600 || exit 1 +gA=$(graph_of_instance "$A" | awk '{print $1}' | head -1); gB=$(graph_of_instance "$B" | awk '{print $1}' | head -1) +log "graphs: A $(graph_of_instance "$A" | tr '\n' ';') B $(graph_of_instance "$B" | tr '\n' ';')" +wait_all graph "$gA" L2Recorded 300 || exit 1 +wait_all graph "$gB" L2Recorded 300 || exit 1 +echo "$gB" > "$E2E_RUN/graph.txt" +kA=$(graph_of_instance "$A" | awk '{print $2}' | head -1); kB=$(graph_of_instance "$B" | awk '{print $2}' | head -1) +bset=$(db verifier_0 "select created_at from engine_job where aggregate_id='$gB' and action='verifier_setup' order by created_at limit 1") +asol=$(db verifier_0 "select finished_at from engine_job where aggregate_id='$gA' and action='verifier_solder' order by created_at limit 1") +log "slots A=$kA B=$kB; verifier: B setup queued at $bset, A solder finished at $asol" +rc=0 +[[ -n $kA && -n $kB && $kB -eq $((kA + 1)) ]] || { log "FAIL: graphs not in consecutive slots (A=$kA B=$kB)"; rc=1; } +[[ -n $bset && -n $asol && $bset -lt $asol ]] || { log "FAIL: the verifier did not take B's setup while A's solder ran (B setup $bset, A solder done $asol)"; rc=1; } +set_tuning MOCK_GC_SOLDER_DELAY_SECS "$SOLDER_BEFORE" +NODES_SH restart verifier_0 >/dev/null 2>&1; log "verifier_0 restarted with the ${SOLDER_BEFORE} s solder delay" +engine_summary +(( rc == 0 )) || exit 1 +log "BACK-TO-BACK DONE A=$A/$gA B=$B/$gB tip=$(tip)" diff --git a/localnet/e2e/flows/challenge-continue.sh b/localnet/e2e/flows/challenge-continue.sh new file mode 100755 index 000000000..eff5e3575 --- /dev/null +++ b/localnet/e2e/flows/challenge-continue.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Challenge flow continuation: the withdraw is already initialized (kickoff queued or broadcast). Same steps as +# flows/challenge.sh from the kickoff broadcast on. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id}; MODE=${2:-valid} +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +wait_log operator_0 "Pre-Kickoff and Kickoff broadcast.*$gid\|$gid.*Pre-Kickoff and Kickoff broadcast" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1; K=$(tip) +wait_all graph "$gid" OperatorKickOff 180 || exit 1 +mine 1 +wait_log committee_0 "$gid.*proceedWithdraw sent\|proceedWithdraw sent.*$gid" 180 || { engine_summary; exit 1; } +log "challenging from the verifier while tip=$(tip) (< K+2=$((K+2)))" +( set -a; source "$CLI_ENV/verifier.env"; set +a; "$BIN_DIR/challenge" --rpc-url "$(rpc_url verifier_0)" --graph-id "$gid" 2>&1 | tail -2 ) +wait_mempool 60 || exit 1 +mine 1 +wait_log operator_0 "$gid.*Watchtower-Challenge-Init broadcast\|Watchtower-Challenge-Init broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1; W=$(tip); log "WCI confirmed at W=$W; holding for the watchtower proof (30 s)" +wait_log watchtower_0 "$gid.*Watchtower-Challenge broadcast\|Watchtower-Challenge broadcast.*$gid" 300 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +wait_log operator_0 "$gid.*Operator-Challenge-ACK broadcast\|Operator-Challenge-ACK broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +# commit-pubin needs the watchtower challenge buried 2 blocks (= verifier commit timeout at W+3): keep the +# verifier out of the race, as the runbook does. +"$ROOT/scripts/nodes.sh" stop verifier_0 >/dev/null 2>&1; log "verifier_0 stopped for the pubin block" +mine 1; log "tip $(tip) = W+3" +wait_log operator_0 "$gid.*Operator-Commit-Pubin broadcast\|Operator-Commit-Pubin broadcast.*$gid" 240 || { engine_summary; NODES_SH start verifier_0 >/dev/null 2>&1; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +NODES_SH start verifier_0 >/dev/null 2>&1; log "verifier_0 restarted (verdict: $(current_verdict))" +log "waiting for the operator proof (30 s) and the assert" +wait_log operator_0 "$gid.*Prover-Assert broadcast\|Prover-Assert broadcast.*$gid" 300 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1; A=$(tip); log "assert confirmed at A=$A" +if [[ $MODE == valid ]]; then + wait_log verifier_0 "operator assert proof is valid" 240 || { engine_summary; exit 1; } + for i in $(seq 1 15); do + if fresh_log operator_0 | grep -q "Take-2 broadcast"; then break; fi + sleep 20; mine 1 + done + wait_log operator_0 "$gid.*Take-2 broadcast\|Take-2 broadcast.*$gid" 120 || { engine_summary; exit 1; } + [[ -n "$(mempool)" ]] && mine 1 + wait_all graph "$gid" OperatorTake2 180 || exit 1 + mine 1 + wait_log committee_0 "$gid.*finishWithdrawUnhappyPath sent\|finishWithdrawUnhappyPath sent.*$gid" 240 || { engine_summary; exit 1; } +else + wait_log verifier_0 "ChallengeAssert broadcast\|Verifier-Assert broadcast\|verifier assert broadcast\|verifier connector spent by unconfirmed" 300 || { engine_summary; exit 1; } + wait_mempool 60 || exit 1 + mine 1; log "ChallengeAssert confirmed at $(tip); the operator cannot answer under always-invalid" + sleep 30 + for i in $(seq 1 8); do + if [[ "$(all_graph "$gid") " == *"verifier_0=Disprove "* ]]; then break; fi + mine 1; sleep 25 + done + wait_any graph "$gid" Disprove 120 || { engine_summary; exit 1; } + [[ -n "$(mempool)" ]] && mine 1 + wait_all graph "$gid" Disprove 180 || exit 1 + mine 1 + wait_log committee_0 "$gid.*finishWithdrawDisproved sent\|finishWithdrawDisproved sent.*$gid" 240 || { engine_summary; exit 1; } +fi +engine_summary +log "CHALLENGE FLOW ($MODE) DONE graph=$gid tip=$(tip)" diff --git a/localnet/e2e/flows/challenge.sh b/localnet/e2e/flows/challenge.sh new file mode 100755 index 000000000..23f59ecf2 --- /dev/null +++ b/localnet/e2e/flows/challenge.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Challenge flow on a posted graph. MODE=valid: verifier finds the assert valid -> Take-2 -> finishWithdrawUnhappyPath. +# MODE=invalid (operator_0 + verifier_0 restarted with MOCK_GC_PROOF_VERDICT=always-invalid): ChallengeAssert -> +# Wrongly-Challenged cannot be built -> Disprove -> finishWithdrawDisproved. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id}; MODE=${2:-valid} +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +topup_pegbtc +log "tip $(tip); MODE=$MODE; initWithdraw on graph $gid" +( set -a; source "$CLI_ENV/operator.env"; set +a; "$BIN_DIR/pegout" --rpc-url "$(rpc_url operator_0)" once --graph-id "$gid" 2>&1 ) | tail -2 +wait_log operator_0 "Pre-Kickoff and Kickoff broadcast.*$gid\|$gid.*Pre-Kickoff and Kickoff broadcast" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1; K=$(tip) +wait_all graph "$gid" OperatorKickOff 180 || exit 1 +mine 1 +wait_log committee_0 "$gid.*proceedWithdraw sent\|proceedWithdraw sent.*$gid" 180 || { engine_summary; exit 1; } +log "challenging from the verifier while tip=$(tip) (< K+2=$((K+2)))" +( set -a; source "$CLI_ENV/verifier.env"; set +a; "$BIN_DIR/challenge" --rpc-url "$(rpc_url verifier_0)" --graph-id "$gid" 2>&1 | tail -2 ) +wait_mempool 60 || exit 1 +mine 1 +wait_log operator_0 "$gid.*Watchtower-Challenge-Init broadcast\|Watchtower-Challenge-Init broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1; W=$(tip); log "WCI confirmed at W=$W; holding for the watchtower proof (30 s)" +wait_log watchtower_0 "$gid.*Watchtower-Challenge broadcast\|Watchtower-Challenge broadcast.*$gid" 300 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +wait_log operator_0 "$gid.*Operator-Challenge-ACK broadcast\|Operator-Challenge-ACK broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +# commit-pubin needs the watchtower challenge buried 2 blocks (= verifier commit timeout at W+3): keep the +# verifier out of the race, as the runbook does. +"$ROOT/scripts/nodes.sh" stop verifier_0 >/dev/null 2>&1; log "verifier_0 stopped for the pubin block" +mine 1; log "tip $(tip) = W+3" +wait_log operator_0 "$gid.*Operator-Commit-Pubin broadcast\|Operator-Commit-Pubin broadcast.*$gid" 240 || { engine_summary; NODES_SH start verifier_0 >/dev/null 2>&1; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +NODES_SH start verifier_0 >/dev/null 2>&1; log "verifier_0 restarted (verdict: $(current_verdict))" +log "waiting for the operator proof (30 s) and the assert" +wait_log operator_0 "$gid.*Prover-Assert broadcast\|Prover-Assert broadcast.*$gid" 300 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1; A=$(tip); log "assert confirmed at A=$A" +if [[ $MODE == valid ]]; then + wait_log verifier_0 "operator assert proof is valid" 240 || { engine_summary; exit 1; } + for i in $(seq 1 15); do + if fresh_log operator_0 | grep -q "Take-2 broadcast"; then break; fi + sleep 20; mine 1 + done + wait_log operator_0 "$gid.*Take-2 broadcast\|Take-2 broadcast.*$gid" 120 || { engine_summary; exit 1; } + [[ -n "$(mempool)" ]] && mine 1 + wait_all graph "$gid" OperatorTake2 180 || exit 1 + mine 1 + wait_log committee_0 "$gid.*finishWithdrawUnhappyPath sent\|finishWithdrawUnhappyPath sent.*$gid" 240 || { engine_summary; exit 1; } +else + wait_log verifier_0 "ChallengeAssert broadcast\|Verifier-Assert broadcast\|verifier assert broadcast\|verifier connector spent by unconfirmed" 300 || { engine_summary; exit 1; } + wait_mempool 60 || exit 1 + mine 1; log "ChallengeAssert confirmed at $(tip); the operator cannot answer under always-invalid" + sleep 30 + for i in $(seq 1 8); do + if [[ "$(all_graph "$gid") " == *"verifier_0=Disprove "* ]]; then break; fi + mine 1; sleep 25 + done + wait_any graph "$gid" Disprove 120 || { engine_summary; exit 1; } + [[ -n "$(mempool)" ]] && mine 1 + wait_all graph "$gid" Disprove 180 || exit 1 + mine 1 + wait_log committee_0 "$gid.*finishWithdrawDisproved sent\|finishWithdrawDisproved sent.*$gid" 240 || { engine_summary; exit 1; } +fi +engine_summary +log "CHALLENGE FLOW ($MODE) DONE graph=$gid tip=$(tip)" diff --git a/localnet/e2e/flows/finish-pegout.sh b/localnet/e2e/flows/finish-pegout.sh new file mode 100755 index 000000000..f96640a21 --- /dev/null +++ b/localnet/e2e/flows/finish-pegout.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Finish a peg-out that is already past the confirmed kickoff: proceedWithdraw -> Take-1 -> finishWithdrawHappyPath. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id} +wait_log committee_0 "$gid.*proceedWithdraw sent\|proceedWithdraw sent.*$gid" ${2:-600} || { engine_summary; exit 1; } +mine 2 +wait_log operator_0 "$gid.*Take-1 broadcast\|Take-1 broadcast.*$gid" 300 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +wait_all graph "$gid" OperatorTake1 180 || exit 1 +mine 1 +wait_log committee_0 "$gid.*finishWithdrawHappyPath sent\|finishWithdrawHappyPath sent.*$gid" 300 || { engine_summary; exit 1; } +engine_summary +log "PEGOUT FINISHED graph=$gid tip=$(tip)" diff --git a/localnet/e2e/flows/pegin-restart.sh b/localnet/e2e/flows/pegin-restart.sh new file mode 100755 index 000000000..a3cc66516 --- /dev/null +++ b/localnet/e2e/flows/pegin-restart.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Happy peg-in with one node restarted while it is executing a job of this peg-in's ceremony (stability check). +# e2e/flows/pegin-restart.sh [max-wait-secs] RESTART_ON_ACTION= restricts the trigger to one job action +# After Prepare confirms, the target's engine_job table is polled; the node is restarted the moment one of the +# instance's (or its graphs') jobs is Running. Fails when no such job is caught before the instance is Presigned +# on the target, when the interrupted job does not end Done (re-run by the new process, or finished by the old one +# during its graceful shutdown), or when the restart produced a duplicate side effect (a second graph, a second +# Peg-In-Confirm transaction). +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +role=${1:?role}; max_wait=${2:-900} +set -o pipefail +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +log "tip $(tip); requesting a 1_000_000 sat peg-in (restart of $role while it runs one of the ceremony's jobs)" +out=$( set -a; source "$CLI_ENV/user.env"; set +a; "$BIN_DIR/pegin-request" --network regtest --esplora-url "$BTC_CHAIN_URL" request --pegin-amount-sats 1000000 --receiver-evm-address "$OP0_GOAT_ADDRESS" 2>&1 ) || { echo "$out"; log "REQUEST FAILED"; exit 1; } +id=$(grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' <<<"$out" | head -1) +[[ -n "$id" ]] || { log "no instance id in the output"; exit 1; } +echo "$id" > "$E2E_RUN/inst.txt"; log "instance $id" +wait_all inst "$id" UserInited 180 || exit 1 +wait_all inst "$id" CommitteesAnswered 600 || exit 1 +log "prepare (deposit)" +T_PREP=$(date +%s) # graphs whose first job on the target appears after this belong to this peg-in +( set -a; source "$CLI_ENV/user.env"; set +a; "$BIN_DIR/pegin-request" --network regtest --esplora-url "$BTC_CHAIN_URL" prepare --instance-id "$id" 2>&1 ) | tail -1 +wait_mempool 60 || exit 1 +mine 1 +wait_all inst "$id" UserBroadcastPeginPrepare 240 || exit 1 +rport=$((RPC_PORT_BASE + $(role_idx "$role"))); t0=$(date +%s); job="" +while :; do + # the graph is not listed on the target's RPC until it is stored there, so its setup jobs are found by age instead + job=$(db "$role" "select job_id||'|'||action from engine_job j where state='Running' and (aggregate_id='$id' or (aggregate_kind='graph' and (select min(created_at) from engine_job k where k.aggregate_id=j.aggregate_id) >= $T_PREP))${RESTART_ON_ACTION:+ and action='$RESTART_ON_ACTION'} limit 1") + [[ -n $job ]] && break + st=$(inst_status "$id" "$rport") + case $st in Presigned|RelayerL1Broadcasted|RelayerL2Minted) log "FAIL: the ceremony on $role reached $st before any of its jobs was caught running"; exit 1;; esac + (( $(date +%s) - t0 >= max_wait )) && { log "FAIL: no running job of this peg-in on $role within ${max_wait}s"; exit 1; } + sleep 0.5 +done +jid=${job%%|*}; jact=${job#*|} +log "restarting $role while job $jact (${jid:0:12}) is Running; other jobs: $(db $role "select action||':'||state from engine_job where state in ('Running','Pending')" | tr '\n' ' ')" +NODES_SH restart "$role" 2>&1 | grep -E 'started|FAILED' | sed 's/^/ /' +T_UP=$(date +%s) # the old process has exited, the new one is running +wait_all inst "$id" Presigned 1500 || { engine_summary; exit 1; } +log "waiting for the relayer's Peg-In-Confirm" +# wait for the broadcast line itself: a stale mempool entry would otherwise make us mine one block too early +wait_log committee_0 "Peg-In-Confirm broadcast.*$id\|$id.*Peg-In-Confirm broadcast" 300 || exit 1 +wait_mempool 300 || exit 1 +mine 1 +wait_all inst "$id" RelayerL1Broadcasted 180 || exit 1 +mine 1 +wait_all inst "$id" RelayerL2Minted 600 || exit 1 +gid=$(graph_of_instance "$id" | awk '{print $1}' | head -1); echo "$gid" > "$E2E_RUN/graph.txt" +log "graph(s): $(graph_of_instance "$id" | tr '\n' ';')" +wait_all graph "$gid" L2Recorded 300 || exit 1 +log "$role recovery lines: $(fresh_log $role | grep -E 'engine_recovery|lost its holder|re-claimed|resumed' | grep -oE '"message":"[^"]*"' | sort | uniq -c | tr '\n' '|')" +engine_summary +rc=0 +jrow=$(db "$role" "select state||'|'||attempts||'|'||coalesce(finished_at,0) from engine_job where job_id='$jid'") +IFS='|' read -r jst jatt jfin <<<"$jrow" +log "interrupted job $jact: state=$jst attempts=$jatt finished_at=$jfin (new process up at $T_UP)" +if [[ $jst != Done ]]; then log "FAIL: the interrupted job $jact did not end Done ($jst)"; rc=1 +elif (( jatt < 2 && jfin > T_UP )); then log "FAIL: $jact finished after the restart without being re-run (attempts=$jatt)"; rc=1 +elif (( jatt < 2 )); then log "$jact finished in the old process during its graceful shutdown" +else log "$jact was re-run by the new process"; fi +ng=$(graph_of_instance "$id" | wc -l) +[[ $ng == 1 ]] || { log "FAIL: $ng graphs for the instance, expected 1"; rc=1; } +nc=$(fresh_log committee_0 | jq -r --arg id "$id" 'select(.message=="Peg-In-Confirm broadcast" and .instance_id==$id) | .txid' 2>/dev/null | sort -u | wc -l) +[[ $nc == 1 ]] || { log "FAIL: $nc distinct Peg-In-Confirm transactions for the instance, expected 1"; rc=1; } +(( rc == 0 )) || exit 1 +log "PEGIN(restart $role) DONE instance=$id graph=$gid tip=$(tip)" diff --git a/localnet/e2e/flows/pegin.sh b/localnet/e2e/flows/pegin.sh new file mode 100755 index 000000000..737629883 --- /dev/null +++ b/localnet/e2e/flows/pegin.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Happy peg-in: request -> answers -> Prepare -> ceremony -> PeginConfirm -> L2 mint. Prints the instance and graph ids. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +set -o pipefail +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +set -a; source "$CLI_ENV/user.env"; set +a +log "tip $(tip); requesting a 1_000_000 sat peg-in to the operator address" +out=$("$BIN_DIR/pegin-request" --network regtest --esplora-url "$BTC_CHAIN_URL" request --pegin-amount-sats 1000000 --receiver-evm-address "$OP0_GOAT_ADDRESS" 2>&1) || { echo "$out"; log "REQUEST FAILED"; exit 1; } +echo "$out" | tail -5 +id=$(grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' <<<"$out" | head -1) +[[ -n "$id" ]] || { log "no instance id in the output"; exit 1; } +echo "$id" > "$E2E_RUN/inst.txt"; log "instance $id" +wait_all inst "$id" UserInited 180 || exit 1 +wait_all inst "$id" CommitteesAnswered 600 || exit 1 +log "prepare (deposit)" +"$BIN_DIR/pegin-request" --network regtest --esplora-url "$BTC_CHAIN_URL" prepare --instance-id "$id" 2>&1 | tail -3 +wait_mempool 60 || exit 1 +mine 1 +wait_all inst "$id" UserBroadcastPeginPrepare 240 || exit 1 +log "ceremony running (mock delays 15 s each)" +wait_all inst "$id" Presigned 1200 || exit 1 +log "waiting for the relayer's Peg-In-Confirm" +wait_log committee_0 "Peg-In-Confirm broadcast.*$id\|$id.*Peg-In-Confirm broadcast" 300 || exit 1 +wait_mempool 300 || exit 1 +mine 1 +wait_all inst "$id" RelayerL1Broadcasted 180 || exit 1 +mine 1 +log "SPV fed; waiting for postPeginData" +wait_all inst "$id" RelayerL2Minted 600 || exit 1 +gid=$(graph_of_instance "$id" | awk '{print $1}' | head -1); echo "$gid" > "$E2E_RUN/graph.txt" +log "graph(s): $(graph_of_instance "$id" | tr '\n' ';')" +wait_all graph "$gid" L2Recorded 300 || exit 1 +engine_summary +log "PEGIN DONE instance=$id graph=$gid tip=$(tip)" diff --git a/localnet/e2e/flows/pegout.sh b/localnet/e2e/flows/pegout.sh new file mode 100755 index 000000000..8261c267f --- /dev/null +++ b/localnet/e2e/flows/pegout.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Happy peg-out: initWithdraw -> kickoff -> proceedWithdraw -> Take-1 -> finishWithdrawHappyPath. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id} +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +topup_pegbtc +log "tip $(tip); initWithdraw on graph $gid" +( set -a; source "$CLI_ENV/operator.env"; set +a; "$BIN_DIR/pegout" --rpc-url "$(rpc_url operator_0)" once --graph-id "$gid" 2>&1 ) | tail -3 +# Earlier unsettled slots are skipped first; each Skip needs a block to confirm. +for i in $(seq 1 10); do + if fresh_log operator_0 | grep -q "$gid.*Pre-Kickoff and Kickoff broadcast\|Pre-Kickoff and Kickoff broadcast.*$gid"; then break; fi + if fresh_log operator_0 | grep -q "graph_skip" && [[ -n "$(mempool)" ]]; then mine 1; fi + sleep 20 +done +wait_log operator_0 "$gid.*Pre-Kickoff and Kickoff broadcast\|Pre-Kickoff and Kickoff broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +wait_all graph "$gid" OperatorKickOff 180 || exit 1 +mine 1 +wait_log committee_0 "$gid.*proceedWithdraw sent\|proceedWithdraw sent.*$gid" 180 || { engine_summary; exit 1; } +log "verifier verdicts: $(grep -h "graph_job" "$LOGS_DIR/nodes/verifier_0.log" | grep "$gid" | grep -oE 'note=[^ ]*.*' | tail -3 | cut -c1-160 | tr '\n' '|')" +mine 2 +wait_log operator_0 "$gid.*Take-1 broadcast\|Take-1 broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +mine 1 +wait_all graph "$gid" OperatorTake1 180 || exit 1 +mine 1 +wait_log committee_0 "$gid.*finishWithdrawHappyPath sent\|finishWithdrawHappyPath sent.*$gid" 240 || { engine_summary; exit 1; } +engine_summary +log "PEGOUT DONE graph=$gid tip=$(tip)" diff --git a/localnet/e2e/flows/take2.sh b/localnet/e2e/flows/take2.sh new file mode 100755 index 000000000..62b5bb164 --- /dev/null +++ b/localnet/e2e/flows/take2.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Continue a challenged graph whose assert stands unchallenged: mine until Take-2 appears, confirm it, finish on Goat. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id} +log "tip $(tip); stepping blocks until the operator broadcasts Take-2" +for i in $(seq 1 15); do + if fresh_log operator_0 | grep -q "Take-2 broadcast"; then break; fi + sleep 20; mine 1 +done +wait_log operator_0 "$gid.*Take-2 broadcast\|Take-2 broadcast.*$gid" 120 || { engine_summary; exit 1; } +[[ -n "$(mempool)" ]] && mine 1 +wait_all graph "$gid" OperatorTake2 180 || { mine 1; wait_all graph "$gid" OperatorTake2 120 || exit 1; } +mine 1 +wait_log committee_0 "$gid.*finishWithdrawUnhappyPath sent\|finishWithdrawUnhappyPath sent.*$gid" 240 || { engine_summary; exit 1; } +engine_summary +log "TAKE2 DONE graph=$gid tip=$(tip)" diff --git a/localnet/e2e/lib.sh b/localnet/e2e/lib.sh new file mode 100644 index 000000000..c3c01390a --- /dev/null +++ b/localnet/e2e/lib.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# Helpers sourced by the e2e flow / scenario scripts. Everything about *this* localnet comes from +# scripts/localnet.env (code root, data root, ports) and $LOCALNET_DATA/state (keys, contracts, subgraph). +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +source "$ROOT/scripts/localnet.env" +set -a; source "$STATE_DIR/keys.env"; set +a +GW=$(jq -r '.addresses.gateway_proxy' "$STATE_DIR/contracts.json") +PEG=$(jq -r '.addresses.peg_btc' "$STATE_DIR/contracts.json") +SUBGRAPH_URL=$(jq -r '.query_url' "$STATE_DIR/subgraph.json") +CLI_ENV="$STATE_DIR/cli" # user.env / operator.env / verifier.env (gen-node-envs.sh) +E2E_RUN="$RUN_DIR/e2e"; mkdir -p "$E2E_RUN" # inst.txt / graph.txt / last-*.out scratch of the flows +ROLES=(committee_0 committee_1 operator_0 verifier_0 watchtower_0) +NODES=(); for _i in "${!ROLES[@]}"; do NODES+=("${ROLES[$_i]}:$((RPC_PORT_BASE + _i))"); done +role_idx() { local i=0; for n in "${NODES[@]}"; do [[ ${n%%:*} == "$1" ]] && { echo $i; return; }; i=$((i+1)); done; echo 0; } +rpc_url() { echo "http://127.0.0.1:$((RPC_PORT_BASE + $(role_idx "$1")))"; } +metrics_url() { echo "http://127.0.0.1:$((METRICS_PORT_BASE + $(role_idx "$1")))/metrics"; } +CHAOS_CTRL="http://127.0.0.1:$CHAOS_CTRL_PORT" +# nodes.sh must not inherit a CLI identity sourced by a flow +NODES_SH() { env -u BITVM_SECRET -u GOAT_PRIVATE_KEY -u GOAT_ADDRESS "$ROOT/scripts/nodes.sh" "$@"; } +POLL=${E2E_POLL_SECS:-2} # seconds between status / log / mempool polls +ts() { date -u +%H:%M:%S; } +log() { echo "[$(ts)] $*"; } +tip() { $BCLI getblockcount; } +mine() { local n=${1:-1}; local addr; addr=$($BCLI -rpcwallet="$BTC_RPC_WALLET" getnewaddress bvlocal-miner bech32); $BCLI generatetoaddress "$n" "$addr" >/dev/null; log "mined $n -> tip $(tip)"; } +mempool() { $BCLI getrawmempool | python3 -c 'import sys,json; print(" ".join(x[:8] for x in json.load(sys.stdin)))'; } +inst_status() { curl -s --max-time 5 "http://127.0.0.1:$2/v1/instances/$1" | python3 -c 'import sys,json +try: print(json.load(sys.stdin)["instance_wrap"]["instance"]["status"]) +except Exception: print("?")' 2>/dev/null; } +graph_status() { curl -s --max-time 5 "http://127.0.0.1:$2/v1/graphs/$1" | python3 -c 'import sys,json +try: print(json.load(sys.stdin)["graph"]["status"]) +except Exception: print("?")' 2>/dev/null; } +all_inst() { local out=""; for n in "${NODES[@]}"; do out+="${n%%:*}=$(inst_status "$1" "${n##*:}") "; done; echo "$out"; } +all_graph() { local out=""; for n in "${NODES[@]}"; do out+="${n%%:*}=$(graph_status "$1" "${n##*:}") "; done; echo "$out"; } +# wait_all kind id status timeout -> 0 when every node reports the status +wait_all() { local kind=$1 id=$2 want=$3 timeout=${4:-300} t=0 line + while :; do + if [[ $kind == inst ]]; then line=$(all_inst "$id"); else line=$(all_graph "$id"); fi + if [[ $(grep -o "=$want " <<<"$line " | wc -l) -eq ${#NODES[@]} ]]; then log "$kind $id: all $want"; return 0; fi + if (( t >= timeout )); then log "TIMEOUT waiting $kind $id -> $want after ${timeout}s: $line"; return 1; fi + sleep "$POLL"; t=$((t+POLL)) + done; } +wait_any() { local kind=$1 id=$2 want=$3 timeout=${4:-300} t=0 line + while :; do + if [[ $kind == inst ]]; then line=$(all_inst "$id"); else line=$(all_graph "$id"); fi + if [[ "$line " == *"=$want "* ]]; then log "$kind $id: some node $want ($line)"; return 0; fi + if (( t >= timeout )); then log "TIMEOUT waiting $kind $id -> any $want after ${timeout}s: $line"; return 1; fi + sleep "$POLL"; t=$((t+POLL)) + done; } +wait_mempool() { local timeout=${1:-120} t=0 m + while :; do m=$(mempool); if [[ -n "$m" ]]; then log "mempool: $m"; return 0; fi + if (( t >= timeout )); then log "TIMEOUT: mempool still empty after ${timeout}s"; return 1; fi; sleep "$POLL"; t=$((t+POLL)); done; } +# One log file per boot (logs/nodes//.log, `current` -> newest). Only lines written after this +# helper was sourced count; a node restarted mid-flow continues in a newer file, which is read whole. +declare -A LOG_START LOG_FILE +for _n in "${NODES[@]}"; do _r=${_n%%:*}; LOG_FILE[$_r]=$(readlink -f "$LOGS_DIR/nodes/$_r/current" 2>/dev/null || echo ""); LOG_START[$_r]=$(wc -l < "${LOG_FILE[$_r]:-/dev/null}" 2>/dev/null || echo 0); done +fresh_log() { local d="$LOGS_DIR/nodes/$1" f + for f in "$d"/*.log; do + [[ -f $f ]] || continue + if [[ $f == "${LOG_FILE[$1]}" ]]; then tail -n "+$(( ${LOG_START[$1]:-0} + 1 ))" "$f" + elif [[ $f > "${LOG_FILE[$1]}" ]]; then cat "$f"; fi + done; } +# wait_log role pattern timeout +wait_log() { local role=$1 pat=$2 timeout=${3:-120} t=0 + while :; do local hit; hit=$(fresh_log "$role" | grep -a -- "$pat" | tail -1); if [[ -n $hit ]]; then log "$role log: $(cut -c1-220 <<<"$hit")"; return 0; fi + if (( t >= timeout )); then log "TIMEOUT: no '$pat' in $role log after ${timeout}s"; return 1; fi; sleep "$POLL"; t=$((t+POLL)); done; } +engine_summary() { for n in "${NODES[@]}"; do curl -s --max-time 5 "http://127.0.0.1:${n##*:}/v1/debug/engine" | python3 -c ' +import sys,json +d=json.load(sys.stdin); c={(x["table"],x["state"]):x["count"] for x in d["counts"]} +print(" %-12s inbox Pending=%d Failed=%d Poisoned=%d waiting=%d poisoned_actions=%d jobs Pending=%d Running=%d Failed=%d Poisoned=%d" % ("'"${n%%:*}"'", c.get(("inbox","Pending"),0), c.get(("inbox","Failed"),0), c.get(("inbox","Poisoned"),0), len(d["waiting"]), len(d["poisoned"]), c.get(("job","Pending"),0), c.get(("job","Running"),0), c.get(("job","Failed"),0), c.get(("job","Poisoned"),0))) +for w in d["waiting"]: print(" waiting:", w["aggregate_kind"], w["aggregate_id"][:8], w["kind"], w.get("waiting_ref"), (w.get("waiting_detail") or "")[:80]) +for p in d["poisoned"]: print(" POISONED:", p["aggregate_kind"], p["aggregate_id"][:8], p["action"], "count=%d blocked_until=%d" % (p["poison_count"], p["blocked_until"]), (p.get("reason") or "")[:80])' 2>/dev/null || echo " ${n%%:*}: engine endpoint unavailable"; done; } +# alerts role -> instance alert counters of one node (metrics) +alerts() { curl -s --max-time 5 "$(metrics_url "$1")" | grep '^bitvm_node_instance_alerts' | sed 's/^/ /'; } +# db role "sql" +db() { sqlite3 -noheader -list -cmd ".timeout 8000" "$NODES_DIR/$1/bitvm-node.db" "$2"; } +# warn_summary role -> new WARN/ERROR lines since the script started, grouped by event/outcome +warn_summary() { fresh_log "$1" | grep -E ' (WARN|ERROR) ' | sed -E 's/\x1b\[[0-9;]*m//g' | grep -oE 'event=[a-z_]+( outcome=[a-z_]+)?|(WARN|ERROR) [^ ]+' | sort | uniq -c | sort -rn | head -20 | sed 's/^/ /'; } +graphs_on() { curl -s --max-time 5 "$(rpc_url "$1")/v1/graphs?limit=200" | python3 -c ' +import sys,json +try: + for g in json.load(sys.stdin)["graphs"]: + g=g["graph"] + if g["instance_id"]=="'"$2"'": print(g["graph_id"]) +except Exception: pass'; } +graph_of_instance() { curl -s --max-time 5 "$(rpc_url committee_0)/v1/graphs?limit=200" | python3 -c ' +import sys,json +for g in json.load(sys.stdin)["graphs"]: + g=g["graph"] + if g["instance_id"]=="'"$1"'": print(g["graph_id"], g["kickoff_index"], g["status"])'; } +pegbtc_sats() { $CAST call $PEG "balanceOf(address)(uint256)" "$OP0_GOAT_ADDRESS" --rpc-url "$GOAT_CHAIN_URL" | awk '{printf "%d\n", $1/1e10}'; } +topup_pegbtc() { # ensure the operator holds >= 1_100_000 sats of pegBTC + local have; have=$(pegbtc_sats); if (( have < 1100000 )); then + $CAST send $GW "debugMintPegBTC(address,uint256)" "$OP0_GOAT_ADDRESS" $(( (1100000 - have) * 10000000000 )) --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$GOAT_CHAIN_URL" >/dev/null && log "pegBTC topped up: $have -> $(pegbtc_sats) sats"; else log "pegBTC $have sats, no top-up"; fi; } + +# ---- chaos / backend helpers ---- +CHAOS="$ROOT/scripts/chaos.sh" +chaos_state() { "$CHAOS" status | sed 's/^/ /'; } +proxy_stats() { curl -s --max-time 5 "$CHAOS_CTRL/stats" | python3 -c ' +import sys,json +for k,v in json.load(sys.stdin).items(): + if v["requests"]: print(" %-20s req=%-6d fwd=%-6d err=%-4d active=%d faulted=%s" % (k, v["requests"], v["forwarded"], v["errors"], v["active"], v["faulted"] or ""))'; } +# backend_probe role -> ready bit, backend request counters by outcome, last-success ages +backend_probe() { local m now; m=$(curl -s --max-time 5 "$(metrics_url "$1")"); now=$(date +%s) + echo " $1 ready=$(awk '/^bitvm_node_ready /{print $2}' <<<"$m") event_watch_ready=$(awk '/^bitvm_node_event_watch_ready /{print $2}' <<<"$m") $(grep -E '^bitvm_node_(btc|goat)_backend_requests_total' <<<"$m" | sed -E 's/bitvm_node_//; s/_backend_requests_total//' | tr '\n' ' ')" + grep -E '_last_success_timestamp_seconds' <<<"$m" | awk -v now="$now" '{n=$1; sub(/bitvm_node_/,"",n); printf " %s age=%ds\n", n, now-$2}'; } +# warn_delta role -> WARN/ERROR lines since the script started, grouped by event/outcome (or message prefix) +warn_delta() { fresh_log "$1" | jq -r 'select(.level=="WARN" or .level=="ERROR") | [.level, (.event // "-"), (.outcome // "-"), (.message[0:60])] | @tsv' 2>/dev/null | sort | uniq -c | sort -rn | head -15 | sed 's/^/ /'; } +# snapshot_heights -> truth heights for the scenario log +snapshot_heights() { echo " tip=$(tip) anvil=$($CAST block-number --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null) subgraph=$(curl -s --max-time 5 -XPOST -H 'content-type: application/json' -d '{"query":"{_meta{block{number}}}"}' "$SUBGRAPH_URL" | jq -r .data._meta.block.number)"; } +scenario_begin() { log "=== SCENARIO $* BEGIN"; snapshot_heights; chaos_state; } +scenario_end() { log "=== SCENARIO $1 END"; snapshot_heights; engine_summary; local r; for r in committee_0 committee_1 operator_0 verifier_0 watchtower_0; do echo " $r warn/error:"; warn_delta "$r"; done; proxy_stats; chaos_state; } + +# ---- e2e helpers ---- +# next_posted_graph -> lowest-slot graph still L2Recorded on the operator (peg-outs go in slot order) +next_posted_graph() { curl -s --max-time 5 "$(rpc_url operator_0)/v1/graphs?limit=500" | python3 -c ' +import sys,json +gs=[g["graph"] for g in json.load(sys.stdin)["graphs"] if g["graph"]["status"]=="L2Recorded"] +gs.sort(key=lambda g: g["kickoff_index"]) +print(gs[0]["graph_id"] if gs else "")'; } +# set_tuning VAR VALUE / clear_tuning VAR -> runtime override of one MOCK_* knob in run/tuning.env (sourced by +# nodes.sh at launch, so it takes effect on the next (re)start of a node; localnet.env itself is never edited) +set_tuning() { local f="$RUN_DIR/tuning.env"; touch "$f"; sed -i "/^export $1=/d" "$f"; echo "export $1=$2" >> "$f"; } +clear_tuning() { [[ -f "$RUN_DIR/tuning.env" ]] && sed -i "/^export $1=/d" "$RUN_DIR/tuning.env"; true; } +# tuning_get VAR -> the value a (re)started node would see: run/tuning.env first, then localnet.env +tuning_get() { local v; v=$(grep -h "^export $1=" "$RUN_DIR/tuning.env" 2>/dev/null | tail -1 | cut -d= -f2-); [[ -n $v ]] && echo "$v" || echo "${!1}"; } +current_verdict() { { grep -h '^export MOCK_GC_PROOF_VERDICT=' "$RUN_DIR/tuning.env" 2>/dev/null || echo "export MOCK_GC_PROOF_VERDICT=$MOCK_GC_PROOF_VERDICT"; } | tail -1 | cut -d= -f2; } +# set_verdict valid|always-invalid -> override the verdict and restart operator_0 + verifier_0 +set_verdict() { if [[ $1 == valid ]]; then clear_tuning MOCK_GC_PROOF_VERDICT; else set_tuning MOCK_GC_PROOF_VERDICT "$1"; fi + NODES_SH restart operator_0 verifier_0 >/dev/null 2>&1; sleep 10; log "verdict=$1, operator_0 + verifier_0 restarted"; } +# wait_ready secs -> every node reports bitvm_node_ready 1 (or timeout) +wait_ready() { local t=0 ok; while :; do ok=1; for n in "${NODES[@]}"; do [[ "$(curl -s --max-time 3 "$(metrics_url "${n%%:*}")" | awk '/^bitvm_node_ready /{print $2}')" == 1 ]] || ok=0; done + (( ok )) && return 0; (( t >= ${1:-180} )) && { log "TIMEOUT: nodes not ready after ${1:-180}s"; return 1; }; sleep "$POLL"; t=$((t+POLL)); done; } +# engine_snapshot -> remember every node's Failed job ids (taken by the runner's pre-check); engine_clean then +# rejects any Failed job that is not in that snapshot, so an old failure (e.g. a give-up test on a long-lived +# localnet) is tolerated by identity, never by count. Nonzero (and the reason on stdout) when a query fails. +engine_snapshot() { local r out; for r in "${ROLES[@]}"; do + out=$(db "$r" "select job_id from engine_job where state='Failed' order by 1" 2>&1) || { echo " engine snapshot: query failed on $r: ${out:0:160}"; return 1; } + sed '/^$/d' <<<"$out" > "$E2E_RUN/failed-before.$r"; done; } +# new_failed_jobs -> "role action aggregate8 job_id" for every Failed job absent from the snapshot; a failed query +# prints "role QUERY-FAILED " and makes the function return 1 +new_failed_jobs() { local r j a g out rc=0; for r in "${ROLES[@]}"; do + out=$(db "$r" "select job_id||'|'||action||'|'||aggregate_id from engine_job where state='Failed'" 2>&1) || { echo "$r QUERY-FAILED ${out:0:160}"; rc=1; continue; } + while IFS='|' read -r j a g; do [[ -n $j ]] || continue; grep -qx "$j" "$E2E_RUN/failed-before.$r" 2>/dev/null || echo "$r $a ${g:0:8} $j"; done <<<"$out" + done; return $rc; } +# engine_clean -> 0 when no node has Failed/Poisoned/waiting/Running inbox or job rows, no Failed job is new (DB, by +# job_id) and the debug endpoint reports no more Failed jobs than the snapshot holds (independent of the DB query) +engine_clean() { local bad=0 n r d nf snap; for n in "${NODES[@]}"; do r=${n%%:*} + d=$(curl -s --max-time 5 "http://127.0.0.1:${n##*:}/v1/debug/engine") || { echo " $r: debug endpoint unreachable"; bad=1; continue; } + snap=$(grep -c . "$E2E_RUN/failed-before.$r" 2>/dev/null); snap=${snap:-0} + python3 -c ' +import sys,json +d=json.load(sys.stdin); c={(x["table"],x["state"]):x["count"] for x in d["counts"]} +bad = c.get(("inbox","Failed"),0)+c.get(("inbox","Poisoned"),0)+len(d["waiting"])+len(d["poisoned"])+c.get(("job","Running"),0)+c.get(("job","Poisoned"),0) +extra = c.get(("job","Failed"),0) - int(sys.argv[1]) +if extra > 0: print(" %s: %d more Failed job(s) than before the case" % (sys.argv[2], extra)) +sys.exit(1 if bad or extra > 0 else 0)' "$snap" "$r" <<<"$d" || bad=1; done + nf=$(new_failed_jobs) || bad=1 + [[ -n $nf ]] && { echo " new Failed jobs / failed queries: $(tr '\n' ';' <<<"$nf")"; bad=1; } + return $bad; } +# ensure_operator_stake -> top the operator's locked stake back up to STAKE_TARGET_WEI (0.1 WGBTC) when a +# Disprove slash pushed it under the gateway minimum (an unstaked operator is silently ignored by every peer). +ensure_operator_stake() { local sm w locked min target=${STAKE_TARGET_WEI:-100000000000000000} h + sm=$(jq -r '.addresses.stake_management_proxy' "$STATE_DIR/contracts.json"); w=$(jq -r '.addresses.stake_token_wgbtc' "$STATE_DIR/contracts.json") + locked=$($CAST call "$sm" "lockedStakeOf(address)(uint256)" "$OP0_GOAT_ADDRESS" --rpc-url "$GOAT_CHAIN_URL" | awk '{print $1}') + min=$($CAST call "$GW" "minStakeAmount()(uint256)" --rpc-url "$GOAT_CHAIN_URL" | awk '{print $1}') + if (( locked >= min )); then log "operator locked stake $locked >= min $min"; return 0; fi + log "operator locked stake $locked < min $min: topping up $target" + local tx; for tx in "$w mint(address,uint256) $OP0_GOAT_ADDRESS $target" "$w approve(address,uint256) $sm $target" "$sm stake(uint256) $target" "$sm lockStake(address,uint256) $OP0_GOAT_ADDRESS $target"; do + set -- $tx; local key=$OP0_GOAT_PRIVATE_KEY; [[ $2 == mint* ]] && key=$DEPLOYER_GOAT_PRIVATE_KEY + h=$($CAST send "$@" --private-key "$key" --rpc-url "$GOAT_CHAIN_URL" --async 2>&1) || { log "stake top-up: $2 failed: $h"; return 1; } + local i; for i in 1 2 3 4 5 6; do sleep 2; $CAST receipt "$h" --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null | grep -q '^status *1' && break; done + done + locked=$($CAST call "$sm" "lockedStakeOf(address)(uint256)" "$OP0_GOAT_ADDRESS" --rpc-url "$GOAT_CHAIN_URL" | awk '{print $1}'); log "operator locked stake now $locked"; (( locked >= min )); } diff --git a/localnet/e2e/run.sh b/localnet/e2e/run.sh new file mode 100755 index 000000000..0e42a8661 --- /dev/null +++ b/localnet/e2e/run.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# On-demand end-to-end regression of the node on the localnet (localnet/scripts/up.sh or reset.sh first): +# e2e/run.sh # every case, in order +# e2e/run.sh --list +# e2e/run.sh pegin_happy pegout_happy challenge_take2 +# e2e/run.sh --stop-on-fail ... +# Env: E2E_CASE_TIMEOUT (seconds per case, default 1200), E2E_DELAY_TIER=fast|default (mock delays applied to the +# nodes + proof builder for the run through run/tuning.env, default fast = 2 s GC / 5 s proof; localnet.env keeps the +# manual 15/30 s), E2E_POLL_SECS (default 2). +# Each case: fresh scratch (run/e2e), pre-check (nodes ready, no chaos injection, operator stake, snapshot of the +# Failed jobs), the flow, post-check (engine clean on every node and no new Failed job, chaos status empty). run/tuning.env +# is saved before every case and restored after it on every path (pass, fail, timeout, interrupt); a restore that +# had to change something restarts the nodes and is noted in the report. Results in $LOCALNET_DATA/results//: .log, report.md, report.json, logs.tar.gz +# (every node/support log written during the run); results/latest -> newest run. Exit code 0 only when nothing failed +# (a SKIP is not a failure). +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/e2e/lib.sh" >/dev/null 2>&1 +E="$ROOT/e2e"; CHAOS="$ROOT/scripts/chaos.sh" +CASE_TIMEOUT=${E2E_CASE_TIMEOUT:-1200} +TIER=${E2E_DELAY_TIER:-fast} +RUN_ID=$(date -u +%Y%m%dT%H%M%SZ); RUN="$RESULTS_DIR/$RUN_ID"; mkdir -p "$RUN"; touch "$RUN/.start" +CASES_ALL=(pegin_happy pegout_happy challenge_take2 dispute_disprove backtoback pegin_restart_committee0 pegin_restart_operator pegin_restart_verifier + goat_timeout_at_kickoff subgraph_latency proof_builder_blackhole p2p_isolate_operator boot_with_goat_down) +STOP_ON_FAIL=0; CASES=() +for a in "$@"; do case $a in --list) printf '%s\n' "${CASES_ALL[@]}"; exit 0;; --stop-on-fail) STOP_ON_FAIL=1;; *) CASES+=("$a");; esac; done +[[ ${#CASES[@]} -eq 0 ]] && CASES=("${CASES_ALL[@]}") +for c in "${CASES[@]}"; do [[ " ${CASES_ALL[*]} " == *" $c "* ]] || { echo "unknown case: $c (see --list)" >&2; rm -rf "$RUN"; exit 2; }; done + +# ---- cases: each returns 0 on pass; graph ids for peg-out cases come from the lowest posted slot ---- +need_graph() { local g; g=$(next_posted_graph); if [[ -z $g ]]; then log "no posted graph; running a peg-in first" >&2; "$E/flows/pegin.sh" >&2 || return 1; g=$(cat "$E2E_RUN/graph.txt"); fi; echo "$g"; } +case_pegin_happy() { "$E/flows/pegin.sh"; } +case_pegout_happy() { local g; g=$(need_graph) || return 1; "$E/flows/pegout.sh" "$g"; } +case_challenge_take2() { local g; g=$(need_graph) || return 1; "$E/flows/challenge.sh" "$g" valid; } +case_dispute_disprove() { local g rc; g=$(need_graph) || return 1; set_verdict always-invalid; "$E/flows/challenge.sh" "$g" invalid; rc=$?; set_verdict valid; return $rc; } +case_backtoback() { "$E/flows/backtoback.sh"; } +case_pegin_restart_committee0() { "$E/flows/pegin-restart.sh" committee_0; } +case_pegin_restart_operator() { "$E/flows/pegin-restart.sh" operator_0; } +case_pegin_restart_verifier() { # a 20 s solder keeps the verifier's ceremony job running long enough to be caught + local rc before; before=$(tuning_get MOCK_GC_SOLDER_DELAY_SECS); set_tuning MOCK_GC_SOLDER_DELAY_SECS 20; NODES_SH restart verifier_0 >/dev/null 2>&1 + if wait_ready 300; then RESTART_ON_ACTION=verifier_solder "$E/flows/pegin-restart.sh" verifier_0; rc=$?; else rc=1; fi + set_tuning MOCK_GC_SOLDER_DELAY_SECS "$before"; NODES_SH restart verifier_0 >/dev/null 2>&1; return $rc; } +case_goat_timeout_at_kickoff() { local g; g=$(need_graph) || return 1; "$E/scenarios/N2.sh" "$g"; } +case_subgraph_latency() { "$E/scenarios/N9.sh"; } +case_proof_builder_blackhole() { # the challenge flow times out while the proof is black-holed; the case passes when the graph settles in Take-2 afterwards + local g; g=$(need_graph) || return 1; "$E/scenarios/N10.sh" "$g" || return 1; wait_log operator_0 "Prover-Assert broadcast.*$g\|$g.*Prover-Assert broadcast" 600 || return 1 + [[ -n "$(mempool)" ]] && mine 1; "$E/flows/take2.sh" "$g"; } +case_p2p_isolate_operator() { "$E/scenarios/P2.sh" && [[ "$(all_inst "$(cat "$E2E_RUN/inst.txt")")" == *"watchtower_0=RelayerL2Minted"* ]]; } +case_boot_with_goat_down() { "$E/scenarios/R2.sh" watchtower_0 && wait_ready 180; } + +needs_proxy() { case $1 in goat_timeout_at_kickoff|subgraph_latency|proof_builder_blackhole|boot_with_goat_down) return 0;; *) return 1;; esac; } +needs_sudo() { case $1 in p2p_isolate_operator) return 0;; *) return 1;; esac; } +precheck() { local st; engine_snapshot || return 1; "$CHAOS" reset >/dev/null 2>&1 + if ls "$RUN_DIR/overrides"/*.env >/dev/null 2>&1 && ! "$CHAOS" proxy status 2>/dev/null | grep -q 'proxy up'; then # attached nodes need it + "$CHAOS" proxy start >/dev/null; NODES_SH start >/dev/null 2>&1; fi + wait_ready 240 || return 1; ensure_operator_stake || return 1 + if needs_proxy "$1"; then st=$("$CHAOS" proxy status 2>/dev/null); grep -q 'proxy up' <<<"$st" || "$CHAOS" proxy start >/dev/null + ls "$RUN_DIR/overrides"/*.env >/dev/null 2>&1 || { "$CHAOS" attach all >/dev/null; wait_ready 240; }; fi; return 0; } +postcheck() { local ok=0 st; engine_clean || { echo " engine not clean"; ok=1; }; st=$("$CHAOS" status 2>/dev/null) + grep -qE 'faults: none|proxy: DOWN' <<<"$st" || { echo " chaos faults left"; ok=1; }; grep -q 'rules: 0' <<<"$st" || { echo " iptables rules left"; ok=1; }; return $ok; } + +# ---- delay tier: written to run/tuning.env, applied by restarting the nodes + proof builder ---- +TIER_VARS=(MOCK_GC_SETUP_DELAY_SECS MOCK_GC_SOLDER_DELAY_SECS MOCK_GC_VERIFY_DELAY_SECS MOCK_PROOF_OPERATOR_DELAY_SECS MOCK_PROOF_WATCHTOWER_DELAY_SECS) +tier_values() { case $1 in fast) echo "2 2 2 5 5";; default) echo "$MOCK_GC_SETUP_DELAY_SECS $MOCK_GC_SOLDER_DELAY_SECS $MOCK_GC_VERIFY_DELAY_SECS $MOCK_PROOF_OPERATOR_DELAY_SECS $MOCK_PROOF_WATCHTOWER_DELAY_SECS";; *) echo "unknown E2E_DELAY_TIER $1" >&2; exit 2;; esac; } +apply_tier() { # $1 tier -> 0 when something changed (callers restart) + local want i changed=0; read -ra want <<<"$(tier_values "$1")" + for i in "${!TIER_VARS[@]}"; do [[ "$(tuning_get "${TIER_VARS[$i]}")" == "${want[$i]}" ]] || { set_tuning "${TIER_VARS[$i]}" "${want[$i]}"; changed=1; }; done + (( changed )) || return 1; return 0; } +restart_all() { "$ROOT/scripts/proof-builder.sh" stop >/dev/null 2>&1; "$ROOT/scripts/proof-builder.sh" start >/dev/null 2>&1; NODES_SH restart >/dev/null 2>&1; } + +# ---- run/tuning.env around each case: saved before, restored after on every path ---- +tuning_save() { sort "$RUN_DIR/tuning.env" 2>/dev/null > "$RUN/.tuning-before" || :; } +tuning_restore() { # prints a note when something had to be restored; nonzero when the restore itself failed + local now; now=$(sort "$RUN_DIR/tuning.env" 2>/dev/null) + [[ "$now" == "$(cat "$RUN/.tuning-before" 2>/dev/null)" ]] && return 0 + echo "tuning restored: $(diff <(echo "$now") "$RUN/.tuning-before" | grep '^[<>]' | tr '\n' ' ')" + cp "$RUN/.tuning-before" "$RUN_DIR/tuning.env" || return 1 + restart_all; wait_ready 300 >/dev/null || { echo "nodes not ready after the tuning restore"; return 1; }; } +CASE_PID="" +on_interrupt() { [[ -n $CASE_PID ]] && kill -TERM $(descendants "$CASE_PID") "$CASE_PID" 2>/dev/null + [[ -f "$RUN/.tuning-before" ]] && tuning_restore >/dev/null 2>&1; "$CHAOS" reset >/dev/null 2>&1; echo "interrupted" >&2; exit 130; } +trap on_interrupt INT TERM + +# ---- per-case timeout: the case runs in a subshell; on expiry its whole process tree is killed ---- +descendants() { local p; for p in $(ps -o pid= --ppid "$1" 2>/dev/null); do descendants "$p"; echo "$p"; done; } +run_case() { # $1 case $2 log -> rc (124 = timeout) + local c=$1 logf=$2 pid t=0 rc + rm -f "$E2E_RUN"/*.txt "$E2E_RUN"/last-*.out "$E2E_RUN"/failed-before.* # nothing carries over from an earlier case + { precheck "$c" && "case_$c"; } > "$logf" 2>&1 & + pid=$!; CASE_PID=$pid + while kill -0 "$pid" 2>/dev/null; do + if (( t >= CASE_TIMEOUT )); then + echo "[$(date -u +%H:%M:%S)] TIMEOUT: case exceeded ${CASE_TIMEOUT}s, killing it" >> "$logf" + kill -TERM $(descendants "$pid") "$pid" 2>/dev/null; sleep 3; kill -KILL $(descendants "$pid") "$pid" 2>/dev/null + wait "$pid" 2>/dev/null; CASE_PID=""; return 124 + fi + sleep 5; t=$((t+5)) + done + wait "$pid"; rc=$?; CASE_PID=""; return $rc; } + +BUILD=$(git -C "$REPO_ROOT" log --oneline -1 2>/dev/null || echo "?") +T_RUN0=$(date +%s) +if apply_tier "$TIER"; then echo "applying the $TIER delay tier (restart nodes + proof builder)"; restart_all; wait_ready 300 || echo "WARN: nodes not ready after the tier restart"; fi +printf '# e2e run %s\n\nbuild: %s tier: %s case timeout: %ss\n\n| case | result | duration | notes |\n|---|---|---|---|\n' "$RUN_ID" "$BUILD" "$TIER" "$CASE_TIMEOUT" > "$RUN/report.md" +: > "$RUN/cases.tsv" +pass=0; fail=0; skip=0 +for c in "${CASES[@]}"; do + declare -F "case_$c" >/dev/null || { echo "unknown case $c"; continue; } + t0=$(date +%s); echo "=== $c ($(date -u +%T))" + if needs_sudo "$c" && ! sudo -n true 2>/dev/null; then + res=SKIP; rc=0; prc=0; post=""; note="needs passwordless sudo (iptables)"; skip=$((skip+1)); echo "$note" > "$RUN/$c.log" + else + tuning_save + run_case "$c" "$RUN/$c.log"; rc=$? + if (( rc == 124 )); then "$CHAOS" reset >/dev/null 2>&1; NODES_SH start >/dev/null 2>&1; fi + tnote=$(tuning_restore 2>&1); trc=$? + post=$(postcheck 2>&1); prc=$? + [[ -n $tnote ]] && post="$post${post:+;}$tnote"; (( trc != 0 )) && prc=1 + note=$(grep -oE 'instance=[0-9a-f-]{8}|graph=[0-9a-f-]{8}|TIMEOUT[^|]{0,60}' "$RUN/$c.log" | sort -u | tr '\n' ' ' | cut -c1-120) + if (( rc == 0 && prc == 0 )); then res=PASS; pass=$((pass+1)); else res=FAIL; fail=$((fail+1)); fi + fi + "$ROOT/scripts/miner.sh" start >/dev/null 2>&1 + dur=$(( $(date +%s) - t0 )) + printf '| %s | %s | %dm%02ds | %s %s |\n' "$c" "$res" $((dur/60)) $((dur%60)) "$note" "$(tr '\n' ';' <<<"$post")" >> "$RUN/report.md" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$c" "$res" "$dur" "$rc" "$prc" "$note" "$(tr '\n' ';' <<<"$post")" >> "$RUN/cases.tsv" + echo " $res in ${dur}s rc=$rc post=$prc $note" + if [[ $res == FAIL ]] && (( STOP_ON_FAIL )); then break; fi +done +if [[ $TIER != default ]]; then + if apply_tier default; then echo "restoring the default delay tier (restart nodes + proof builder)"; restart_all; fi +fi +TOTAL=$(( $(date +%s) - T_RUN0 )) +printf '\n%d passed, %d failed, %d skipped. tip %s. total %dm%02ds.\n' "$pass" "$fail" "$skip" "$(tip)" $((TOTAL/60)) $((TOTAL%60)) >> "$RUN/report.md" +python3 - "$RUN" "$RUN_ID" "$BUILD" "$TIER" "$CASE_TIMEOUT" "$T_RUN0" "$TOTAL" "$pass" "$fail" "$skip" <<'PY' +import json, sys, datetime +run, rid, build, tier, ct, t0, total, p, f, s = sys.argv[1:] +cases = [] +for line in open(f"{run}/cases.tsv"): + c, res, dur, rc, prc, note, post = (line.rstrip("\n").split("\t") + [""] * 7)[:7] + cases.append({"case": c, "result": res, "duration_secs": int(dur), "rc": int(rc), "postcheck_rc": int(prc), "notes": note.strip(), "postcheck": post.strip(), "log": f"{c}.log"}) +doc = {"run": rid, "build": build, "tier": tier, "case_timeout_secs": int(ct), + "started": datetime.datetime.fromtimestamp(int(t0), datetime.timezone.utc).isoformat(), "total_secs": int(total), + "passed": int(p), "failed": int(f), "skipped": int(s), "cases": cases} +json.dump(doc, open(f"{run}/report.json", "w"), indent=2); open(f"{run}/report.json", "a").write("\n") +PY +rm -f "$RUN/cases.tsv" +# every node / support log written during the run, relative to logs/ +( cd "$LOGS_DIR" && find . -type f -name '*.log' -newer "$RUN/.start" | sed 's#^\./##' | tar -czf "$RUN/logs.tar.gz" -T - 2>/dev/null ) || true +rm -f "$RUN/.start" +cat "$RUN/report.md"; ln -sfn "$RUN" "$RESULTS_DIR/latest" +(( fail == 0 )) diff --git a/localnet/e2e/scenarios/N10.sh b/localnet/e2e/scenarios/N10.sh new file mode 100755 index 000000000..c9eea6cfa --- /dev/null +++ b/localnet/e2e/scenarios/N10.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# N10: the operator's proof-builder requests are black-holed (accepted, never answered) for 240 s, injected once +# Commit-Pubin is broadcast (the operator then polls the proof builder for the Assert proof). The challenge flow is +# expected to time out meanwhile (the runner's case finishes the graph with Take-2); the scenario itself fails only +# when the black hole never caught a request of the operator. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id} +scenario_begin N10 proof-builder-blackhole challenge "$gid" +"$ROOT/e2e/flows/challenge.sh" "$gid" valid > "$E2E_RUN/last-N10-challenge.out" 2>&1 & +flow=$! +for i in $(seq 1 400); do fresh_log operator_0 | grep -q "Operator-Commit-Pubin broadcast" && break; sleep 3; done +hits() { curl -s --max-time 5 "$CHAOS_CTRL/stats" | jq -r '.["operator_0/proof"].faulted.blackhole // 0'; } +h0=$(hits) +"$CHAOS" fault operator_0 proof blackhole 240 --ttl 300; T0=$(date +%s) +log "proof blackhole set at $(ts) (proxy closes the hung connections after 240 s)" +for i in $(seq 1 30); do sleep 20 + log "t+$(( $(date +%s)-T0 ))s proof proxy: $(curl -s $CHAOS_CTRL/stats | jq -c '.["operator_0/proof"]') jobs: $(db operator_0 "select action||':'||state||':'||attempts from engine_job where aggregate_id='$gid' and state in ('Running','Pending') order by updated_at desc limit 3" | tr '\n' ' ')" + fresh_log operator_0 | grep -q "Prover-Assert broadcast" && break +done +wait $flow; rc=$? +cat "$E2E_RUN/last-N10-challenge.out" +h1=$(hits); log "black-holed operator proof requests: $((h1 - h0))" +scenario_end N10; log "N10 flow rc=$rc" +(( h1 > h0 )) || { log "FAIL: no operator proof request hit the black hole"; exit 1; } +exit 0 diff --git a/localnet/e2e/scenarios/N2.sh b/localnet/e2e/scenarios/N2.sh new file mode 100755 index 000000000..f8a2ff8f0 --- /dev/null +++ b/localnet/e2e/scenarios/N2.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# N2: Goat latency 25 s (> 20 s timeout) on every node for 60 s from the kickoff block (the docker-pause case, via proxy). +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id} +scenario_begin N2 goat-timeout-60s pegout "$gid" +"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true +topup_pegbtc +( set -a; source "$CLI_ENV/operator.env"; set +a; "$BIN_DIR/pegout" --rpc-url "$(rpc_url operator_0)" once --graph-id "$gid" 2>&1 ) | tail -1 +wait_log operator_0 "$gid.*Pre-Kickoff and Kickoff broadcast\|Pre-Kickoff and Kickoff broadcast.*$gid" 300 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1 +"$CHAOS" fault all goat latency 25000 --ttl 60; T0=$(date +%s) +mine 1; K=$(tip); log "kickoff confirmed at K=$K with Goat timing out" +sleep 30; mine 1; log "K+1 mined"; sleep 35 +log "fault expired after $(( $(date +%s)-T0 )) s; WARN/ERROR so far:"; for r in committee_0 operator_0; do echo " $r:"; warn_delta "$r"; done +wait_all graph "$gid" OperatorKickOff 180 || exit 1 +wait_log committee_0 "$gid.*proceedWithdraw sent\|proceedWithdraw sent.*$gid" 300 || { engine_summary; exit 1; } +mine 2 +wait_log operator_0 "$gid.*Take-1 broadcast\|Take-1 broadcast.*$gid" 240 || { engine_summary; exit 1; } +wait_mempool 60 || exit 1; mine 1 +wait_all graph "$gid" OperatorTake1 180 || exit 1; mine 1 +wait_log committee_0 "$gid.*finishWithdrawHappyPath sent\|finishWithdrawHappyPath sent.*$gid" 240 || { engine_summary; exit 1; } +scenario_end N2; log "N2 DONE graph=$gid" diff --git a/localnet/e2e/scenarios/N3.sh b/localnet/e2e/scenarios/N3.sh new file mode 100755 index 000000000..2f838c4be --- /dev/null +++ b/localnet/e2e/scenarios/N3.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# N3: Goat RPC flaky for the whole peg-out (50% HTTP 502 + 10% mid-response RST on every node), injected right after +# the initWithdraw CLI call returned (the CLI goes through the operator's RPC and fails fast on a 502 otherwise). +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +gid=${1:?graph id} +scenario_begin N3 goat-flaky pegout "$gid" +"$ROOT/e2e/flows/pegout.sh" "$gid" > "$E2E_RUN/last-N3-pegout.out" 2>&1 & +flow=$! +for i in $(seq 1 60); do grep -q 'initWithdraw submitted' "$E2E_RUN/last-N3-pegout.out" && break; sleep 2; done +"$CHAOS" fault all goat http 502 0.5 --ttl 1500 +"$CHAOS" fault all goat cut 0.1 --ttl 1500 +wait $flow; rc=$? +"$CHAOS" clear all goat +grep -v '^\s*$' "$E2E_RUN/last-N3-pegout.out" | grep '^\[' | cut -c1-120 +scenario_end N3; log "N3 rc=$rc" diff --git a/localnet/e2e/scenarios/N9.sh b/localnet/e2e/scenarios/N9.sh new file mode 100755 index 000000000..af9956fd7 --- /dev/null +++ b/localnet/e2e/scenarios/N9.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# N9: subgraph latency 8 s (< 10 s timeout) on every node for a full peg-in. Exits with the peg-in flow's status. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +scenario_begin N9 subgraph-latency-8s pegin +"$CHAOS" fault all graph latency 8000 --ttl 2400 +"$ROOT/e2e/flows/pegin.sh"; rc=$? +log "event watcher lag/ages:"; for r in committee_0 operator_0; do backend_probe "$r" | grep -E 'event_watch|ready'; done +"$CHAOS" clear all graph +scenario_end N9; log "N9 rc=$rc" +exit $rc diff --git a/localnet/e2e/scenarios/P2.sh b/localnet/e2e/scenarios/P2.sh new file mode 100755 index 000000000..987da5371 --- /dev/null +++ b/localnet/e2e/scenarios/P2.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# P2: operator_0 fully p2p-isolated for 90 s while the setup ceremony runs (after Prepare confirmed). Fails when the +# isolation did not take (operator still has peers half way through), when it does not reconnect to both committee +# members within 60 s of the heal, or when the peg-in flow fails. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +peers() { curl -s --max-time 3 "$(metrics_url operator_0)" | awk '/^bitvm_node_p2p_connected_peers/{print $2}'; } +scenario_begin P2 p2p-isolate-operator pegin +"$ROOT/e2e/flows/pegin.sh" > "$E2E_RUN/last-P2-pegin.out" 2>&1 & +flow=$! +for i in $(seq 1 200); do grep -q 'ceremony running' "$E2E_RUN/last-P2-pegin.out" && break; sleep 3; done +sleep 10 +before=$(peers) +"$CHAOS" p2p isolate operator_0; log "operator_0 isolated for 90 s; peers before=$before" +sleep 45; mid=$(peers); log "operator peers mid-isolation=$mid $(all_inst "$(cat "$E2E_RUN/inst.txt")")" +sleep 45 +"$CHAOS" p2p heal; log "healed" +p=""; for i in $(seq 1 12); do sleep 5; p=$(peers); [[ $p == 2 ]] && break; done; log "operator peers after heal=$p (after $((i*5)) s)" +wait $flow; rc=$? +cat "$E2E_RUN/last-P2-pegin.out" +[[ ${mid:-x} == 0 ]] || { log "FAIL: isolation did not take, operator still had ${mid:-?} peers"; rc=1; } +[[ $p == 2 ]] || { log "FAIL: operator did not reconnect to both committee members within 60 s of the heal"; rc=1; } +scenario_end P2; log "P2 rc=$rc" +exit $rc diff --git a/localnet/e2e/scenarios/R2.sh b/localnet/e2e/scenarios/R2.sh new file mode 100755 index 000000000..600974cad --- /dev/null +++ b/localnet/e2e/scenarios/R2.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# R2: restart a node while its Goat RPC (then esplora, then subgraph) is unreachable. Passes only when the node +# survives each outage in the process it was started as, and that same process is ready again within 240 s of the +# outage ending (the pre-check's ready window; a plain restart takes 60-120 s to ready on a busy host). +# A death is still cleaned up (the node is started again) so later cases run, but the scenario then fails. +source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" +role=${1:-watchtower_0} +pidf="$RUN_DIR/nodes/$role.pid" +ready_of() { curl -s --max-time 3 "$(metrics_url "$role")" | awk '/^bitvm_node_ready /{print $2}'; } +scenario_begin R2 boot-with-backend-down "$role" +fail=0 +for b in goat btc graph; do + "$CHAOS" fault "$role" "$b" down --ttl 120 + NODES_SH restart "$role" | tail -1 + pid0=$(cat "$pidf" 2>/dev/null) + sleep 25 + if [[ -n $pid0 && -d /proc/$pid0 && "$(cat "$pidf" 2>/dev/null)" == "$pid0" ]]; then st=alive; else st=DEAD; fi + log "$b down: $role (pid $pid0) $st after 25 s; last log lines:"; tail -n 3 "$LOGS_DIR/nodes/$role/current" | jq -r '" \(.timestamp[11:19]) \(.level) \(.message[0:150])"' 2>/dev/null || tail -n 3 "$LOGS_DIR/nodes/$role/current" | cut -c1-200 | sed 's/^/ /' + "$CHAOS" clear "$role" "$b" + if [[ $st == DEAD ]]; then + log "FAIL: $role died while $b was down; starting it again for the later cases"; fail=1 + NODES_SH start "$role" | tail -1; sleep 30; continue + fi + ok=0; for i in $(seq 1 120); do + [[ "$(cat "$pidf" 2>/dev/null)" == "$pid0" && -d /proc/$pid0 ]] || break + [[ "$(ready_of)" == 1 ]] && { ok=1; break; }; sleep 2; done + if (( ok )); then log "$b restored: $role pid $pid0 ready after $((i*2)) s, peers=$(curl -s "$(metrics_url "$role")" | awk '/^bitvm_node_p2p_connected_peers/{print $2}')" + else log "FAIL: $role pid $pid0 not ready (or gone) within 240 s of $b coming back"; fail=1; fi +done +scenario_end R2; log "R2 rc=$fail" +exit $fail diff --git a/localnet/infra/docker-compose.yml b/localnet/infra/docker-compose.yml new file mode 100644 index 000000000..305b7c4f2 --- /dev/null +++ b/localnet/infra/docker-compose.yml @@ -0,0 +1,130 @@ +# Localnet infrastructure. Every name derives from COMPOSE_PROJECT, every port from PORT_BASE and all +# data lives under LOCALNET_DATA/data, so two localnets can share one host. Always drive it through +# scripts/*.sh (they export the variables from scripts/localnet.env and pass -p "$COMPOSE_PROJECT"). +# Ports bind 127.0.0.1 only; containers run as the invoking uid so the data dirs stay owned by you. + +x-user: &user "${LOCALNET_UID:-1000}:${LOCALNET_GID:-1000}" + +services: + bitcoind: + image: ${BITCOIND_IMAGE:-bitcoin/bitcoin:29} + container_name: ${COMPOSE_PROJECT:-bvlocal}-bitcoind + user: *user + entrypoint: ["bitcoind"] + command: + - -regtest=1 + - -datadir=/bitcoin + - -server=1 + - -txindex=1 + - -acceptnonstdtxn=1 + - -fallbackfee=0.0002 + - -rpcbind=0.0.0.0 + - -rpcallowip=0.0.0.0/0 + - -rpcuser=bvlocal + - -rpcpassword=bvlocal + - -printtoconsole=1 + ports: + - "127.0.0.1:${BTC_RPC_PORT:-22443}:18443" + volumes: + - ${LOCALNET_DATA:?}/data/bitcoin:/bitcoin + networks: [localnet] + restart: "no" + + electrs: + image: ${ELECTRS_IMAGE:-mempool/electrs:latest} + container_name: ${COMPOSE_PROJECT:-bvlocal}-electrs + user: *user + entrypoint: ["electrs"] + command: + - --network=regtest + - --daemon-rpc-addr=bitcoind:18443 + - --daemon-dir=/bitcoin + - --db-dir=/electrs + - --http-addr=0.0.0.0:3002 + - --cookie=bvlocal:bvlocal + - --jsonrpc-import + - -v + ports: + - "127.0.0.1:${ESPLORA_PORT:-22002}:3002" + volumes: + - ${LOCALNET_DATA:?}/data/bitcoin:/bitcoin:ro + - ${LOCALNET_DATA:?}/data/electrs:/electrs + depends_on: [bitcoind] + networks: [localnet] + restart: "no" + + anvil: + image: ${FOUNDRY_IMAGE:-ghcr.io/foundry-rs/foundry:stable} + container_name: ${COMPOSE_PROJECT:-bvlocal}-anvil + user: *user + entrypoint: ["anvil"] + command: + - --host=0.0.0.0 + - --port=8545 + - --chain-id=${ANVIL_CHAIN_ID:-48816} + - --block-time=${ANVIL_BLOCK_TIME:-2} + - --accounts=20 + - --balance=1000000 + - --state=/anvil/state.json + - --state-interval=30 + ports: + - "127.0.0.1:${ANVIL_PORT:-22545}:8545" + volumes: + - ${LOCALNET_DATA:?}/data/anvil:/anvil + networks: [localnet] + restart: "no" + + postgres: + image: ${POSTGRES_IMAGE:-postgres:16} + container_name: ${COMPOSE_PROJECT:-bvlocal}-postgres + user: *user + command: ["postgres", "-cshared_preload_libraries=pg_stat_statements", "-cmax_connections=200"] + environment: + POSTGRES_USER: graph-node + POSTGRES_PASSWORD: let-me-in + POSTGRES_DB: graph-node + PGDATA: /var/lib/postgresql/data/pgdata + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + volumes: + - ${LOCALNET_DATA:?}/data/postgres:/var/lib/postgresql/data + networks: [localnet] + restart: "no" + + ipfs: + image: ${IPFS_IMAGE:-ipfs/kubo:v0.17.0} + container_name: ${COMPOSE_PROJECT:-bvlocal}-ipfs + user: *user + environment: + IPFS_PATH: /data/ipfs + ports: + - "127.0.0.1:${IPFS_PORT:-22501}:5001" + volumes: + - ${LOCALNET_DATA:?}/data/ipfs:/data/ipfs + networks: [localnet] + restart: "no" + + graph-node: + image: ${GRAPH_NODE_IMAGE:-graphprotocol/graph-node:latest} + container_name: ${COMPOSE_PROJECT:-bvlocal}-graph-node + ports: + - "127.0.0.1:${GRAPH_QUERY_PORT:-22800}:8000" + - "127.0.0.1:${GRAPH_ADMIN_PORT:-22820}:8020" + - "127.0.0.1:${GRAPH_INDEX_PORT:-22830}:8030" + - "127.0.0.1:${GRAPH_METRICS_PORT:-22840}:8040" + depends_on: [postgres, ipfs, anvil] + environment: + postgres_host: postgres + postgres_user: graph-node + postgres_pass: let-me-in + postgres_db: graph-node + ipfs: "ipfs:5001" + ethereum: "goat-local:http://anvil:8545" + ETHEREUM_POLLING_INTERVAL: "1000" + GRAPH_LOG: info + networks: [localnet] + restart: "no" + +networks: + localnet: + name: ${COMPOSE_PROJECT:-bvlocal}-net + driver: bridge diff --git a/localnet/scripts/_miner-loop.sh b/localnet/scripts/_miner-loop.sh new file mode 100755 index 000000000..936d10bb5 --- /dev/null +++ b/localnet/scripts/_miner-loop.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Internal: the regtest mining loop backgrounded by scripts/miner.sh. +# Distinct filename so the process is unambiguously identifiable in /proc. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +ADDR=$($BCLI -rpcwallet="$BTC_RPC_WALLET" getnewaddress bvlocal-miner bech32 2>/dev/null) +echo "[$(date -u +%FT%TZ)] miner loop start: interval=${MINE_INTERVAL_SECS}s addr=$ADDR" +while true; do + h=$($BCLI generatetoaddress 1 "$ADDR" 2>/dev/null | tr -d '[]" \n' | cut -c1-64) + tip=$($BCLI getblockcount 2>/dev/null) + echo "[$(date -u +%FT%TZ)] mined -> height=$tip" + sleep "$MINE_INTERVAL_SECS" +done diff --git a/localnet/scripts/chaos.sh b/localnet/scripts/chaos.sh new file mode 100755 index 000000000..40917eefb --- /dev/null +++ b/localnet/scripts/chaos.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Fault injection for the localnet. +# proxy start|stop|status fault proxy (tools/faultproxy.py, ports PROXY_PORT_BASE+0..19, control CHAOS_CTRL_PORT) +# attach | detach point a role's backend URLs at its proxies (run/overrides/.env) + restart it +# fault [args] [--ttl N] +# latency [jitter] | down | blackhole [secs] | cut

| http [p] | slow +# | stall [secs] | corrupt [p] | flap +# clear [role] [backend] [type] clear proxy faults +# p2p cut | isolate | heal | show +# iptables chain _CHAOS on INPUT -i lo, rules only on this localnet's p2p port pairs +# freeze [secs] | thaw SIGSTOP / SIGCONT (secs: auto-thaw from a detached timer) +# infra pause|unpause +# status everything currently injected (must be empty between scenarios) +# reset clear + heal + thaw all + unpause all +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +ROLES=(committee_0 committee_1 operator_0 verifier_0 watchtower_0) +BACKENDS=(btc goat graph proof) +CTRL=http://127.0.0.1:$CHAOS_CTRL_PORT +PIDF="$RUN_DIR/faultproxy.pid"; LOG="$LOGS_DIR/faultproxy.log"; OVR="$RUN_DIR/overrides" +CHAIN=${COMPOSE_PROJECT^^}_CHAOS +mkdir -p "$RUN_DIR" "$OVR" "$LOGS_DIR" + +die() { echo "chaos: $*" >&2; exit 2; } +role_idx() { local i; for i in "${!ROLES[@]}"; do [[ ${ROLES[$i]} == "$1" ]] && { echo "$i"; return; }; done; die "unknown role $1"; } +backend_idx() { local i; for i in "${!BACKENDS[@]}"; do [[ ${BACKENDS[$i]} == "$1" ]] && { echo "$i"; return; }; done; die "unknown backend $1"; } +proxy_port() { echo $((PROXY_PORT_BASE + $(role_idx "$1") * 4 + $(backend_idx "$2"))); } +p2p_port() { echo $((P2P_PORT_BASE + $(role_idx "$1"))); } +node_pid() { local f="$RUN_DIR/nodes/$1.pid"; [[ -f $f ]] && cat "$f"; } +roles_of() { if [[ $1 == all ]]; then printf '%s\n' "${ROLES[@]}"; else role_idx "$1" >/dev/null && echo "$1"; fi; } +ctl() { curl -s --max-time 5 -X "$1" "$CTRL$2" ${3:+-d "$3"} -H 'content-type: application/json'; } + +proxy_pid() { [[ -f $PIDF ]] && cat "$PIDF"; } +proxy_alive() { local p; p=$(proxy_pid); [[ -n $p && -d /proc/$p ]] && tr '\0' ' ' < "/proc/$p/cmdline" | grep -q faultproxy.py; } + +cmd_proxy() { + case "${1:-status}" in + start) if proxy_alive; then echo "proxy already running (pid $(proxy_pid))"; return; fi + PORT_BASE="$PORT_BASE" nohup python3 "$ROOT/tools/faultproxy.py" >> "$LOG" 2>&1 & echo $! > "$PIDF"; sleep 1 + proxy_alive && echo "proxy started (pid $(proxy_pid)): $(ctl GET /health)" || { echo "proxy FAILED"; tail -3 "$LOG"; return 1; } ;; + stop) if proxy_alive; then kill -TERM "$(proxy_pid)"; sleep 1; echo "proxy stopped"; else echo "proxy not running"; fi; rm -f "$PIDF" ;; + status) if proxy_alive; then echo "proxy up (pid $(proxy_pid)) $(ctl GET /health)"; else echo "proxy DOWN"; fi ;; + *) die "proxy start|stop|status" ;; + esac +} + +write_override() { # role + local r=$1 env="$NODES_DIR/$r/.env" gw sw + gw=$(grep '^GOAT_GATEWAY_EVENT_THE_GRAPH_URL=' "$env" | cut -d= -f2- | sed -E "s#http://127.0.0.1:[0-9]+#http://127.0.0.1:$(proxy_port "$r" graph)#") + sw=$(grep '^GOAT_SWAP_EVENT_THE_GRAPH_URL=' "$env" | cut -d= -f2- | sed -E "s#http://127.0.0.1:[0-9]+#http://127.0.0.1:$(proxy_port "$r" graph)#") + cat > "$OVR/$r.env" </dev/null; backend_idx "$backend" >/dev/null + echo "fault $role/$backend $type $params ttl=$ttl -> $(ctl POST /fault "{\"role\":\"$role\",\"backend\":\"$backend\",\"type\":\"$type\",\"params\":$params,\"ttl\":$ttl}")" +} +cmd_clear() { local b="{"; [[ -n ${1:-} ]] && b+="\"role\":\"$1\","; [[ -n ${2:-} ]] && b+="\"backend\":\"$2\","; [[ -n ${3:-} ]] && b+="\"type\":\"$3\","; b="${b%,}}" + echo "clear -> $(ctl POST /clear "$b")"; } + +ipt() { sudo -n iptables "$@"; } +chain_ensure() { ipt -N $CHAIN 2>/dev/null || true; ipt -C INPUT -i lo -j $CHAIN 2>/dev/null || ipt -I INPUT 1 -i lo -j $CHAIN; } +cut_pair() { # pa pb : drop both directions + kill established + local pa=$1 pb=$2 + ipt -C $CHAIN -p tcp --sport "$pa" --dport "$pb" -j DROP 2>/dev/null || ipt -A $CHAIN -p tcp --sport "$pa" --dport "$pb" -j DROP + ipt -C $CHAIN -p tcp --sport "$pb" --dport "$pa" -j DROP 2>/dev/null || ipt -A $CHAIN -p tcp --sport "$pb" --dport "$pa" -j DROP + sudo -n ss -K -tn "sport = :$pa and dport = :$pb" >/dev/null 2>&1 || true + sudo -n ss -K -tn "sport = :$pb and dport = :$pa" >/dev/null 2>&1 || true +} +cmd_p2p() { + case "${1:-show}" in + cut) chain_ensure; cut_pair "$(p2p_port "${2:?roleA}")" "$(p2p_port "${3:?roleB}")"; echo "p2p cut $2 <-> $3"; ipt -S $CHAIN ;; + isolate) chain_ensure; local r; for r in "${ROLES[@]}"; do [[ $r == "$2" ]] || cut_pair "$(p2p_port "${2:?role}")" "$(p2p_port "$r")"; done; echo "p2p isolated $2"; ipt -S $CHAIN ;; + heal) ipt -F $CHAIN 2>/dev/null; ipt -D INPUT -i lo -j $CHAIN 2>/dev/null; ipt -X $CHAIN 2>/dev/null; echo "p2p healed (chain $CHAIN removed)" ;; + show) ipt -S $CHAIN 2>/dev/null || echo "p2p: no $CHAIN chain" ;; + *) die "p2p cut|isolate|heal|show" ;; + esac +} + +cmd_freeze() { local r=${1:?role} secs=${2:-} pid; pid=$(node_pid "$r"); [[ -n $pid && -d /proc/$pid ]] || die "$r not running" + kill -STOP "$pid" && echo "$r (pid $pid) FROZEN${secs:+ for ${secs}s}" + [[ -n $secs ]] && nohup bash -c "sleep $secs; kill -CONT $pid 2>/dev/null" >/dev/null 2>&1 & true; } +cmd_thaw() { local pid; pid=$(node_pid "${1:?role}"); [[ -n $pid ]] && kill -CONT "$pid" 2>/dev/null && echo "$1 thawed"; } +cmd_infra() { docker "${1:?pause|unpause}" "$COMPOSE_PROJECT-${2:?name}" >/dev/null && echo "$COMPOSE_PROJECT-$2 ${1}d"; } + +cmd_status() { + echo "--- chaos status ---" + if proxy_alive; then local f; f=$(ctl GET /faults); echo " proxy faults: $([[ $f == '{}' ]] && echo none || echo "$f")"; else echo " proxy: DOWN"; fi + local rules; rules=$(ipt -S $CHAIN 2>/dev/null | grep -c -- '-A') ; echo " iptables $CHAIN rules: ${rules:-0}"; ipt -S $CHAIN 2>/dev/null | grep -- '-A' | sed 's/^/ /' + local r pid st; for r in "${ROLES[@]}"; do pid=$(node_pid "$r"); [[ -n $pid && -f /proc/$pid/status ]] && { st=$(awk '/^State:/{print $2}' "/proc/$pid/status"); [[ $st == T ]] && echo " FROZEN: $r (pid $pid)"; }; done + local paused; paused=$(docker ps --filter status=paused --format '{{.Names}}' | grep "^$COMPOSE_PROJECT-" || true); [[ -n $paused ]] && echo " paused containers: $paused" + echo " overrides: $(ls "$OVR" 2>/dev/null | tr '\n' ' ')" +} +cmd_reset() { proxy_alive && cmd_clear; cmd_p2p heal; local r; for r in "${ROLES[@]}"; do cmd_thaw "$r" >/dev/null; done + docker ps --filter status=paused --format '{{.Names}}' | grep "^$COMPOSE_PROJECT-" | xargs -r docker unpause >/dev/null; cmd_status; } + +cmd=${1:-status}; shift || true +case $cmd in + proxy) cmd_proxy "$@" ;; attach) cmd_attach "$@" ;; detach) cmd_detach "$@" ;; + fault) cmd_fault "$@" ;; clear) cmd_clear "$@" ;; p2p) cmd_p2p "$@" ;; + freeze) cmd_freeze "$@" ;; thaw) cmd_thaw "$@" ;; infra) cmd_infra "$@" ;; + status) cmd_status ;; reset) cmd_reset ;; + *) sed -n '2,16p' "$0"; exit 2 ;; +esac diff --git a/localnet/scripts/down.sh b/localnet/scripts/down.sh new file mode 100755 index 000000000..2f3c255a9 --- /dev/null +++ b/localnet/scripts/down.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Bring the localnet down in reverse order: nodes, proof-builder, miner, fault proxy. +# By default LEAVES the infra (anvil/graph-node/bitcoind/...) running. Pass `infra` to also +# `docker compose stop` the containers (data is preserved). +# scripts/down.sh # stop nodes + support processes only +# scripts/down.sh infra # also docker compose stop the infra +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" + +echo "==== nodes ====" +"$ROOT/scripts/nodes.sh" stop +echo "==== proof builder ====" +"$ROOT/scripts/proof-builder.sh" stop +echo "==== miner ====" +"$ROOT/scripts/miner.sh" stop +echo "==== fault proxy ====" +"$ROOT/scripts/chaos.sh" proxy stop + +if [[ "${1:-}" == "infra" ]]; then + echo "==== infra (docker compose stop, data preserved) ====" + docker compose -p "$COMPOSE_PROJECT" -f "$ROOT/infra/docker-compose.yml" stop +else + echo "==== infra left running (pass 'infra' to also stop it) ====" +fi +echo "done." diff --git a/localnet/scripts/fund.sh b/localnet/scripts/fund.sh new file mode 100755 index 000000000..ce4d0bd3f --- /dev/null +++ b/localnet/scripts/fund.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Fund every role on BTC regtest (P2WSH funding address) and on GOAT (anvil). +# Idempotent: skips a role already funded above its threshold. Mines 1 block to +# confirm the BTC sends. GOAT registrations/stake are done by the contracts agent; +# this only tops up gas + BTC spending funds. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +set -a; source "$STATE_DIR/keys.env"; set +a + +RPC="$GOAT_CHAIN_URL" + +btc_utxos() { curl -s --max-time 5 "$BTC_CHAIN_URL/address/$1/utxo"; } +btc_count() { btc_utxos "$1" | jq 'length' 2>/dev/null || echo 0; } +btc_sats() { btc_utxos "$1" | jq '[.[].value] | add // 0' 2>/dev/null || echo 0; } + +echo "==== GOAT funding (faucet = anvil acct0) ====" +for name in DEPLOYER C0 C1 OP0 VF0 WT0 USER0; do + var="${name}_GOAT_ADDRESS"; addr="${!var}" + wei=$($CAST balance "$addr" --rpc-url "$RPC" 2>/dev/null || echo 0) + eth=$($CAST from-wei "${wei:-0}") + if awk "BEGIN{exit !(${eth%%.*}+0 < $GOAT_FUND_MIN_ETH)}"; then + $CAST send "$addr" --value "${GOAT_FUND_ETH}ether" --private-key "$ANVIL_FAUCET_KEY" --rpc-url "$RPC" >/dev/null + echo " funded $name $addr (+${GOAT_FUND_ETH} ETH)" + else + echo " skip $name $addr (${eth} ETH)" + fi +done + +echo "==== BTC funding (miner wallet -> P2WSH funding addresses) ====" +BTCLI="$BCLI -rpcwallet=$BTC_RPC_WALLET" +sent=0 + +fund_btc() { # $1 addr $2 amount(btc) $3 label + $BTCLI sendtoaddress "$1" "$2" >/dev/null && { echo " sent $2 BTC -> $3 ($1)"; sent=1; } +} + +# operator: OP_BTC_UTXOS separate UTXOs of OP_BTC_EACH +opc=$(btc_count "$OP0_BTC_FUNDING_ADDRESS_REGTEST") +if (( opc < OP_BTC_UTXOS )); then + for _ in $(seq 1 $((OP_BTC_UTXOS - opc))); do fund_btc "$OP0_BTC_FUNDING_ADDRESS_REGTEST" "$OP_BTC_EACH" operator_0; done +else echo " skip operator_0 (already $opc UTXOs)"; fi + +fund_role_min() { # $1 addr $2 min(btc) $3 label + local sats; sats=$(btc_sats "$1"); local need; need=$(awk "BEGIN{print int($2*100000000)}") + if (( sats < need )); then fund_btc "$1" "$2" "$3"; else echo " skip $3 (already $sats sats)"; fi +} +fund_role_min "$VF0_BTC_FUNDING_ADDRESS_REGTEST" "$VF_BTC" verifier_0 +fund_role_min "$WT0_BTC_FUNDING_ADDRESS_REGTEST" "$WT_BTC" watchtower_0 +fund_role_min "$USER0_BTC_FUNDING_ADDRESS_REGTEST" "$USER_BTC" user_0 +# committee_0/1 need no BTC. + +if (( sent )); then + ADDR=$($BTCLI getnewaddress bvlocal-miner bech32); $BCLI generatetoaddress 1 "$ADDR" >/dev/null + echo " mined 1 block to confirm; height=$($BCLI getblockcount)" +fi + +echo "==== balances ====" +for pair in "OP0:operator_0" "VF0:verifier_0" "WT0:watchtower_0" "USER0:user_0"; do + p="${pair%%:*}"; role="${pair##*:}"; var="${p}_BTC_FUNDING_ADDRESS_REGTEST"; a="${!var}" + printf ' %-13s %s sats in %s UTXOs\n' "$role" "$(btc_sats "$a")" "$(btc_count "$a")" +done diff --git a/localnet/scripts/gen-node-envs.sh b/localnet/scripts/gen-node-envs.sh new file mode 100755 index 000000000..a130e394b --- /dev/null +++ b/localnet/scripts/gen-node-envs.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Generate $LOCALNET_DATA/nodes//.env for all 5 roles (and state/cli/.env for the CLIs) from: +# scripts/localnet.env (shared scalars) state/contracts.json state/subgraph.json +# state/keys.env (identities) +# Idempotent: safe to re-run after a contracts/subgraph redeploy. MOCK_* delay vars +# are deliberately NOT written here -- they live only in scripts/localnet.env and are +# exported by scripts/nodes.sh at launch (process env wins over .env), so they stay +# tunable in one place. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +set -a; source "$STATE_DIR/keys.env"; set +a + +CONTRACTS="$STATE_DIR/contracts.json" +SUBGRAPH="$STATE_DIR/subgraph.json" +[[ -f "$CONTRACTS" ]] || { echo "missing $CONTRACTS (run contracts-local/deploy-local.sh)" >&2; exit 1; } +[[ -f "$SUBGRAPH" ]] || { echo "missing $SUBGRAPH (run subgraph/deploy-local.sh)" >&2; exit 1; } + +GW=$(jq -r '.node_env.GOAT_GATEWAY_CONTRACT_ADDRESS' "$CONTRACTS") +SWAP=$(jq -r '.node_env.GOAT_SWAP_CONTRACT_ADDRESS' "$CONTRACTS") +GW_FROM=$(jq -r '.node_env.GOAT_GATEWAY_EVENT_FILTER_FROM' "$CONTRACTS") +GW_URL=$(jq -r '.node_env.GOAT_GATEWAY_EVENT_THE_GRAPH_URL' "$SUBGRAPH") +SWAP_URL=$(jq -r '.node_env.GOAT_SWAP_EVENT_THE_GRAPH_URL' "$SUBGRAPH") + +[[ "$GW" =~ ^0x[0-9a-fA-F]{40}$ ]] || { echo "bad gateway address '$GW'" >&2; exit 1; } + +# role -> key prefix in keys.env +declare -A PFX=( [committee_0]=C0 [committee_1]=C1 [operator_0]=OP0 [verifier_0]=VF0 [watchtower_0]=WT0 ) + +# Common header shared by every role .env. +common() { + local role="$1" + cat < "$NODES_DIR/$role/.env" +} + +gen_operator() { + local role=operator_0 p=OP0 + { common "$role" + echo + echo "# actor" + echo "ACTOR=Operator" + echo "EXTERNAL_SOCKET_ADDR=127.0.0.1:$((RPC_PORT_BASE+2))" + echo + echo "# identity" + echo "BITVM_SECRET=${OP0_BITVM_SECRET}" + echo "PEER_KEY=${OP0_PEER_KEY}" + echo "GOAT_PRIVATE_KEY=${OP0_GOAT_PRIVATE_KEY}" + echo + echo "# proof / soldering (mock)" + echo "GOAT_PROOF_BUILD_URL=$GOAT_PROOF_BUILD_URL" + echo "SOLDERING_PROOF_PAYLOAD_STORE_PATH=$SOLDERING_PROOF_PAYLOAD_STORE_PATH" + echo "OPERATOR_VK_HASH=$OPERATOR_VK_HASH" + echo "OPERATOR_ZKM_VERSION=$OPERATOR_ZKM_VERSION" + echo "ZKM_IMM_WRAP_VK=$ZKM_IMM_WRAP_VK" + echo "WATCHTOWER_PROOF_WAIT_SECS=$WATCHTOWER_PROOF_WAIT_SECS" + echo "OPERATOR_PROOF_WAIT_SECS=$OPERATOR_PROOF_WAIT_SECS" + echo "ENABLE_BABE_SETUP_STATE_CLEANUP=true" + } > "$NODES_DIR/$role/.env" +} + +gen_verifier() { + local role=verifier_0 p=VF0 + { common "$role" + echo + echo "# actor" + echo "ACTOR=Verifier" + echo + echo "# identity (GOAT key required: Challenge tx embeds the verifier GOAT addr in OP_RETURN)" + echo "BITVM_SECRET=${VF0_BITVM_SECRET}" + echo "PEER_KEY=${VF0_PEER_KEY}" + echo "GOAT_PRIVATE_KEY=${VF0_GOAT_PRIVATE_KEY}" + echo + echo "# proof / soldering (mock)" + echo "SOLDERING_PROOF_PAYLOAD_STORE_PATH=$SOLDERING_PROOF_PAYLOAD_STORE_PATH" + echo "OPERATOR_VK_HASH=$OPERATOR_VK_HASH" + echo "OPERATOR_ZKM_VERSION=$OPERATOR_ZKM_VERSION" + echo "ZKM_IMM_WRAP_VK=$ZKM_IMM_WRAP_VK" + echo "ENABLE_BABE_SETUP_STATE_CLEANUP=true" + } > "$NODES_DIR/$role/.env" +} + +gen_watchtower() { + local role=watchtower_0 p=WT0 + { common "$role" + echo + echo "# actor" + echo "ACTOR=Watchtower" + echo + echo "# identity (GOAT key not required by code, provided for convenience)" + echo "BITVM_SECRET=${WT0_BITVM_SECRET}" + echo "PEER_KEY=${WT0_PEER_KEY}" + echo "GOAT_PRIVATE_KEY=${WT0_GOAT_PRIVATE_KEY}" + echo + echo "# proof (mock)" + echo "GOAT_PROOF_BUILD_URL=$GOAT_PROOF_BUILD_URL" + echo "WATCHTOWER_PROOF_WAIT_SECS=$WATCHTOWER_PROOF_WAIT_SECS" + } > "$NODES_DIR/$role/.env" +} + +mkdir -p "$NODES_DIR"/{committee_0,committee_1,operator_0,verifier_0,watchtower_0} "$STATE_DIR/cli" +gen_committee committee_0 true +gen_committee committee_1 "" +gen_operator +gen_verifier +gen_watchtower + +chmod 600 "$NODES_DIR"/*/.env + +# CLI identities for pegin-request / pegout / challenge (sourced by the e2e flows): the user, +# the operator and the verifier. Same test-only keys as the nodes. +gen_cli() { # $1 file $2 key prefix + { echo "# $STATE_DIR/cli/$1 -- GENERATED by scripts/gen-node-envs.sh (TEST-ONLY keys)" + echo "export GOAT_CHAIN_URL=$GOAT_CHAIN_URL" + echo "export GOAT_GATEWAY_CONTRACT_ADDRESS=$GW" + echo "export BITCOIN_NETWORK=$BITCOIN_NETWORK" + local s="${2}_BITVM_SECRET" k="${2}_GOAT_PRIVATE_KEY" a="${2}_GOAT_ADDRESS" + echo "export BITVM_SECRET=${!s}" + echo "export GOAT_PRIVATE_KEY=${!k}" + echo "export GOAT_ADDRESS=${!a}" + } > "$STATE_DIR/cli/$1" +} +gen_cli user.env USER0; gen_cli operator.env OP0; gen_cli verifier.env VF0 +chmod 600 "$STATE_DIR"/cli/*.env +echo "Generated .env for: committee_0 committee_1 operator_0 verifier_0 watchtower_0" +echo " gateway=$GW filter_from=$GW_FROM" +echo " gateway_subgraph=$GW_URL" diff --git a/localnet/scripts/install-bins.sh b/localnet/scripts/install-bins.sh new file mode 100755 index 000000000..aff099058 --- /dev/null +++ b/localnet/scripts/install-bins.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copy the freshly built (mock-enabled) release binaries into $LOCALNET_DATA/bin +# so a later rebuild in target/release never swaps the binary under a running node. +# Nodes/scripts always exec from $BIN_DIR, never from target/release. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" + +SRC="${CARGO_TARGET_DIR:-$REPO_ROOT/target}/release" +mkdir -p "$BIN_DIR" + +BINS=(bitvm-noded mock-proof-builder rpc-sign pegin-request pegout challenge verifier-challenge) + +missing=0 +for b in "${BINS[@]}"; do + if [[ ! -x "$SRC/$b" ]]; then + echo "MISSING: $SRC/$b (run localnet/build.sh)" >&2 + missing=1 + fi +done +[[ $missing -eq 0 ]] || { echo "Refusing to install: binaries missing." >&2; exit 1; } + +for b in "${BINS[@]}"; do + # Copy to a temp name then atomic-rename, so a running node keeps its old inode. + cp -f "$SRC/$b" "$BIN_DIR/.$b.new" + chmod +x "$BIN_DIR/.$b.new" + mv -f "$BIN_DIR/.$b.new" "$BIN_DIR/$b" + printf ' installed %-20s <- %s\n' "$b" "$(stat -c '%y' "$SRC/$b" | cut -d. -f1)" +done + +echo "Installed ${#BINS[@]} binaries into $BIN_DIR" +"$BIN_DIR/bitvm-noded" --help >/dev/null 2>&1 && echo "bitvm-noded runs OK" || echo "WARN: bitvm-noded --help failed" >&2 diff --git a/localnet/scripts/localnet.env b/localnet/scripts/localnet.env new file mode 100644 index 000000000..dd28bee11 --- /dev/null +++ b/localnet/scripts/localnet.env @@ -0,0 +1,173 @@ +# ============================================================================ +# localnet/scripts/localnet.env -- single source of shared, tunable config for +# the in-repo BitVM2 localnet (bitcoin regtest + anvil + local graph-node, five +# mock-gc nodes, everything on 127.0.0.1). +# +# Sourced (bash) by every script under localnet/scripts, localnet/e2e and the +# contracts / subgraph deployers. Two roots: +# LOCALNET_ROOT the code: this directory's parent (scripts, e2e, compose, ...) +# LOCALNET_DATA the state: bin/ state/ nodes/ data/ logs/ run/ tmp/ results/ +# default localnet/.local (git-ignored); point it at another +# directory to run a second, independent localnet. +# Everything else that must differ between two localnets on one host derives +# from PORT_BASE (all ports are PORT_BASE + a fixed offset) and COMPOSE_PROJECT +# (container, network and iptables chain names). +# +# Per-DATA overrides: $LOCALNET_DATA/localnet.override.env (sourced last; use it +# to adopt an existing localnet, e.g. ANVIL_CHAIN_ID=48815 for a chain created +# before the chain id changed). Runtime knobs set by the e2e suite live in +# $LOCALNET_DATA/run/tuning.env (sourced by nodes.sh at launch, see e2e/lib.sh). +# +# Contract addresses / subgraph URLs are NOT here -- they are read live from +# $LOCALNET_DATA/state/contracts.json + state/subgraph.json so a redeploy never +# leaves this file stale. Tune a value here, then +# scripts/gen-node-envs.sh # rebake nodes//.env (static scalars) +# scripts/nodes.sh restart # nodes.sh also re-exports MOCK_*/RUST_LOG +# ============================================================================ + +# ---- roots ------------------------------------------------------------------- +export LOCALNET_ROOT="${LOCALNET_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +export REPO_ROOT="${REPO_ROOT:-$(cd "$LOCALNET_ROOT/.." && pwd)}" +export LOCALNET_DATA="${LOCALNET_DATA:-$LOCALNET_ROOT/.local}" +export BIN_DIR="$LOCALNET_DATA/bin" +export STATE_DIR="$LOCALNET_DATA/state" +export NODES_DIR="$LOCALNET_DATA/nodes" +export DATA_DIR="$LOCALNET_DATA/data" +export LOGS_DIR="$LOCALNET_DATA/logs" +export RUN_DIR="$LOCALNET_DATA/run" +export TMP_DIR="$LOCALNET_DATA/tmp" +export RESULTS_DIR="$LOCALNET_DATA/results" +# Shared directory the Operator and every Verifier read/write soldering payloads. +export SOLDERING_PROOF_PAYLOAD_STORE_PATH="$STATE_DIR/soldering-store" + +# ---- naming / ports ------------------------------------------------------------ +export COMPOSE_PROJECT="${COMPOSE_PROJECT:-bvlocal}" # containers are $COMPOSE_PROJECT- +export PORT_BASE="${PORT_BASE:-22000}" # every port below is PORT_BASE + offset +export BTC_RPC_PORT=$((PORT_BASE + 443)) # bitcoind regtest RPC +export ESPLORA_PORT=$((PORT_BASE + 2)) # electrs esplora HTTP +export ANVIL_PORT=$((PORT_BASE + 545)) +export IPFS_PORT=$((PORT_BASE + 501)) +export GRAPH_QUERY_PORT=$((PORT_BASE + 800)) +export GRAPH_ADMIN_PORT=$((PORT_BASE + 820)) +export GRAPH_INDEX_PORT=$((PORT_BASE + 830)) +export GRAPH_METRICS_PORT=$((PORT_BASE + 840)) +# role index i in committee_0,committee_1,operator_0,verifier_0,watchtower_0 +export RPC_PORT_BASE=$((PORT_BASE + 900)) # rpc = RPC_PORT_BASE + i +export P2P_PORT_BASE=$((PORT_BASE + 910)) # p2p = P2P_PORT_BASE + i +export METRICS_PORT_BASE=$((PORT_BASE + 920)) # metrics = METRICS_PORT_BASE + i +export PROOF_BUILDER_PORT=$((PORT_BASE + 950)) +export PROXY_PORT_BASE=$((PORT_BASE + 960)) # faultproxy: + role*4 + backend (btc,goat,graph,proof) +export CHAOS_CTRL_PORT=$((PORT_BASE + 989)) # faultproxy control API +export KEYCHECK_PORT=$((PORT_BASE + 999)) # throwaway node used by tools/gen-keys.sh +export LOCALNET_UID="${LOCALNET_UID:-$(id -u)}" +export LOCALNET_GID="${LOCALNET_GID:-$(id -g)}" + +# ---- infra endpoints ----------------------------------------------------------- +export BTC_CHAIN_URL="http://127.0.0.1:$ESPLORA_PORT" +export GOAT_CHAIN_URL="http://127.0.0.1:$ANVIL_PORT" +export PROOF_BUILDER_ADDR="127.0.0.1:$PROOF_BUILDER_PORT" +export GOAT_PROOF_BUILD_URL="http://$PROOF_BUILDER_ADDR" # mock-proof-builder (path-less) +export GRAPH_ADMIN="http://127.0.0.1:$GRAPH_ADMIN_PORT" +export GRAPH_IPFS="http://127.0.0.1:$IPFS_PORT" +export GRAPH_QUERY_BASE="http://127.0.0.1:$GRAPH_QUERY_PORT" +export GRAPH_INDEX="http://127.0.0.1:$GRAPH_INDEX_PORT" +export SUBGRAPH_NAME="${SUBGRAPH_NAME:-bitvm_gateway_local}" +export BITCOIN_NETWORK=regtest +export ANVIL_CHAIN_ID="${ANVIL_CHAIN_ID:-48816}" # == Constants.CHAINID_TESTNET, so the stock contracts accept it +export ANVIL_BLOCK_TIME="${ANVIL_BLOCK_TIME:-2}" +# bitcoind CLI prefix (regtest) +export BCLI="docker exec $COMPOSE_PROJECT-bitcoind bitcoin-cli -regtest -datadir=/bitcoin -rpcuser=bvlocal -rpcpassword=bvlocal" +export BTC_RPC_WALLET=miner + +# ---- images (digests of the images this localnet was last validated with) ------- +export BITCOIND_IMAGE="${BITCOIND_IMAGE:-bitcoin/bitcoin:29}" +export ELECTRS_IMAGE="${ELECTRS_IMAGE:-mempool/electrs@sha256:de9dd2f5a328d5f464936dc4128d848f9b8bf72221ab3c9b8e2949df50899b74}" # mempool/electrs:latest 2026-10 +export FOUNDRY_IMAGE="${FOUNDRY_IMAGE:-ghcr.io/foundry-rs/foundry@sha256:043752653d5be351c71709091b3db97c4421c907eb40ea294195e7f532aadf46}" # foundry 1.5.1-stable +export POSTGRES_IMAGE="${POSTGRES_IMAGE:-postgres:16}" +export IPFS_IMAGE="${IPFS_IMAGE:-ipfs/kubo:v0.17.0}" +export GRAPH_NODE_IMAGE="${GRAPH_NODE_IMAGE:-graphprotocol/graph-node@sha256:b0436347fb24f9ae45b6d3959cac97bf60b3238ad1633a875c118ff86a07a0d3}" # graph-node:latest 2026-10 +export NODE_IMAGE="${NODE_IMAGE:-node@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32}" # node:22-alpine (graph-cli build) + +# ---- tools: forge always runs in the foundry container; cast uses the host binary +# when one is installed (much faster per call) and falls back to the container. +export FORGE="$LOCALNET_ROOT/tools/forge.sh" +if [[ -z "${CAST:-}" ]]; then + if command -v cast >/dev/null 2>&1; then CAST=$(command -v cast) + elif [[ -x "$HOME/.foundry/bin/cast" ]]; then CAST="$HOME/.foundry/bin/cast" + else CAST="$LOCALNET_ROOT/tools/cast.sh"; fi +fi +export CAST + +# ---- p2p / protocol ------------------------------------------------------------ +# Bind the p2p listener to loopback only (shared host). +export P2P_LISTEN_IP=127.0.0.1 +# Protocol name distinct from every public bitvm net (two localnets on one host only ever dial +# their own bootnodes, so they never mesh either). +export PROTO_NAME="${PROTO_NAME:-bitvm_localnet}" +export MIN_REQUIRED_VERIFIER=1 + +# ---- shared node constants ----------------------------------------------------- +export RUST_LOG="${RUST_LOG:-info}" +# Must be identical on every node AND on the mock proof builder (arbitrary 64-hex). +export GENESIS_SEQUENCER_COMMIT_TXID=662757d502c483f42a6ef7af8c0ba9cd14e28c6d31100f1161a45bf8d500b330 +# The mock builder and nodes must use the same program identity and Ziren version. +# These localnet values match the mock builder defaults, not the real testnet program. +export OPERATOR_VK_HASH=0x00ba1ff974eb6e5890f238f8a11c1aeff2d5f4a68a860274bcb602c3c5c681b7 +export OPERATOR_ZKM_VERSION=v2.0.0 +export ZKM_IMM_WRAP_VK=1 +# Small proof-wait retry windows (default 60). +export WATCHTOWER_PROOF_WAIT_SECS=60 +export OPERATOR_PROOF_WAIT_SECS=60 + +# ---- contracts (fresh deploys only) ---------------------------------------------- +# Gateway.responseWindowBlocks: upstream initialises 200 (~10 min of 3 s goat blocks); on +# 2 s anvil blocks that is ~400 s during which every peg-in stalls on WindowNotExpired. +# GatewayLocal lets the committee lower it right after the deploy. +export RESPONSE_WINDOW_BLOCKS="${RESPONSE_WINDOW_BLOCKS:-50}" + +# ---- MOCK GC delays (seconds, fractional OK; read at call time) ------------------ +# Heavy ops (setup/solder/verify/builder-init) are UNCAPPED. challenge/eval run +# inline on the p2p-inbox worker and are HARD-CAPPED at 60s in the binary; keep +# them to a few seconds (well under the block-mining cadence) or the honest +# operator loses the WronglyChallenged race. +export MOCK_GC_BUILDER_INIT_DELAY_SECS=2 +export MOCK_GC_SETUP_DELAY_SECS=15 +export MOCK_GC_SOLDER_DELAY_SECS=15 +export MOCK_GC_VERIFY_DELAY_SECS=15 +export MOCK_GC_CHALLENGE_DELAY_SECS=1 +export MOCK_GC_EVAL_DELAY_SECS=1 +# Fallback for any MOCK_GC op whose specific var is unset: +export MOCK_GC_DELAY_SECS=0 +# Dispute policy: valid => happy path (Assert->Take2). The e2e suite switches the +# operator's + verifier's copy to always-invalid through run/tuning.env. +export MOCK_GC_PROOF_VERDICT=valid +export MOCK_GC_VERIFY_OPENINGS=1 + +# ---- MOCK proof builder delays (seconds) ------------------------------------------- +export MOCK_PROOF_DELAY_SECS=0 +export MOCK_PROOF_OPERATOR_DELAY_SECS=30 +export MOCK_PROOF_WATCHTOWER_DELAY_SECS=30 +export MOCK_PROOF_RESPONSE_LATENCY_MS=0 + +# ---- regtest miner ----------------------------------------------------------------- +# 1 block every MINE_INTERVAL_SECS. The node assumes ~60s regtest cadence; do not +# mine much faster or CSV races (1-4 block windows) are lost. +export MINE_INTERVAL_SECS=60 + +# ---- funding (scripts/fund.sh) ----------------------------------------------------- +# anvil dev account 0 = the ONLY GOAT faucet. +export ANVIL_FAUCET_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 +export GOAT_FUND_ETH=100 # top-up target per role +export GOAT_FUND_MIN_ETH=10 # skip GOAT funding a role already above this +# BTC amounts (BTC). committees need none. operator gets OP_BTC_UTXOS UTXOs of OP_BTC_EACH. +export OP_BTC_EACH=0.5 +export OP_BTC_UTXOS=3 +export VF_BTC=0.5 +export WT_BTC=0.2 +export USER_BTC=1.0 + +# ---- per-DATA override (adopting an existing localnet, local experiments) -------- +if [[ -f "$LOCALNET_DATA/localnet.override.env" ]]; then + # shellcheck disable=SC1091 + source "$LOCALNET_DATA/localnet.override.env" +fi diff --git a/localnet/scripts/logq.sh b/localnet/scripts/logq.sh new file mode 100755 index 000000000..4ec83b915 --- /dev/null +++ b/localnet/scripts/logq.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Query the JSON node logs (one file per boot under logs/nodes//). +# scripts/logq.sh [-r role|all] [-s 10m|2h|1d|] [-l INFO|WARN|ERROR] [-e event] [-g substring] [-n N] [-a] [-j] +# -r role (default all) -s since (default 10m; WARN includes ERROR) +# -e exact `event` field -g substring anywhere in the JSON line (ids, txids, ...) +# -n keep the last N lines -a all boot files of the role, not only `current` +# -j raw JSON lines instead of the compact form +# Output: time level role event/target outcome message extra-fields, merged across roles by time. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +ROLES=(committee_0 committee_1 operator_0 verifier_0 watchtower_0) +role=all since=10m level="" event="" grep_="" n="" all=0 raw=0 +while getopts 'r:s:l:e:g:n:aj' o; do case $o in r) role=$OPTARG;; s) since=$OPTARG;; l) level=${OPTARG^^};; e) event=$OPTARG;; g) grep_=$OPTARG;; n) n=$OPTARG;; a) all=1;; j) raw=1;; *) exit 2;; esac; done +case $since in + *s) SINCE=$(date -u -d "-${since%s} sec" +%Y-%m-%dT%H:%M:%S) ;; *m) SINCE=$(date -u -d "-${since%m} min" +%Y-%m-%dT%H:%M:%S) ;; + *h) SINCE=$(date -u -d "-${since%h} hour" +%Y-%m-%dT%H:%M:%S) ;; *d) SINCE=$(date -u -d "-${since%d} day" +%Y-%m-%dT%H:%M:%S) ;; + *) SINCE=$since ;; +esac +files_of() { local d="$LOGS_DIR/nodes/$1"; if (( all )); then ls "$d"/*.log 2>/dev/null; else readlink -f "$d/current" 2>/dev/null; fi; } +targets=("${ROLES[@]}"); [[ $role != all ]] && targets=("$role") +{ +for r in "${targets[@]}"; do + for f in $(files_of "$r"); do + [[ -n $grep_ ]] && src="grep -F -- \"$grep_\" \"$f\"" || src="cat \"$f\"" + eval "$src" | jq -R -r --arg since "$SINCE" --arg lv "$level" --arg e "$event" --arg role "$r" --argjson raw "$raw" ' + (try fromjson catch empty) + | select(type=="object" and (.timestamp // "") >= $since) + | select($lv=="" or ($lv=="WARN" and (.level=="WARN" or .level=="ERROR")) or .level==$lv) + | select($e=="" or .event==$e) + | if $raw==1 then tostring else + [ .timestamp, .level, $role, (.event // ((.target // "?")|split("::")|last)), (.outcome // "-"), (.message // ""), + ( to_entries | map(select(.key|IN("timestamp","level","message","target","span","spans","event","outcome")|not)) + | map("\(.key)=\(.value|tostring|.[0:80])") | join(" ") ) ] | @tsv end' 2>/dev/null + done +done +} | sort | { if [[ -n $n ]]; then tail -n "$n"; else cat; fi; } | { if (( raw )); then cat; else awk -F'\t' '{printf "%s %-5s %-12s %-28s %-12s %s %s\n", substr($1,12,12), $2, $3, substr($4,1,28), $5, $6, $7}'; fi; } diff --git a/localnet/scripts/miner.sh b/localnet/scripts/miner.sh new file mode 100755 index 000000000..4d02474bf --- /dev/null +++ b/localnet/scripts/miner.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Background regtest miner: mines 1 block every MINE_INTERVAL_SECS to the miner +# wallet. scripts/miner.sh start|stop|status|once +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +PIDF="$RUN_DIR/miner.pid"; LOG="$LOGS_DIR/miner.log"; mkdir -p "$RUN_DIR" +LOOP="$ROOT/scripts/_miner-loop.sh" + +is_ours() { local pid="$1"; [[ -n "$pid" && -d "/proc/$pid" ]] && + tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -q "_miner-loop.sh"; } +pid_of() { [[ -f "$PIDF" ]] && cat "$PIDF" || echo ""; } + +case "${1:-status}" in + start) + pid=$(pid_of); if is_ours "$pid"; then echo "miner already running (pid $pid)"; exit 0; fi + nohup bash "$LOOP" >> "$LOG" 2>&1 & + echo $! > "$PIDF"; sleep 1 + pid=$(pid_of) + if is_ours "$pid"; then echo "miner started (pid $pid, every ${MINE_INTERVAL_SECS}s) -> $LOG" + else echo "miner FAILED to start (see $LOG)"; tail -3 "$LOG" 2>/dev/null; exit 1; fi ;; + stop) + pid=$(pid_of) + if is_ours "$pid"; then kill -TERM "$pid" 2>/dev/null || true; sleep 1 + is_ours "$pid" && kill -KILL "$pid" 2>/dev/null || true; echo "miner stopped" + else echo "miner not running"; fi + rm -f "$PIDF" ;; + once) + ADDR=$($BCLI -rpcwallet="$BTC_RPC_WALLET" getnewaddress bvlocal-miner bech32) + $BCLI generatetoaddress "${2:-1}" "$ADDR" >/dev/null + echo "mined ${2:-1} block(s); height=$($BCLI getblockcount)" ;; + status) + pid=$(pid_of) + if is_ours "$pid"; then echo "miner up (pid $pid), tip=$($BCLI getblockcount 2>/dev/null)" + else echo "miner down, tip=$($BCLI getblockcount 2>/dev/null)"; fi ;; + *) echo "usage: $0 start|stop|status|once [n]" >&2; exit 2 ;; +esac diff --git a/localnet/scripts/nodes.sh b/localnet/scripts/nodes.sh new file mode 100755 index 000000000..90740b0e5 --- /dev/null +++ b/localnet/scripts/nodes.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Manage the 5 bitvm-noded processes of the localnet. +# scripts/nodes.sh start|stop|restart|status [role...] +# roles: committee_0 committee_1 operator_0 verifier_0 watchtower_0 (default: all) +# +# Safety: only ever signals a PID recorded in run/nodes/.pid whose /proc +# cmdline actually points at $BIN_DIR/bitvm-noded -- never a stray bitvm-noded of +# another user. Graceful SIGTERM, wait, then SIGKILL as a last resort. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" + +ROLES=(committee_0 committee_1 operator_0 verifier_0 watchtower_0) +PIDDIR="$RUN_DIR/nodes"; mkdir -p "$PIDDIR" "$LOGS_DIR/nodes" +NODE_BIN="$BIN_DIR/bitvm-noded" +TERM_WAIT=25 # seconds to wait for graceful shutdown before SIGKILL + +role_index() { local r="$1" i; for i in "${!ROLES[@]}"; do [[ "${ROLES[$i]}" == "$r" ]] && { echo "$i"; return; }; done; echo -1; } +rpc_port() { echo $((RPC_PORT_BASE + $(role_index "$1"))); } +p2p_port() { echo $((P2P_PORT_BASE + $(role_index "$1"))); } +metrics_port(){ echo $((METRICS_PORT_BASE + $(role_index "$1"))); } + +peer_id() { jq -r ".roles.$1.peer_id" "$STATE_DIR/keys.json"; } + +bootnodes_for() { # echo repeated --bootnodes flags for a role + local role="$1"; local c0 c1 + c0="/ip4/127.0.0.1/tcp/$((P2P_PORT_BASE+0))/p2p/$(peer_id committee_0)" + c1="/ip4/127.0.0.1/tcp/$((P2P_PORT_BASE+1))/p2p/$(peer_id committee_1)" + case "$role" in + committee_0) : ;; # bootnode: none + committee_1) printf ' --bootnodes %s' "$c0" ;; + *) printf ' --bootnodes %s --bootnodes %s' "$c0" "$c1" ;; + esac +} + +pid_of() { local f="$PIDDIR/$1.pid"; [[ -f "$f" ]] && cat "$f" || echo ""; } + +is_ours() { # $1 pid -> 0 if a live bitvm-noded launched from our BIN_DIR + local pid="$1" + [[ -n "$pid" && -d "/proc/$pid" ]] || return 1 + tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -q "$NODE_BIN" || return 1 + return 0 +} + +running() { local pid; pid=$(pid_of "$1"); is_ours "$pid"; } + +start_one() { + local role="$1" + if running "$role"; then echo " $role already running (pid $(pid_of "$role"))"; return; fi + [[ -f "$NODES_DIR/$role/.env" ]] || { echo " $role: missing nodes/$role/.env (run scripts/gen-node-envs.sh)" >&2; return 1; } + [[ -x "$NODE_BIN" ]] || { echo " $NODE_BIN missing (run scripts/install-bins.sh)" >&2; return 1; } + local rpc p2p met; rpc=$(rpc_port "$role"); p2p=$(p2p_port "$role"); met=$(metrics_port "$role") + local boots; boots=$(bootnodes_for "$role") + # One log file per boot: logs/nodes//.log, with logs/nodes//current + # (and the legacy logs/nodes/.log) pointing at the newest one. + local logdir="$LOGS_DIR/nodes/$role"; mkdir -p "$logdir" + local log="$logdir/$(date -u +%Y%m%dT%H%M%SZ).log" + : > "$log"; ln -sfn "$log" "$logdir/current"; ln -sfn "$role/current" "$LOGS_DIR/nodes/$role.log" + # MOCK_* + RUST_LOG come from the sourced localnet.env (process env wins over .env), + # so tuning localnet.env + restart applies without regenerating .env. + # run/tuning.env (written by the e2e suite, e2e/lib.sh set_tuning) overrides single MOCK_* knobs; + # run/overrides/.env (written by scripts/chaos.sh attach) is sourced LAST so a + # single role can be pointed at the fault proxies. + ( cd "$NODES_DIR/$role" + if [[ -f "$RUN_DIR/tuning.env" ]]; then set -a; source "$RUN_DIR/tuning.env"; set +a; fi + if [[ -f "$RUN_DIR/overrides/$role.env" ]]; then set -a; source "$RUN_DIR/overrides/$role.env"; set +a; fi + # shellcheck disable=SC2086 + nohup "$NODE_BIN" \ + --rpc-addr "127.0.0.1:$rpc" \ + --metrics-addr "127.0.0.1:$met" \ + --db-path "sqlite:$NODES_DIR/$role/bitvm-node.db" \ + --p2p-port "$p2p" $boots \ + >> "$log" 2>&1 & + echo $! > "$PIDDIR/$role.pid" ) + sleep 0.5 + if running "$role"; then + echo " started $role pid=$(pid_of "$role") rpc=127.0.0.1:$rpc p2p=$p2p metrics=$met" + else + echo " FAILED $role (see $log)"; tail -3 "$log" 2>/dev/null | sed 's/^/ /' + fi +} + +stop_one() { + local role="$1" pid; pid=$(pid_of "$role") + if ! is_ours "$pid"; then echo " $role not running"; rm -f "$PIDDIR/$role.pid"; return; fi + echo " stopping $role (pid $pid) ..." + kill -TERM "$pid" 2>/dev/null || true + local i=0 + while (( i < TERM_WAIT )) && is_ours "$pid"; do sleep 1; i=$((i + 1)); done + if is_ours "$pid"; then + echo " still alive after ${TERM_WAIT}s, SIGKILL"; kill -KILL "$pid" 2>/dev/null || true; sleep 1 + fi + rm -f "$PIDDIR/$role.pid" + echo " $role stopped" +} + +status_one() { + local role="$1" pid rpc met; pid=$(pid_of "$role"); rpc=$(rpc_port "$role"); met=$(metrics_port "$role") + local state="DOWN" peers="-" topics="-" bh="-" gh="-" spv="-" ready="-" ovr="" + [[ -f "$RUN_DIR/overrides/$role.env" ]] && ovr=" [proxied]" + if is_ours "$pid"; then + state="up(pid $pid)" + local m; m=$(curl -s --max-time 3 "http://127.0.0.1:$met/metrics" 2>/dev/null || true) + if [[ -n "$m" ]]; then + peers=$(awk '/^bitvm_node_p2p_connected_peers/{print $2; exit}' <<<"$m"); peers=${peers:-?} + topics=$(awk '/^bitvm_node_p2p_required_topics_healthy/{print $2; exit}' <<<"$m"); topics=${topics:-?} + ready=$(awk '/^bitvm_node_ready /{print $2; exit}' <<<"$m"); ready=${ready:-?} + fi + local d; d=$(curl -s --max-time 3 "http://127.0.0.1:$rpc/v1/debug/status" 2>/dev/null || true) + if [[ -n "$d" ]]; then + bh=$(jq -r '.bitcoin_height // .data.bitcoin_height // "?"' <<<"$d" 2>/dev/null || echo "?") + gh=$(jq -r '.goat_height // .data.goat_height // "?"' <<<"$d" 2>/dev/null || echo "?") + spv=$(jq -r '.goat_spv_height // .data.goat_spv_height // "?"' <<<"$d" 2>/dev/null || echo "?") + fi + fi + printf ' %-13s %-14s ready=%-2s peers=%-3s topics_healthy=%-3s btc_h=%-5s goat_h=%-7s spv_h=%-5s%s\n' \ + "$role" "$state" "$ready" "$peers" "$topics" "$bh" "$gh" "$spv" "$ovr" +} + +targets() { if [[ $# -gt 0 ]]; then printf '%s\n' "$@"; else printf '%s\n' "${ROLES[@]}"; fi; } + +cmd="${1:-status}"; shift || true +case "$cmd" in + start) echo "starting nodes:"; for r in $(targets "$@"); do start_one "$r"; sleep 1.5; done ;; + stop) echo "stopping nodes:"; for r in $(targets "$@"); do stop_one "$r"; done ;; + restart) for r in $(targets "$@"); do stop_one "$r"; done; echo "---"; for r in $(targets "$@"); do start_one "$r"; sleep 1.5; done ;; + status) echo "node status:"; for r in $(targets "$@"); do status_one "$r"; done ;; + *) echo "usage: $0 start|stop|restart|status [role...]" >&2; exit 2 ;; +esac diff --git a/localnet/scripts/proof-builder.sh b/localnet/scripts/proof-builder.sh new file mode 100755 index 000000000..88e52a151 --- /dev/null +++ b/localnet/scripts/proof-builder.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Start/stop the mock-proof-builder (serves the operator/watchtower proof HTTP API +# the Operator and Watchtower nodes call). scripts/proof-builder.sh start|stop|status +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" # exports GENESIS_SEQUENCER_COMMIT_TXID + MOCK_PROOF_* + PROOF_BUILDER_ADDR +PIDF="$RUN_DIR/proof-builder.pid"; LOG="$LOGS_DIR/proof-builder.log"; mkdir -p "$RUN_DIR" +BIN="$BIN_DIR/mock-proof-builder" + +is_ours() { local pid="$1"; [[ -n "$pid" && -d "/proc/$pid" ]] && + tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | grep -q "$BIN"; } +pid_of() { [[ -f "$PIDF" ]] && cat "$PIDF" || echo ""; } + +case "${1:-status}" in + start) + pid=$(pid_of); if is_ours "$pid"; then echo "proof-builder already running (pid $pid)"; exit 0; fi + [[ -x "$BIN" ]] || { echo "$BIN missing (run scripts/install-bins.sh)" >&2; exit 1; } + # run/tuning.env (e2e delay tier) overrides the MOCK_PROOF_* knobs of localnet.env + if [[ -f "$RUN_DIR/tuning.env" ]]; then set -a; source "$RUN_DIR/tuning.env"; set +a; fi + nohup "$BIN" --listen "$PROOF_BUILDER_ADDR" >> "$LOG" 2>&1 & + echo $! > "$PIDF"; sleep 1 + pid=$(pid_of) + if is_ours "$pid"; then + hello=$(curl -s --max-time 3 "http://$PROOF_BUILDER_ADDR/" || true) + echo "proof-builder started (pid $pid) on $PROOF_BUILDER_ADDR GET/=\"$hello\"" + else echo "proof-builder FAILED (see $LOG)"; tail -5 "$LOG" 2>/dev/null; exit 1; fi ;; + stop) + pid=$(pid_of) + if is_ours "$pid"; then kill -TERM "$pid" 2>/dev/null || true; sleep 1 + is_ours "$pid" && kill -KILL "$pid" 2>/dev/null || true; echo "proof-builder stopped" + else echo "proof-builder not running"; fi + rm -f "$PIDF" ;; + status) + pid=$(pid_of) + if is_ours "$pid"; then + hello=$(curl -s --max-time 3 "http://$PROOF_BUILDER_ADDR/" || echo "") + echo "proof-builder up (pid $pid) on $PROOF_BUILDER_ADDR GET/=\"$hello\"" + else echo "proof-builder down"; fi ;; + *) echo "usage: $0 start|stop|status" >&2; exit 2 ;; +esac diff --git a/localnet/scripts/reset.sh b/localnet/scripts/reset.sh new file mode 100755 index 000000000..860e82674 --- /dev/null +++ b/localnet/scripts/reset.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# DESTRUCTIVE full (re)build of the localnet under $LOCALNET_DATA. Stops everything, WIPES all +# infra data (bitcoin/electrs/anvil/postgres/ipfs) and every node DB/state, recreates the infra +# from scratch, generates the keys if missing, mines 101 blocks, deploys the contracts and the +# subgraph, funds every role, and starts the miner, proof builder and nodes. +# +# This DELETES on-chain state. Requires explicit confirmation: scripts/reset.sh --yes +# Binaries: builds them (localnet/build.sh) when none are in the cargo target yet; set +# LOCALNET_SKIP_BUILD=1 to refuse instead. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +COMPOSE="docker compose -p $COMPOSE_PROJECT -f $ROOT/infra/docker-compose.yml" +T_START=$(date +%s) +step() { echo "==== $* ($(( $(date +%s) - T_START ))s) ===="; } + +[[ "${1:-}" == "--yes" ]] || { echo "Refusing without --yes. This WIPES all localnet data under $LOCALNET_DATA." >&2; exit 2; } +echo "code $LOCALNET_ROOT"; echo "data $LOCALNET_DATA (project $COMPOSE_PROJECT, ports $PORT_BASE+, anvil chain id $ANVIL_CHAIN_ID)" +mkdir -p "$LOCALNET_DATA" "$STATE_DIR" "$LOGS_DIR" "$RUN_DIR" "$TMP_DIR" "$BIN_DIR" + +step "0. binaries" +TARGET="${CARGO_TARGET_DIR:-$REPO_ROOT/target}/release" +if [[ ! -x "$TARGET/bitvm-noded" || ! -x "$TARGET/mock-proof-builder" ]]; then + [[ "${LOCALNET_SKIP_BUILD:-0}" == 1 ]] && { echo "no binaries in $TARGET and LOCALNET_SKIP_BUILD=1" >&2; exit 1; } + "$ROOT/build.sh" 2>&1 | tee "$LOGS_DIR/build.log" | tail -n 3 +fi +"$ROOT/scripts/install-bins.sh" + +step "1. stop everything" +"$ROOT/scripts/down.sh" || true + +step "2. remove infra containers + wipe data" +$COMPOSE down --remove-orphans || true +for d in bitcoin electrs anvil postgres ipfs; do + rm -rf "${DATA_DIR:?}/$d"; mkdir -p "$DATA_DIR/$d" +done +for r in committee_0 committee_1 operator_0 verifier_0 watchtower_0; do + mkdir -p "$NODES_DIR/$r" + rm -f "$NODES_DIR/$r/"bitvm-node.db* 2>/dev/null || true + rm -rf "$NODES_DIR/$r/cache" "$NODES_DIR/$r/.bitvm-babe-state" 2>/dev/null || true +done +mkdir -p "$SOLDERING_PROOF_PAYLOAD_STORE_PATH"; rm -rf "${SOLDERING_PROOF_PAYLOAD_STORE_PATH:?}"/* 2>/dev/null || true +rm -f "$RUN_DIR/nodes/"*.pid "$RUN_DIR/"*.pid "$RUN_DIR/tuning.env" 2>/dev/null || true +rm -rf "$RUN_DIR/overrides" "$RUN_DIR/e2e" 2>/dev/null || true +rm -f "$STATE_DIR/contracts.json" "$STATE_DIR/subgraph.json" + +step "3. keys" +if [[ -f "$STATE_DIR/keys.json" ]]; then echo " keeping $STATE_DIR/keys.json"; else "$ROOT/tools/gen-keys.sh" | tail -n 2; fi + +step "4. recreate infra" +$COMPOSE up -d +echo " waiting for anvil + bitcoind ..." +for i in $(seq 1 60); do + a=$($CAST chain-id --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null || true) + b=$($BCLI getblockcount 2>/dev/null || true) + [[ -n "$a" && -n "$b" ]] && break + sleep 2 +done +cid=$($CAST chain-id --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null || true) +[[ "$cid" == "$ANVIL_CHAIN_ID" ]] || { echo "anvil not ready / wrong chain id ('$cid')" >&2; exit 1; } + +step "5. bitcoind wallet + 101 blocks" +$BCLI createwallet "$BTC_RPC_WALLET" >/dev/null 2>&1 || $BCLI loadwallet "$BTC_RPC_WALLET" >/dev/null 2>&1 || true +ADDR=$($BCLI -rpcwallet="$BTC_RPC_WALLET" getnewaddress bvlocal-miner bech32) +$BCLI generatetoaddress 101 "$ADDR" >/dev/null +echo " height=$($BCLI getblockcount)" + +step "6. deploy contracts (fund gas + deploy + register + stake)" +bash "$ROOT/contracts-local/deploy-local.sh" +step "7. deploy subgraph" +bash "$ROOT/subgraph/deploy-local.sh" + +step "8. node env files" +"$ROOT/scripts/gen-node-envs.sh" + +step "9. fund roles (BTC + GOAT top-up)" +"$ROOT/scripts/fund.sh" + +step "10. start miner, proof builder, nodes" +"$ROOT/scripts/miner.sh" start +"$ROOT/scripts/proof-builder.sh" start +"$ROOT/scripts/nodes.sh" start + +step "11. settle + status" +sleep 15 +"$ROOT/scripts/status.sh" +echo "reset done in $(( $(date +%s) - T_START ))s" diff --git a/localnet/scripts/status.sh b/localnet/scripts/status.sh new file mode 100755 index 000000000..d43a6d7dd --- /dev/null +++ b/localnet/scripts/status.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# One-shot health overview of the localnet: infra containers, support +# processes, chain heights, subgraph sync, and per-node readiness/peers. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" + +echo "================= localnet status ($COMPOSE_PROJECT, data $LOCALNET_DATA) =================" + +echo "--- infra containers ($COMPOSE_PROJECT) ---" +docker ps --format '{{.Names}}\t{{.Status}}' 2>/dev/null | grep "^$COMPOSE_PROJECT-" | sort | sed 's/^/ /' || echo " (docker unavailable)" + +echo "--- chains ---" +tip=$($BCLI getblockcount 2>/dev/null || echo "?") +echo " bitcoind regtest height : $tip" +if abn=$($CAST block-number --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null); then + echo " anvil block : $abn (chain-id $($CAST chain-id --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null))" +else echo " anvil : unreachable"; fi +sg=$(curl -s --max-time 5 -XPOST -H 'content-type: application/json' \ + -d '{"query":"{indexingStatuses{synced health chains{latestBlock{number} chainHeadBlock{number}}}}"}' \ + $GRAPH_INDEX/graphql 2>/dev/null) +if [[ -n "$sg" ]]; then + echo " gateway subgraph : $(jq -rc '.data.indexingStatuses[0] | "synced=\(.synced) health=\(.health) latest=\(.chains[0].latestBlock.number) head=\(.chains[0].chainHeadBlock.number)"' <<<"$sg" 2>/dev/null || echo "$sg")" +fi + +echo "--- support processes ---" +"$ROOT/scripts/miner.sh" status | sed 's/^/ /' +"$ROOT/scripts/proof-builder.sh" status | sed 's/^/ /' + +echo "--- nodes ---" +"$ROOT/scripts/nodes.sh" status | tail -n +2 +echo "===========================================================" diff --git a/localnet/scripts/trace.sh b/localnet/scripts/trace.sh new file mode 100755 index 000000000..422d2d113 --- /dev/null +++ b/localnet/scripts/trace.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Cross-node timeline of one peg-in instance or one graph. +# scripts/trace.sh [-s 6h] [-n 300] [-a] (-a: all boot logs; default current boots) +# Prints RPC statuses, DB rows (instance / graphs / engine aggregate, jobs, parked inbox, poison) on every node and the +# merged log timeline of every line mentioning the id (and its related graph/instance ids). Copy in tmp/trace-.txt. +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +ROLES=(committee_0 committee_1 operator_0 verifier_0 watchtower_0) +id=${1:?instance or graph id}; shift +since=6h n=300 all="" +while getopts 's:n:a' o; do case $o in s) since=$OPTARG;; n) n=$OPTARG;; a) all=-a;; *) exit 2;; esac; done +out="$TMP_DIR/trace-${id:0:8}.txt" +db() { sqlite3 -readonly -noheader -list -cmd ".timeout 8000" "$NODES_DIR/$1/bitvm-node.db" "$2" 2>/dev/null; } +# uuid text for a column stored either as TEXT or as a 16-byte BLOB +uf() { echo "CASE WHEN typeof($1)='blob' THEN lower(substr(hex($1),1,8)||'-'||substr(hex($1),9,4)||'-'||substr(hex($1),13,4)||'-'||substr(hex($1),17,4)||'-'||substr(hex($1),21)) ELSE $1 END"; } +hexid=$(tr -d - <<<"$id" | tr a-f A-F) +{ +echo "### trace $id ($(date -u +%FT%TZ), tip $($BCLI getblockcount 2>/dev/null))" +# related ids: graphs of the instance, or the instance of the graph +related=$(for r in "${ROLES[@]}"; do db "$r" "select $(uf graph_id) from graph where instance_id='$id' or hex(instance_id)='$hexid'; select $(uf instance_id) from graph where graph_id='$id' or hex(graph_id)='$hexid';"; done | sort -u | grep -av '^$') +ids="$id"; [[ -n $related ]] && ids+=$'\n'"$related" +echo "related ids: $(tr '\n' ' ' <<<"$related")" +echo +echo "### RPC status per node" +for i in "${!ROLES[@]}"; do r=${ROLES[$i]}; p=$((RPC_PORT_BASE+i)) + is=$(curl -s --max-time 5 "http://127.0.0.1:$p/v1/instances/$id" | jq -r '.instance_wrap.instance.status // "-"' 2>/dev/null) + gs=$(curl -s --max-time 5 "http://127.0.0.1:$p/v1/graphs/$id" | jq -r '.graph | "\(.status // "-")/\(.sub_status // "")" ' 2>/dev/null) + printf ' %-13s instance=%-26s graph=%s\n' "$r" "${is:-?}" "${gs:-?}"; done +echo +for r in "${ROLES[@]}"; do + echo "### $r DB" + db "$r" "select ' instance', status, 'btc_h='||btc_height, 'status_updated_at='||status_updated_at, 'goat_tx='||substr(goat_tx_hash,1,10) from instance where instance_id='$id' or hex(instance_id)='$hexid';" | tr '|' ' ' + for g in $ids; do gh=$(tr -d - <<<"$g" | tr a-f A-F); db "$r" "select ' graph', $(uf graph_id), 'slot='||kickoff_index, status, sub_status, 'status_updated_at='||status_updated_at, 'kickoff='||substr(kickoff_txid,1,10) from graph where graph_id='$g' or hex(graph_id)='$gh';" | tr '|' ' '; done + db "$r" "select ' pending_graph_init', $(uf graph_id), operator_pubkey, updated_at from pending_graph_init where hex(instance_id)='$hexid' or instance_id='$id';" | tr '|' ' ' + for g in $ids; do gh=$(tr -d - <<<"$g" | tr a-f A-F) + db "$r" "select ' aggregate', aggregate_kind, status, phase, 'v='||version, 'updated='||updated_at from engine_aggregate where aggregate_id='$g';" | tr '|' ' ' + db "$r" "select ' job', aggregate_kind, phase||'/'||action, state, 'attempts='||attempts, 'next_at='||next_at, 'lease='||lease_until, 'cancel='||cancel_requested, substr(coalesce(last_error,''),1,70), 'updated='||updated_at from engine_job where aggregate_id='$g' order by updated_at;" | tr '|' ' ' + db "$r" "select ' inbox!', kind, state, 'attempts='||attempts, coalesce(waiting_ref,''), substr(coalesce(waiting_detail,last_error,''),1,70) from engine_inbox where aggregate_id='$g' and state!='Applied';" | tr '|' ' ' + db "$r" "select ' POISON', action, 'count='||poison_count, 'blocked_until='||blocked_until, substr(reason,1,70) from engine_aggregate_poison where aggregate_id='$g';" | tr '|' ' ' + done +done +echo +echo "### log timeline (last $n lines, since $since)" +for g in $ids; do "$ROOT/scripts/logq.sh" -s "$since" -g "${g:0:8}" $all; done | grep -av " http_re[sq]" | sort -u | tail -n "$n" +} | tee "$out" +echo "(saved to $out)" diff --git a/localnet/scripts/triage.sh b/localnet/scripts/triage.sh new file mode 100755 index 000000000..e37e465c7 --- /dev/null +++ b/localnet/scripts/triage.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# One-screen "why is this node unhealthy" view. scripts/triage.sh [role|all] [-s 10m] +set -uo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +ROLES=(committee_0 committee_1 operator_0 verifier_0 watchtower_0) +role=${1:-all}; since=${3:-10m}; [[ ${2:-} == -s ]] && since=$3 +role_idx() { local i; for i in "${!ROLES[@]}"; do [[ ${ROLES[$i]} == "$1" ]] && { echo "$i"; return; }; done; echo 0; } +now=$(date +%s) +tip=$($BCLI getblockcount 2>/dev/null || echo "?") +anvil=$($CAST block-number --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null || echo "?") +sg=$(curl -s --max-time 5 -XPOST -H 'content-type: application/json' -d '{"query":"{_meta{block{number}}}"}' $(jq -r .query_url "$STATE_DIR/subgraph.json" 2>/dev/null) | jq -r '.data._meta.block.number // "?"' 2>/dev/null) +echo "truth: btc tip=$tip anvil=$anvil subgraph=$sg miner=$("$ROOT/scripts/miner.sh" status | grep -oE '^miner [a-z]+') proof-builder=$("$ROOT/scripts/proof-builder.sh" status | grep -oE '^proof-builder [a-z]+')" +"$ROOT/scripts/chaos.sh" status | tail -n +2 | grep -v 'overrides:' | grep -v 'rules: 0' | grep -v 'faults: none' | sed 's/^/chaos:/' +metric() { awk -v k="$1" '$1==k{print $2; exit}' <<<"$2"; } +one() { + local r=$1 i; i=$(role_idx "$r"); local rpc=$((RPC_PORT_BASE+i)) met=$((METRICS_PORT_BASE+i)) + local pid; pid=$(cat "$RUN_DIR/nodes/$r.pid" 2>/dev/null) + local ovr=""; [[ -f $RUN_DIR/overrides/$r.env ]] && ovr=" [proxied]" + if [[ -z $pid || ! -d /proc/$pid ]]; then echo "== $r DOWN (no process)"; return; fi + local et rss fds; et=$(ps -o etimes= -p "$pid" | tr -d ' '); rss=$(awk '/VmRSS/{printf "%dMB",$2/1024}' /proc/$pid/status); fds=$(ls /proc/$pid/fd 2>/dev/null | wc -l) + local logf; logf=$(readlink -f "$LOGS_DIR/nodes/$r/current") + local boot; boot=$(jq -r 'select(.event=="service_started") | "\(.timestamp[0:19])Z v\(.version)"' "$logf" 2>/dev/null | head -1) + echo "== $r pid=$pid up=${et}s rss=$rss fds=$fds boot=${boot:-?} log=$(du -h "$logf" | cut -f1)$ovr" + local m; m=$(curl -s --max-time 4 "http://127.0.0.1:$met/metrics") + if [[ -z $m ]]; then echo " metrics: UNREACHABLE"; else + echo " ready=$(metric bitvm_node_ready "$m") event_watch_ready=$(metric bitvm_node_event_watch_ready "$m") event_watch_lag=$(metric bitvm_node_event_watch_lag_blocks "$m") spv_lag=$(metric bitvm_node_spv_lag_blocks "$m") p2p peers=$(metric bitvm_node_p2p_connected_peers "$m") topics_healthy=$(metric bitvm_node_p2p_required_topics_healthy "$m") gas_wei=$(metric bitvm_node_goat_gas_balance_wei "$m") fee_sats=$(metric bitvm_node_fee_wallet_balance_sats "$m")" + echo " backends: $(grep -E '^bitvm_node_(btc|goat)_backend_requests_total' <<<"$m" | sed -E 's/bitvm_node_(btc|goat)_backend_requests_total\{outcome="([a-z]+)"\} ([0-9]+)/\1.\2=\3/' | tr '\n' ' ') btc_last_ok=$(( now - $(metric bitvm_node_btc_backend_last_success_timestamp_seconds "$m" || echo 0) ))s goat_last_ok=$(( now - $(metric bitvm_node_goat_backend_last_success_timestamp_seconds "$m" || echo 0) ))s event_watch_last_ok=$(( now - $(metric bitvm_node_event_watch_last_success_timestamp_seconds "$m" || echo 0) ))s" + echo " stale tasks (>90s): $(grep '^bitvm_node_task_last_success_timestamp_seconds' <<<"$m" | awk -v now="$now" -F'[{}" ]+' '{age=now-$NF; if (age>90) printf "%s=%ds ", $3, age}')" + local al; al=$(grep '^bitvm_node_instance_alerts_total' <<<"$m" | sed -E 's/.*alert="([a-z_]+)"\} ([0-9]+)/\1=\2/' | tr '\n' ' '); [[ -n $al ]] && echo " alerts: $al" + fi + local d; d=$(curl -s --max-time 6 "http://127.0.0.1:$rpc/v1/debug/status") + if [[ -n $d ]]; then + echo " heights: $(jq -r '"btc=\(.bitcoin_height // "null") goat=\(.goat_height // "null") spv=\(.goat_spv_height // "null")"' <<<"$d") (truth btc=$tip anvil=$anvil) engine: $(jq -c '.engine // {}' <<<"$d" | cut -c1-200)" + else echo " rpc: UNREACHABLE"; fi + local e; e=$(curl -s --max-time 6 "http://127.0.0.1:$rpc/v1/debug/engine") + [[ -n $e ]] && jq -r '(.waiting[]? | " WAITING \(.aggregate_kind) \(.aggregate_id[0:8]) \(.kind) \(.waiting_ref // "") \((.waiting_detail // "")[0:80])"), (.poisoned[]? | " POISONED \(.aggregate_kind) \(.aggregate_id[0:8]) \(.action) count=\(.poison_count) blocked_until=\(.blocked_until) \((.reason // "")[0:80])")' <<<"$e" + echo " warn/error last $since (top 8):" + "$ROOT/scripts/logq.sh" -r "$r" -s "$since" -l WARN | awk '{printf "%s %s %s\n", $2, $4, $5}' | sort | uniq -c | sort -rn | head -8 | sed 's/^/ /' + local le; le=$("$ROOT/scripts/logq.sh" -r "$r" -s "$since" -l ERROR -n 1); [[ -n $le ]] && echo " last ERROR: $le"; return 0 +} +if [[ $role == all ]]; then for r in "${ROLES[@]}"; do one "$r"; done; else one "$role"; fi diff --git a/localnet/scripts/up.sh b/localnet/scripts/up.sh new file mode 100755 index 000000000..d8ab5923a --- /dev/null +++ b/localnet/scripts/up.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Bring the whole localnet up: start the infra if it is not running, (re)install binaries, +# regenerate node .env files, then start miner, proof-builder and the 5 nodes. +# Safe/idempotent and state-preserving: an existing localnet (containers, chains, contracts, +# node DBs under $LOCALNET_DATA) is adopted, never rebuilt. Only manages the $COMPOSE_PROJECT +# docker project and processes launched from $BIN_DIR. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +COMPOSE="docker compose -p $COMPOSE_PROJECT -f $ROOT/infra/docker-compose.yml" + +echo "==== 0. layout ====" +echo " code $LOCALNET_ROOT" +echo " data $LOCALNET_DATA (project $COMPOSE_PROJECT, ports $PORT_BASE+)" +[[ -f "$STATE_DIR/keys.json" ]] || { echo " no $STATE_DIR/keys.json: this is a fresh data dir, run scripts/reset.sh --yes first" >&2; exit 1; } +[[ -f "$STATE_DIR/contracts.json" && -f "$STATE_DIR/subgraph.json" ]] || { echo " contracts/subgraph not deployed: run scripts/reset.sh --yes" >&2; exit 1; } +mkdir -p "$LOGS_DIR" "$RUN_DIR" "$TMP_DIR" "$SOLDERING_PROOF_PAYLOAD_STORE_PATH" + +echo "==== 1. infra ====" +need=(bitcoind electrs anvil postgres ipfs graph-node) +missing=0 +for c in "${need[@]}"; do + docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "$COMPOSE_PROJECT-$c" || { echo " $COMPOSE_PROJECT-$c is down"; missing=1; } +done +if (( missing )); then + echo " starting $COMPOSE_PROJECT infra (docker compose up -d) ..." + $COMPOSE up -d +else + echo " all 6 $COMPOSE_PROJECT containers already running" +fi +# wait for anvil + bitcoind to answer +for i in $(seq 1 30); do + a=$($CAST chain-id --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null || true) + b=$($BCLI getblockcount 2>/dev/null || true) + [[ -n "$a" && -n "$b" ]] && break + sleep 2 +done +cid=$($CAST chain-id --rpc-url "$GOAT_CHAIN_URL" 2>/dev/null || true) +echo " anvil chain-id=$cid btc height=$($BCLI getblockcount 2>/dev/null)" +# bitcoind does not reload the miner wallet after its container restarted (down.sh infra, host reboot) +$BCLI loadwallet "$BTC_RPC_WALLET" >/dev/null 2>&1 || true +$BCLI -rpcwallet="$BTC_RPC_WALLET" getwalletinfo >/dev/null 2>&1 || { echo " bitcoind wallet $BTC_RPC_WALLET not available" >&2; exit 1; } +[[ -n "$cid" ]] || { echo " anvil not reachable at $GOAT_CHAIN_URL" >&2; exit 1; } +if [[ "$cid" != "$ANVIL_CHAIN_ID" ]]; then + echo " anvil runs chain id $cid but ANVIL_CHAIN_ID=$ANVIL_CHAIN_ID; set ANVIL_CHAIN_ID=$cid in $LOCALNET_DATA/localnet.override.env to adopt this chain" >&2 + exit 1 +fi + +echo "==== 2. binaries + node env files ====" +"$ROOT/scripts/install-bins.sh" +"$ROOT/scripts/gen-node-envs.sh" + +echo "==== 3. miner ====" +"$ROOT/scripts/miner.sh" start + +echo "==== 4. mock proof builder ====" +"$ROOT/scripts/proof-builder.sh" start + +echo "==== 5. funding ====" +"$ROOT/scripts/fund.sh" + +echo "==== 6. nodes ====" +# nodes attached to the fault proxy (scripts/chaos.sh attach) reach their backends only through it +if ls "$RUN_DIR/overrides"/*.env >/dev/null 2>&1; then "$ROOT/scripts/chaos.sh" proxy start; fi +"$ROOT/scripts/nodes.sh" start +# A node whose DB needs a migration the binary cannot apply exits within seconds. +sleep 5 +dead=() +for r in committee_0 committee_1 operator_0 verifier_0 watchtower_0; do + "$ROOT/scripts/nodes.sh" status "$r" | grep -q 'up(pid' || dead+=("$r") +done +if (( ${#dead[@]} )); then + echo " nodes not running: ${dead[*]}" >&2 + for r in "${dead[@]}"; do + if grep -qiE 'migrat|VersionMismatch|schema' "$LOGS_DIR/nodes/$r/current" 2>/dev/null; then + echo " $r: its DB does not fit this binary (migration failed) -- a fresh state is needed: scripts/reset.sh --yes" >&2 + fi + tail -n 3 "$LOGS_DIR/nodes/$r/current" 2>/dev/null | cut -c1-200 | sed 's/^/ /' >&2 + done + exit 1 +fi + +echo "==== 7. settling (15s) then status ====" +sleep 15 +"$ROOT/scripts/status.sh" diff --git a/localnet/subgraph/deploy-local.sh b/localnet/subgraph/deploy-local.sh new file mode 100755 index 000000000..3da4c07ae --- /dev/null +++ b/localnet/subgraph/deploy-local.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Deploy the BitVM2 Gateway subgraph to the localnet graph-node. +# Idempotent: copies the subgraph sources to $LOCALNET_DATA/subgraph/gateway (the checkout stays +# untouched), regenerates the ABI from the forge build, patches the manifest with the gateway +# address + start block from state/contracts.json, runs npm ci / codegen / build / create / deploy +# inside the pinned node container (host needs only docker), and writes state/subgraph.json. +# +# Re-run this AFTER every contracts-local/deploy-local.sh run (addresses + start block change). +# Overrides (env): SUBGRAPH_NAME, VERSION_LABEL +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" + +SG_SRC=$ROOT/subgraph/gateway +SG=$LOCALNET_DATA/subgraph/gateway +CONTRACTS_JSON=$STATE_DIR/contracts.json +ABI_SRC=$LOCALNET_DATA/forge/out/Gateway.sol/GatewayUpgradeable.json +VERSION_LABEL=${VERSION_LABEL:-v$(date +%Y%m%d-%H%M%S)} +NETWORK=goat-local + +echo "[deploy-local] reading $CONTRACTS_JSON" +GW=$(jq -r '.addresses.gateway_proxy' "$CONTRACTS_JSON") +SB=$(jq -r '.subgraph_start_block' "$CONTRACTS_JSON") +echo "[deploy-local] gateway_proxy=$GW startBlock=$SB name=$SUBGRAPH_NAME" + +[ -f "$ABI_SRC" ] || { echo "[deploy-local] ERROR: $ABI_SRC not found (run contracts-local/deploy-local.sh first)" >&2; exit 1; } + +echo "[deploy-local] staging sources in $SG" +mkdir -p "$SG" +for f in schema.graphql subgraph.yaml networks.json package.json package-lock.json tsconfig.json src abis; do + rm -rf "$SG/$f"; cp -r "$SG_SRC/$f" "$SG/" +done +rm -rf "$SG/build" "$SG/generated" +# node_modules is kept between runs; it is rebuilt when the lock file changed. +if [ -f "$SG/node_modules/.package-lock.json" ] && cmp -s "$SG_SRC/package-lock.json" "$SG/.installed-lock.json"; then + NEED_INSTALL=0 +else + NEED_INSTALL=1 +fi + +echo "[deploy-local] regenerating ABI from the forge build" +python3 -c "import json;json.dump(json.load(open('$ABI_SRC'))['abi'], open('$SG/abis/GatewayUpgradeable.json','w'), indent=2)" + +echo "[deploy-local] patching subgraph.yaml + networks.json" +python3 - "$SG/subgraph.yaml" "$GW" "$SB" "$NETWORK" <<'PY' +import re, sys +path, gw, sb, net = sys.argv[1:5] +s = open(path).read() +s = re.sub(r'network: \S+', f'network: {net}', s, count=1) +s = re.sub(r'address: "0x[0-9a-fA-F]{40}"', f'address: "{gw}"', s, count=1) +s = re.sub(r'startBlock: \d+', f'startBlock: {sb}', s, count=1) +open(path, 'w').write(s) +PY +python3 - "$SG/networks.json" "$GW" "$SB" "$NETWORK" <<'PY' +import json, sys +path, gw, sb, net = sys.argv[1:5] +json.dump({net: {"GatewayUpgradeable": {"address": gw, "startBlock": int(sb)}}}, open(path, 'w'), indent=2) +PY + +echo "[deploy-local] graph-cli in the node container (install=$NEED_INSTALL)" +docker run --rm --network host -u "$LOCALNET_UID:$LOCALNET_GID" \ + -v "$SG:/sg" -w /sg -e HOME=/sg/.home -e npm_config_cache=/sg/.npm \ + -e NEED_INSTALL="$NEED_INSTALL" -e GRAPH_ADMIN="$GRAPH_ADMIN" -e GRAPH_IPFS="$GRAPH_IPFS" \ + -e SUBGRAPH_NAME="$SUBGRAPH_NAME" -e VERSION_LABEL="$VERSION_LABEL" \ + "$NODE_IMAGE" sh -ec ' + if [ "$NEED_INSTALL" = 1 ]; then npm ci --no-audit --no-fund --loglevel=error && cp package-lock.json .installed-lock.json; fi + G=./node_modules/.bin/graph + $G codegen subgraph.yaml + $G build subgraph.yaml + $G create --node "$GRAPH_ADMIN" "$SUBGRAPH_NAME" || true + $G deploy --node "$GRAPH_ADMIN" --ipfs "$GRAPH_IPFS" --version-label "$VERSION_LABEL" "$SUBGRAPH_NAME" subgraph.yaml' + +QUERY_URL="$GRAPH_QUERY_BASE/subgraphs/name/$SUBGRAPH_NAME" +echo "[deploy-local] writing $STATE_DIR/subgraph.json" +python3 - "$STATE_DIR/subgraph.json" "$SUBGRAPH_NAME" "$QUERY_URL" "$GW" "$SB" "$VERSION_LABEL" "$GRAPH_ADMIN" "$GRAPH_IPFS" <<'PY' +import json, sys, datetime +path, name, qurl, gw, sb, ver, admin, ipfs = sys.argv[1:9] +doc = { + "_note": "TEST-ONLY local Gateway subgraph on the localnet graph-node. Generated by localnet/subgraph/deploy-local.sh.", + "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "subgraph_name": name, + "network": "goat-local", + "gateway_address": gw, + "start_block": int(sb), + "version_label": ver, + "graph_admin": admin, + "graph_ipfs": ipfs, + "query_url": qurl, + "node_env": { + "GOAT_GATEWAY_EVENT_THE_GRAPH_URL": qurl, + "GOAT_SWAP_EVENT_THE_GRAPH_URL": qurl + } +} +open(path, 'w').write(json.dumps(doc, indent=2) + "\n") +print(json.dumps(doc, indent=2)) +PY +chmod 600 "$STATE_DIR/subgraph.json" || true +echo "[deploy-local] DONE. Query endpoint: $QUERY_URL" +echo "[deploy-local] health: curl -s -XPOST -d '{\"query\":\"{indexingStatuses{synced health chains{latestBlock{number}}}}\"}' -H 'content-type: application/json' $GRAPH_INDEX/graphql" diff --git a/localnet/subgraph/gateway/abis/GatewayUpgradeable.json b/localnet/subgraph/gateway/abis/GatewayUpgradeable.json new file mode 100644 index 000000000..36e4de5a1 --- /dev/null +++ b/localnet/subgraph/gateway/abis/GatewayUpgradeable.json @@ -0,0 +1,2113 @@ +[ + { + "type": "constructor", + "inputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "answerPeginRequest", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "committeePubkey", + "type": "bytes", + "internalType": "bytes" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "bitcoinSPV", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "address", + "internalType": "contract IBitcoinSPV" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "cancelWithdrawTimelock", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "committeeCancelWithdraw", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "nonce", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "committeeSigs", + "type": "bytes[]", + "internalType": "bytes[]" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "committeeManagement", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "address", + "internalType": "contract ICommitteeManagement" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "finishWithdrawDisproved", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "disproveTxType", + "type": "uint8", + "internalType": "enum IGateway.DisproveTxType" + }, + { + "name": "txnIndex", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "rawChallengeStartTx", + "type": "tuple", + "internalType": "struct BitvmTxParser.BitcoinTx", + "components": [ + { + "name": "version", + "type": "bytes4", + "internalType": "bytes4" + }, + { + "name": "inputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "outputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "locktime", + "type": "bytes4", + "internalType": "bytes4" + } + ] + }, + { + "name": "challengeStartTxProof", + "type": "tuple", + "internalType": "struct MerkleProof.BitcoinTxProof", + "components": [ + { + "name": "rawHeader", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "height", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "proof", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "index", + "type": "uint256", + "internalType": "uint256" + } + ] + }, + { + "name": "rawChallengeFinishTx", + "type": "tuple", + "internalType": "struct BitvmTxParser.BitcoinTx", + "components": [ + { + "name": "version", + "type": "bytes4", + "internalType": "bytes4" + }, + { + "name": "inputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "outputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "locktime", + "type": "bytes4", + "internalType": "bytes4" + } + ] + }, + { + "name": "challengeFinishTxProof", + "type": "tuple", + "internalType": "struct MerkleProof.BitcoinTxProof", + "components": [ + { + "name": "rawHeader", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "height", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "proof", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "index", + "type": "uint256", + "internalType": "uint256" + } + ] + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "finishWithdrawHappyPath", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "rawTake1Tx", + "type": "tuple", + "internalType": "struct BitvmTxParser.BitcoinTx", + "components": [ + { + "name": "version", + "type": "bytes4", + "internalType": "bytes4" + }, + { + "name": "inputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "outputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "locktime", + "type": "bytes4", + "internalType": "bytes4" + } + ] + }, + { + "name": "take1Proof", + "type": "tuple", + "internalType": "struct MerkleProof.BitcoinTxProof", + "components": [ + { + "name": "rawHeader", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "height", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "proof", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "index", + "type": "uint256", + "internalType": "uint256" + } + ] + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "finishWithdrawUnhappyPath", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "rawTake2Tx", + "type": "tuple", + "internalType": "struct BitvmTxParser.BitcoinTx", + "components": [ + { + "name": "version", + "type": "bytes4", + "internalType": "bytes4" + }, + { + "name": "inputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "outputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "locktime", + "type": "bytes4", + "internalType": "bytes4" + } + ] + }, + { + "name": "take2Proof", + "type": "tuple", + "internalType": "struct MerkleProof.BitcoinTxProof", + "components": [ + { + "name": "rawHeader", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "height", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "proof", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "index", + "type": "uint256", + "internalType": "uint256" + } + ] + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "getCancelWithdrawDigestNonced", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "nonce", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "bytes32", + "internalType": "bytes32" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getCommitteeAddresses", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "committeeAddresses", + "type": "address[]", + "internalType": "address[]" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getCommitteePubkeys", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "committeePubkeys", + "type": "bytes[]", + "internalType": "bytes[]" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getCommitteePubkeysUnsafe", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "committeePubkeys", + "type": "bytes[]", + "internalType": "bytes[]" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getGraphData", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "", + "type": "tuple", + "internalType": "struct IGateway.GraphData", + "components": [ + { + "name": "operatorPubkeyPrefix", + "type": "bytes1", + "internalType": "bytes1" + }, + { + "name": "operatorPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "kickoffTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take1Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take2Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "watchtowerChallengeInitTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "proverAssertTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "disproveTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "watchtowerChallengeTimeoutTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "operatorChallengeNackTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "operatorCommitTimeoutTxid", + "type": "bytes32", + "internalType": "bytes32" + } + ] + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getGraphIdsByInstanceId", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "", + "type": "bytes16[]", + "internalType": "bytes16[]" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getPeginData", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "", + "type": "tuple", + "internalType": "struct IGateway.PeginData", + "components": [ + { + "name": "status", + "type": "uint8", + "internalType": "enum IGateway.PeginStatus" + }, + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "depositorAddress", + "type": "address", + "internalType": "address" + }, + { + "name": "peginAmountSats", + "type": "uint64", + "internalType": "uint64" + }, + { + "name": "txnFees", + "type": "uint64[3]", + "internalType": "uint64[3]" + }, + { + "name": "userInputs", + "type": "tuple[]", + "internalType": "struct IGateway.Utxo[]", + "components": [ + { + "name": "txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "vout", + "type": "uint32", + "internalType": "uint32" + }, + { + "name": "amountSats", + "type": "uint64", + "internalType": "uint64" + } + ] + }, + { + "name": "userXonlyPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "userChangeAddress", + "type": "string", + "internalType": "string" + }, + { + "name": "userRefundAddress", + "type": "string", + "internalType": "string" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "createdAt", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "committeeAddresses", + "type": "address[]", + "internalType": "address[]" + }, + { + "name": "committeePubkeys", + "type": "bytes[]", + "internalType": "bytes[]" + } + ] + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getPostGraphDigest", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "graphData", + "type": "tuple", + "internalType": "struct IGateway.GraphData", + "components": [ + { + "name": "operatorPubkeyPrefix", + "type": "bytes1", + "internalType": "bytes1" + }, + { + "name": "operatorPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "kickoffTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take1Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take2Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "watchtowerChallengeInitTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "proverAssertTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "disproveTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "watchtowerChallengeTimeoutTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "operatorChallengeNackTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "operatorCommitTimeoutTxid", + "type": "bytes32", + "internalType": "bytes32" + } + ] + } + ], + "outputs": [ + { + "name": "", + "type": "bytes32", + "internalType": "bytes32" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getPostPeginDigest", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + } + ], + "outputs": [ + { + "name": "", + "type": "bytes32", + "internalType": "bytes32" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getUnlockStakeDigestNonced", + "inputs": [ + { + "name": "operator", + "type": "address", + "internalType": "address" + }, + { + "name": "amount", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "nonce", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "bytes32", + "internalType": "bytes32" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "graphDataMap", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "operatorPubkeyPrefix", + "type": "bytes1", + "internalType": "bytes1" + }, + { + "name": "operatorPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "kickoffTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take1Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take2Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "watchtowerChallengeInitTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "proverAssertTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "operatorCommitTimeoutTxid", + "type": "bytes32", + "internalType": "bytes32" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "initWithdraw", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "initialize", + "inputs": [ + { + "name": "_pegBTC", + "type": "address", + "internalType": "contract IPegBTC" + }, + { + "name": "_bitcoinSPV", + "type": "address", + "internalType": "contract IBitcoinSPV" + }, + { + "name": "_committeeManagement", + "type": "address", + "internalType": "contract ICommitteeManagement" + }, + { + "name": "_stakeManagement", + "type": "address", + "internalType": "contract IStakeManagement" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "instanceIdToGraphIds", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "graphIds", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "instanceIds", + "inputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minChallengeAmountSats", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minChallengerReward", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minDisproverReward", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minOperatorRewardSats", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minPeginFeeSats", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minSlashAmount", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "minStakeAmount", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "operatorRewardRate", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "pegBTC", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "address", + "internalType": "contract IPegBTC" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "peginDataMap", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "status", + "type": "uint8", + "internalType": "enum IGateway.PeginStatus" + }, + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "depositorAddress", + "type": "address", + "internalType": "address" + }, + { + "name": "peginAmountSats", + "type": "uint64", + "internalType": "uint64" + }, + { + "name": "userXonlyPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "userChangeAddress", + "type": "string", + "internalType": "string" + }, + { + "name": "userRefundAddress", + "type": "string", + "internalType": "string" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "createdAt", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "peginFeeRate", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "postGraphData", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "graphData", + "type": "tuple", + "internalType": "struct IGateway.GraphData", + "components": [ + { + "name": "operatorPubkeyPrefix", + "type": "bytes1", + "internalType": "bytes1" + }, + { + "name": "operatorPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "kickoffTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take1Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "take2Txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "watchtowerChallengeInitTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "proverAssertTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "disproveTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "watchtowerChallengeTimeoutTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "operatorChallengeNackTxids", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "operatorCommitTimeoutTxid", + "type": "bytes32", + "internalType": "bytes32" + } + ] + }, + { + "name": "committeeSigs", + "type": "bytes[]", + "internalType": "bytes[]" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "postPeginData", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "rawPeginTx", + "type": "tuple", + "internalType": "struct BitvmTxParser.BitcoinTx", + "components": [ + { + "name": "version", + "type": "bytes4", + "internalType": "bytes4" + }, + { + "name": "inputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "outputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "locktime", + "type": "bytes4", + "internalType": "bytes4" + } + ] + }, + { + "name": "peginProof", + "type": "tuple", + "internalType": "struct MerkleProof.BitcoinTxProof", + "components": [ + { + "name": "rawHeader", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "height", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "proof", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "index", + "type": "uint256", + "internalType": "uint256" + } + ] + }, + { + "name": "committeeSigs", + "type": "bytes[]", + "internalType": "bytes[]" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "postPeginRequest", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "peginAmountSats", + "type": "uint64", + "internalType": "uint64" + }, + { + "name": "txnFees", + "type": "uint64[3]", + "internalType": "uint64[3]" + }, + { + "name": "receiverAddress", + "type": "address", + "internalType": "address" + }, + { + "name": "userInputs", + "type": "tuple[]", + "internalType": "struct IGateway.Utxo[]", + "components": [ + { + "name": "txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "vout", + "type": "uint32", + "internalType": "uint32" + }, + { + "name": "amountSats", + "type": "uint64", + "internalType": "uint64" + } + ] + }, + { + "name": "userXonlyPubkey", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "userChangeAddress", + "type": "string", + "internalType": "string" + }, + { + "name": "userRefundAddress", + "type": "string", + "internalType": "string" + } + ], + "outputs": [], + "stateMutability": "payable" + }, + { + "type": "function", + "name": "proceedWithdraw", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "rawKickoffTx", + "type": "tuple", + "internalType": "struct BitvmTxParser.BitcoinTx", + "components": [ + { + "name": "version", + "type": "bytes4", + "internalType": "bytes4" + }, + { + "name": "inputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "outputVector", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "locktime", + "type": "bytes4", + "internalType": "bytes4" + } + ] + }, + { + "name": "kickoffProof", + "type": "tuple", + "internalType": "struct MerkleProof.BitcoinTxProof", + "components": [ + { + "name": "rawHeader", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "height", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "proof", + "type": "bytes32[]", + "internalType": "bytes32[]" + }, + { + "name": "index", + "type": "uint256", + "internalType": "uint256" + } + ] + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "responseWindowBlocks", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "stakeManagement", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "address", + "internalType": "contract IStakeManagement" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "unlockOperatorStake", + "inputs": [ + { + "name": "operator", + "type": "address", + "internalType": "address" + }, + { + "name": "amount", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "nonce", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "committeeSigs", + "type": "bytes[]", + "internalType": "bytes[]" + } + ], + "outputs": [], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "verifyCommitteeSignatures", + "inputs": [ + { + "name": "msgHash", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "signatures", + "type": "bytes[]", + "internalType": "bytes[]" + }, + { + "name": "members", + "type": "address[]", + "internalType": "address[]" + } + ], + "outputs": [ + { + "name": "", + "type": "bool", + "internalType": "bool" + } + ], + "stateMutability": "pure" + }, + { + "type": "function", + "name": "withdrawDataMap", + "inputs": [ + { + "name": "graphId", + "type": "bytes16", + "internalType": "bytes16" + } + ], + "outputs": [ + { + "name": "status", + "type": "uint8", + "internalType": "enum IGateway.WithdrawStatus" + }, + { + "name": "peginTxid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "operatorAddress", + "type": "address", + "internalType": "address" + }, + { + "name": "instanceId", + "type": "bytes16", + "internalType": "bytes16" + }, + { + "name": "lockAmount", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "btcBlockHeightAtWithdraw", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "event", + "name": "BridgeIn", + "inputs": [ + { + "name": "depositorAddress", + "type": "address", + "indexed": true, + "internalType": "address" + }, + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "peginAmountSats", + "type": "uint64", + "indexed": true, + "internalType": "uint64" + }, + { + "name": "feeAmountSats", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "BridgeInRequest", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "depositorAddress", + "type": "address", + "indexed": true, + "internalType": "address" + }, + { + "name": "peginAmountSats", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + }, + { + "name": "txnFees", + "type": "uint64[3]", + "indexed": false, + "internalType": "uint64[3]" + }, + { + "name": "userInputs", + "type": "tuple[]", + "indexed": false, + "internalType": "struct IGateway.Utxo[]", + "components": [ + { + "name": "txid", + "type": "bytes32", + "internalType": "bytes32" + }, + { + "name": "vout", + "type": "uint32", + "internalType": "uint32" + }, + { + "name": "amountSats", + "type": "uint64", + "internalType": "uint64" + } + ] + }, + { + "name": "userXonlyPubkey", + "type": "bytes32", + "indexed": false, + "internalType": "bytes32" + }, + { + "name": "userChangeAddress", + "type": "string", + "indexed": false, + "internalType": "string" + }, + { + "name": "userRefundAddress", + "type": "string", + "indexed": false, + "internalType": "string" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "CancelWithdraw", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "triggerAddress", + "type": "address", + "indexed": true, + "internalType": "address" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "CommitteeResponse", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "committeeAddress", + "type": "address", + "indexed": true, + "internalType": "address" + }, + { + "name": "committeePubkey", + "type": "bytes", + "indexed": false, + "internalType": "bytes" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "InitWithdraw", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "operatorAddress", + "type": "address", + "indexed": true, + "internalType": "address" + }, + { + "name": "withdrawAmountSats", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "Initialized", + "inputs": [ + { + "name": "version", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "PostGraphData", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "ProceedWithdraw", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "kickoffTxid", + "type": "bytes32", + "indexed": false, + "internalType": "bytes32" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "WithdrawDisproved", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "disproveTxType", + "type": "uint8", + "indexed": false, + "internalType": "enum IGateway.DisproveTxType" + }, + { + "name": "txnIndex", + "type": "uint256", + "indexed": false, + "internalType": "uint256" + }, + { + "name": "challengeStartTxid", + "type": "bytes32", + "indexed": false, + "internalType": "bytes32" + }, + { + "name": "challengeFinishTxid", + "type": "bytes32", + "indexed": false, + "internalType": "bytes32" + }, + { + "name": "challengerAddress", + "type": "address", + "indexed": false, + "internalType": "address" + }, + { + "name": "disproverAddress", + "type": "address", + "indexed": false, + "internalType": "address" + }, + { + "name": "challengerRewardAmount", + "type": "uint256", + "indexed": false, + "internalType": "uint256" + }, + { + "name": "disproverRewardAmount", + "type": "uint256", + "indexed": false, + "internalType": "uint256" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "WithdrawHappyPath", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "take1Txid", + "type": "bytes32", + "indexed": false, + "internalType": "bytes32" + }, + { + "name": "operatorAddress", + "type": "address", + "indexed": true, + "internalType": "address" + }, + { + "name": "rewardAmountSats", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "WithdrawUnhappyPath", + "inputs": [ + { + "name": "instanceId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "graphId", + "type": "bytes16", + "indexed": true, + "internalType": "bytes16" + }, + { + "name": "take2Txid", + "type": "bytes32", + "indexed": false, + "internalType": "bytes32" + }, + { + "name": "operatorAddress", + "type": "address", + "indexed": true, + "internalType": "address" + }, + { + "name": "rewardAmountSats", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + } + ], + "anonymous": false + }, + { + "type": "error", + "name": "AlreadyDisproved", + "inputs": [] + }, + { + "type": "error", + "name": "DisproveInvalidHeader", + "inputs": [] + }, + { + "type": "error", + "name": "ECDSAInvalidSignature", + "inputs": [] + }, + { + "type": "error", + "name": "ECDSAInvalidSignatureLength", + "inputs": [ + { + "name": "length", + "type": "uint256", + "internalType": "uint256" + } + ] + }, + { + "type": "error", + "name": "ECDSAInvalidSignatureS", + "inputs": [ + { + "name": "s", + "type": "bytes32", + "internalType": "bytes32" + } + ] + }, + { + "type": "error", + "name": "ERC20TransferFailed", + "inputs": [] + }, + { + "type": "error", + "name": "FeeTooHigh", + "inputs": [] + }, + { + "type": "error", + "name": "GraphAlreadyPosted", + "inputs": [] + }, + { + "type": "error", + "name": "GraphPeginTxidMismatch", + "inputs": [] + }, + { + "type": "error", + "name": "IndexOutOfRange", + "inputs": [] + }, + { + "type": "error", + "name": "InstanceMismatch", + "inputs": [] + }, + { + "type": "error", + "name": "InstanceUsed", + "inputs": [] + }, + { + "type": "error", + "name": "InvalidHeader", + "inputs": [] + }, + { + "type": "error", + "name": "InvalidInitialization", + "inputs": [] + }, + { + "type": "error", + "name": "InvalidPubkeyLen", + "inputs": [] + }, + { + "type": "error", + "name": "InvalidPubkeyParity", + "inputs": [] + }, + { + "type": "error", + "name": "InvalidSignatures", + "inputs": [] + }, + { + "type": "error", + "name": "KickoffHeightLow", + "inputs": [] + }, + { + "type": "error", + "name": "MerkleVerifyFail", + "inputs": [] + }, + { + "type": "error", + "name": "NotCommittee", + "inputs": [] + }, + { + "type": "error", + "name": "NotEnoughCommittee", + "inputs": [] + }, + { + "type": "error", + "name": "NotInitializing", + "inputs": [] + }, + { + "type": "error", + "name": "NotOperator", + "inputs": [] + }, + { + "type": "error", + "name": "NotPending", + "inputs": [] + }, + { + "type": "error", + "name": "NotWithdrawable", + "inputs": [] + }, + { + "type": "error", + "name": "OperatorNotRegistered", + "inputs": [] + }, + { + "type": "error", + "name": "PeginAmountMismatch", + "inputs": [] + }, + { + "type": "error", + "name": "StakeInsufficient", + "inputs": [] + }, + { + "type": "error", + "name": "TimelockNotExpired", + "inputs": [] + }, + { + "type": "error", + "name": "TxidMismatch", + "inputs": [] + }, + { + "type": "error", + "name": "UnknownDisproveType", + "inputs": [] + }, + { + "type": "error", + "name": "UnsupportedChainId", + "inputs": [ + { + "name": "chainId", + "type": "uint256", + "internalType": "uint256" + } + ] + }, + { + "type": "error", + "name": "WindowExpired", + "inputs": [] + }, + { + "type": "error", + "name": "WindowNotExpired", + "inputs": [] + }, + { + "type": "error", + "name": "WithdrawStatusInvalid", + "inputs": [] + } +] diff --git a/localnet/subgraph/gateway/networks.json b/localnet/subgraph/gateway/networks.json new file mode 100644 index 000000000..1a13e9ee8 --- /dev/null +++ b/localnet/subgraph/gateway/networks.json @@ -0,0 +1,8 @@ +{ + "goat-local": { + "GatewayUpgradeable": { + "address": "0x0000000000000000000000000000000000000000", + "startBlock": 0 + } + } +} diff --git a/localnet/subgraph/gateway/package-lock.json b/localnet/subgraph/gateway/package-lock.json new file mode 100644 index 000000000..e43fbc276 --- /dev/null +++ b/localnet/subgraph/gateway/package-lock.json @@ -0,0 +1,5580 @@ +{ + "name": "gateway_graph", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "gateway_graph", + "license": "UNLICENSED", + "dependencies": { + "@graphprotocol/graph-cli": "0.98.1", + "@graphprotocol/graph-ts": "0.37.0" + }, + "devDependencies": { + "matchstick-as": "0.6.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.28.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.28.6.tgz", + "integrity": "sha512-JYgintcMjRiCvS8mMECzaEn+m3PfoQiyqukOMCCVQtoJGYJw8j/8LBJEiqkHLkfwCcs74E3pbAUFNg7d9VNJ+Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@chainsafe/is-ip": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@chainsafe/is-ip/-/is-ip-2.1.0.tgz", + "integrity": "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w==", + "license": "MIT" + }, + "node_modules/@chainsafe/netmask": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@chainsafe/netmask/-/netmask-2.0.0.tgz", + "integrity": "sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==", + "license": "MIT", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1" + } + }, + "node_modules/@dnsquery/dns-packet": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@dnsquery/dns-packet/-/dns-packet-6.1.1.tgz", + "integrity": "sha512-WXTuFvL3G+74SchFAtz3FgIYVOe196ycvGsMgvSH/8Goptb1qpIQtIuM4SOK9G9lhMWYpHxnXyy544ZhluFOew==", + "license": "MIT", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.4", + "utf8-codec": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@fastify/busboy": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.2.0.tgz", + "integrity": "sha512-m9FVDXU3GT2ITSe0UaMA5rU3QkfC/UXtCU8y0gSN/GugTqtVldOBWIB5V6V3sbmenVZUIpU6f+mPEO2+m5iTaA==", + "license": "MIT" + }, + "node_modules/@float-capital/float-subgraph-uncrashable": { + "version": "0.0.0-internal-testing.5", + "resolved": "https://registry.npmjs.org/@float-capital/float-subgraph-uncrashable/-/float-subgraph-uncrashable-0.0.0-internal-testing.5.tgz", + "integrity": "sha512-yZ0H5e3EpAYKokX/AbtplzlvSxEJY7ZfpvQyDzyODkks0hakAAlDG6fQu1SlDJMWorY7bbq1j7fCiFeTWci6TA==", + "license": "MIT", + "dependencies": { + "@rescript/std": "9.0.0", + "graphql": "^16.6.0", + "graphql-import-node": "^0.0.5", + "js-yaml": "^4.1.0" + }, + "bin": { + "uncrashable": "bin/uncrashable" + } + }, + "node_modules/@graphprotocol/graph-cli": { + "version": "0.98.1", + "resolved": "https://registry.npmjs.org/@graphprotocol/graph-cli/-/graph-cli-0.98.1.tgz", + "integrity": "sha512-GrWFcRCBlLcRT+gIGundQl7yyrX3YWUPj66bxThKf5CJvvWXdZoNxrj27dMMqulsSwYmpCkb3YmpCiVJFGdpHw==", + "license": "(Apache-2.0 OR MIT)", + "dependencies": { + "@float-capital/float-subgraph-uncrashable": "0.0.0-internal-testing.5", + "@oclif/core": "4.5.5", + "@oclif/plugin-autocomplete": "^3.2.11", + "@oclif/plugin-not-found": "^3.2.29", + "@oclif/plugin-warn-if-update-available": "^3.1.24", + "@pinax/graph-networks-registry": "^0.7.0", + "@whatwg-node/fetch": "^0.10.1", + "assemblyscript": "0.19.23", + "chokidar": "4.0.3", + "debug": "4.4.3", + "decompress": "^4.2.1", + "docker-compose": "1.3.0", + "fs-extra": "11.3.2", + "glob": "11.0.3", + "gluegun": "5.2.0", + "graphql": "16.11.0", + "immutable": "5.1.4", + "jayson": "4.2.0", + "js-yaml": "4.1.0", + "kubo-rpc-client": "^5.0.2", + "open": "10.2.0", + "prettier": "3.6.2", + "progress": "^2.0.3", + "semver": "7.7.3", + "tmp-promise": "3.0.3", + "undici": "7.16.0", + "web3-eth-abi": "4.4.1", + "yaml": "2.8.1" + }, + "bin": { + "graph": "bin/run.js" + }, + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/@graphprotocol/graph-ts": { + "version": "0.37.0", + "resolved": "https://registry.npmjs.org/@graphprotocol/graph-ts/-/graph-ts-0.37.0.tgz", + "integrity": "sha512-3xp/sO8zFDBkX44ydGB87ow5Cyrfr/SAm/cWzIRzUVL7ROw0KUyFBG1xj4KKlMnAod7/RL99zChYquC15H4Oqg==", + "dependencies": { + "assemblyscript": "0.27.31" + } + }, + "node_modules/@graphprotocol/graph-ts/node_modules/assemblyscript": { + "version": "0.27.31", + "resolved": "https://registry.npmjs.org/assemblyscript/-/assemblyscript-0.27.31.tgz", + "integrity": "sha512-Ra8kiGhgJQGZcBxjtMcyVRxOEJZX64kd+XGpjWzjcjgxWJVv+CAQO0aDBk4GQVhjYbOkATarC83mHjAVGtwPBQ==", + "license": "Apache-2.0", + "dependencies": { + "binaryen": "116.0.0-nightly.20240114", + "long": "^5.2.1" + }, + "bin": { + "asc": "bin/asc.js", + "asinit": "bin/asinit.js" + }, + "engines": { + "node": ">=16", + "npm": ">=7" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/assemblyscript" + } + }, + "node_modules/@graphprotocol/graph-ts/node_modules/binaryen": { + "version": "116.0.0-nightly.20240114", + "resolved": "https://registry.npmjs.org/binaryen/-/binaryen-116.0.0-nightly.20240114.tgz", + "integrity": "sha512-0GZrojJnuhoe+hiwji7QFaL3tBlJoA+KFUN7ouYSDGZLSo9CKM8swQX8n/UcbR0d1VuZKU+nhogNzv423JEu5A==", + "license": "Apache-2.0", + "bin": { + "wasm-opt": "bin/wasm-opt", + "wasm2js": "bin/wasm2js" + } + }, + "node_modules/@inquirer/ansi": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", + "integrity": "sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.3.2.tgz", + "integrity": "sha512-VXukHf0RR1doGe6Sm4F0Em7SWYLTHSsbGfJdS9Ja2bX5/D5uwVOEjr07cncLROdBvmnvCATYEWlHqYmXv2IlQA==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "5.1.21", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.21.tgz", + "integrity": "sha512-KR8edRkIsUayMXV+o3Gv+q4jlhENF9nMYUZs9PA2HzrXeHI8M5uDag70U7RJn9yyiMZSbtF5/UexBtAVtZGSbQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "10.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.3.2.tgz", + "integrity": "sha512-43RTuEbfP8MbKzedNqBrlhhNKVwoK//vUFNW3Q3vZ88BLcrs4kYpGg+B2mm5p2K/HfygoCxuKwJJiv8PbGmE0A==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "cli-width": "^4.1.0", + "mute-stream": "^2.0.0", + "signal-exit": "^4.1.0", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@inquirer/core/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/@inquirer/core/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/@inquirer/core/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/editor": { + "version": "4.2.23", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.23.tgz", + "integrity": "sha512-aLSROkEwirotxZ1pBaP8tugXRFCxW94gwrQLxXfrZsKkfjOYC1aRvAZuhpJOb5cu4IBTJdsCigUlf2iCOu4ZDQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/external-editor": "^1.0.3", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.23.tgz", + "integrity": "sha512-nRzdOyFYnpeYTTR2qFwEVmIWypzdAx/sIkCMeTNTcflFOovfqUk+HcFhQQVBftAh9gmGrpFj6QcGEqrDMDOiew==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", + "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/input": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.3.1.tgz", + "integrity": "sha512-kN0pAM4yPrLjJ1XJBjDxyfDduXOuQHrBB8aLDMueuwUGn+vNpF7Gq7TvyVxx8u4SHlFFj4trmj+a2cbpG4Jn1g==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "3.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.23.tgz", + "integrity": "sha512-5Smv0OK7K0KUzUfYUXDXQc9jrf8OHo4ktlEayFlelCjwMXz0299Y8OrI+lj7i4gCBY15UObk76q0QtxjzFcFcg==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.23.tgz", + "integrity": "sha512-zREJHjhT5vJBMZX/IUbyI9zVtVfOLiTO66MrF/3GFZYZ7T4YILW5MSkEYHceSii/KtRk+4i3RE7E1CUXA2jHcA==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "7.10.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.10.1.tgz", + "integrity": "sha512-Dx/y9bCQcXLI5ooQ5KyvA4FTgeo2jYj/7plWfV5Ak5wDPKQZgudKez2ixyfz7tKXzcJciTxqLeK7R9HItwiByg==", + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^4.3.2", + "@inquirer/confirm": "^5.1.21", + "@inquirer/editor": "^4.2.23", + "@inquirer/expand": "^4.0.23", + "@inquirer/input": "^4.3.1", + "@inquirer/number": "^3.0.23", + "@inquirer/password": "^4.0.23", + "@inquirer/rawlist": "^4.1.11", + "@inquirer/search": "^3.2.2", + "@inquirer/select": "^4.4.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.1.11.tgz", + "integrity": "sha512-+LLQB8XGr3I5LZN/GuAHo+GpDJegQwuPARLChlMICNdwW7OwV2izlCSCxN6cqpL0sMXmbKbFcItJgdQq5EBXTw==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.2.2.tgz", + "integrity": "sha512-p2bvRfENXCZdWF/U2BXvnSI9h+tuA8iNqtUKb9UWbmLYCRQxd8WkvwWvYn+3NgYaNwdUkHytJMGG4MMLucI1kA==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.4.2.tgz", + "integrity": "sha512-l4xMuJo55MAe+N7Qr4rX90vypFwCajSakx59qe/tMaC1aEHWLyw68wF4o0A4SLAY4E0nd+Vt+EyskeDIqu1M6w==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.10.tgz", + "integrity": "sha512-BvziSRxfz5Ov8ch0z/n3oijRSEcEsHnhggm4xFZe93DHcUCTlutlq9Ox4SVENAfcRD22UQq7T/atg9Wr3k09eA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@ipld/dag-cbor": { + "version": "9.2.5", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.2.5.tgz", + "integrity": "sha512-84wSr4jv30biui7endhobYhXBQzQE4c/wdoWlFrKcfiwH+ofaPg8fwsM8okX9cOzkkrsAsNdDyH3ou+kiLquwQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "cborg": "^4.0.0", + "multiformats": "^13.1.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ipld/dag-json": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-10.2.6.tgz", + "integrity": "sha512-51yc5azhmkvc9mp2HV/vtJ8SlgFXADp55wAPuuAjQZ+yPurAYuTVddS3ke5vT4sjcd4DbE+DWjsMZGXjFB2cuA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "cborg": "^4.4.0", + "multiformats": "^13.1.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@ipld/dag-pb": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@ipld/dag-pb/-/dag-pb-4.1.5.tgz", + "integrity": "sha512-w4PZ2yPqvNmlAir7/2hsCRMqny1EY5jj26iZcSgxREJexmbAc2FI21jp26MqiNdfgAxvkCnf2N/TJI18GaDNwA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "multiformats": "^13.1.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", + "license": "MIT", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "license": "MIT", + "dependencies": { + "@isaacs/balanced-match": "^4.0.1" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", + "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", + "license": "MIT" + }, + "node_modules/@libp2p/crypto": { + "version": "5.1.13", + "resolved": "https://registry.npmjs.org/@libp2p/crypto/-/crypto-5.1.13.tgz", + "integrity": "sha512-8NN9cQP3jDn+p9+QE9ByiEoZ2lemDFf/unTgiKmS3JF93ph240EUVdbCyyEgOMfykzb0okTM4gzvwfx9osJebQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@libp2p/interface": "^3.1.0", + "@noble/curves": "^2.0.1", + "@noble/hashes": "^2.0.1", + "multiformats": "^13.4.0", + "protons-runtime": "^5.6.0", + "uint8arraylist": "^2.4.8", + "uint8arrays": "^5.1.0" + } + }, + "node_modules/@libp2p/crypto/node_modules/@libp2p/interface": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-3.1.0.tgz", + "integrity": "sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@multiformats/dns": "^1.0.6", + "@multiformats/multiaddr": "^13.0.1", + "main-event": "^1.0.1", + "multiformats": "^13.4.0", + "progress-events": "^1.0.1", + "uint8arraylist": "^2.4.8" + } + }, + "node_modules/@libp2p/crypto/node_modules/@multiformats/multiaddr": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-13.0.1.tgz", + "integrity": "sha512-XToN915cnfr6Lr9EdGWakGJbPT0ghpg/850HvdC+zFX8XvpLZElwa8synCiwa8TuvKNnny6m8j8NVBNCxhIO3g==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1", + "multiformats": "^13.0.0", + "uint8-varint": "^2.0.1", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/@libp2p/interface": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-2.11.0.tgz", + "integrity": "sha512-0MUFKoXWHTQW3oWIgSHApmYMUKWO/Y02+7Hpyp+n3z+geD4Xo2Rku2gYWmxcq+Pyjkz6Q9YjDWz3Yb2SoV2E8Q==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@multiformats/dns": "^1.0.6", + "@multiformats/multiaddr": "^12.4.4", + "it-pushable": "^3.2.3", + "it-stream-types": "^2.0.2", + "main-event": "^1.0.1", + "multiformats": "^13.3.6", + "progress-events": "^1.0.1", + "uint8arraylist": "^2.4.8" + } + }, + "node_modules/@libp2p/logger": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@libp2p/logger/-/logger-5.2.0.tgz", + "integrity": "sha512-OEFS529CnIKfbWEHmuCNESw9q0D0hL8cQ8klQfjIVPur15RcgAEgc1buQ7Y6l0B6tCYg120bp55+e9tGvn8c0g==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@libp2p/interface": "^2.11.0", + "@multiformats/multiaddr": "^12.4.4", + "interface-datastore": "^8.3.1", + "multiformats": "^13.3.6", + "weald": "^1.0.4" + } + }, + "node_modules/@libp2p/peer-id": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/@libp2p/peer-id/-/peer-id-5.1.9.tgz", + "integrity": "sha512-cVDp7lX187Epmi/zr0Qq2RsEMmueswP9eIxYSFoMcHL/qcvRFhsxOfUGB8361E26s2WJvC9sXZ0oJS9XVueJhQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@libp2p/crypto": "^5.1.8", + "@libp2p/interface": "^2.11.0", + "multiformats": "^13.3.6", + "uint8arrays": "^5.1.0" + } + }, + "node_modules/@multiformats/dns": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@multiformats/dns/-/dns-1.0.13.tgz", + "integrity": "sha512-yr4bxtA3MbvJ+2461kYIYMsiiZj/FIqKI64hE4SdvWJUdWF9EtZLar38juf20Sf5tguXKFUruluswAO6JsjS2w==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@dnsquery/dns-packet": "^6.1.1", + "@libp2p/interface": "^3.1.0", + "hashlru": "^2.3.0", + "p-queue": "^9.0.0", + "progress-events": "^1.0.0", + "uint8arrays": "^5.0.2" + } + }, + "node_modules/@multiformats/dns/node_modules/@libp2p/interface": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-3.1.0.tgz", + "integrity": "sha512-RE7/XyvC47fQBe1cHxhMvepYKa5bFCUyFrrpj8PuM0E7JtzxU7F+Du5j4VXbg2yLDcToe0+j8mB7jvwE2AThYw==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@multiformats/dns": "^1.0.6", + "@multiformats/multiaddr": "^13.0.1", + "main-event": "^1.0.1", + "multiformats": "^13.4.0", + "progress-events": "^1.0.1", + "uint8arraylist": "^2.4.8" + } + }, + "node_modules/@multiformats/dns/node_modules/@multiformats/multiaddr": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-13.0.1.tgz", + "integrity": "sha512-XToN915cnfr6Lr9EdGWakGJbPT0ghpg/850HvdC+zFX8XvpLZElwa8synCiwa8TuvKNnny6m8j8NVBNCxhIO3g==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1", + "multiformats": "^13.0.0", + "uint8-varint": "^2.0.1", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/@multiformats/multiaddr": { + "version": "12.5.1", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-12.5.1.tgz", + "integrity": "sha512-+DDlr9LIRUS8KncI1TX/FfUn8F2dl6BIxJgshS/yFQCNB5IAF0OGzcwB39g5NLE22s4qqDePv0Qof6HdpJ/4aQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1", + "@chainsafe/netmask": "^2.0.0", + "@multiformats/dns": "^1.0.3", + "abort-error": "^1.0.1", + "multiformats": "^13.0.0", + "uint8-varint": "^2.0.1", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/@multiformats/multiaddr-to-uri": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr-to-uri/-/multiaddr-to-uri-11.0.2.tgz", + "integrity": "sha512-SiLFD54zeOJ0qMgo9xv1Tl9O5YktDKAVDP4q4hL16mSq4O4sfFNagNADz8eAofxd6TfQUzGQ3TkRRG9IY2uHRg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@multiformats/multiaddr": "^12.3.0" + } + }, + "node_modules/@noble/curves": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.0.1.tgz", + "integrity": "sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "2.0.1" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.0.1.tgz", + "integrity": "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@oclif/core": { + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.5.5.tgz", + "integrity": "sha512-iQzlaJQgPeUXrtrX71OzDwxPikQ7c2FhNd8U8rBB7BCtj2XYfmzBT/Hmbc+g9OKDIG/JkbJT0fXaWMMBrhi+1A==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^4.3.2", + "ansis": "^3.17.0", + "clean-stack": "^3.0.1", + "cli-spinners": "^2.9.2", + "debug": "^4.4.3", + "ejs": "^3.1.10", + "get-package-type": "^0.1.0", + "indent-string": "^4.0.0", + "is-wsl": "^2.2.0", + "lilconfig": "^3.1.3", + "minimatch": "^9.0.5", + "semver": "^7.7.3", + "string-width": "^4.2.3", + "supports-color": "^8", + "tinyglobby": "^0.2.14", + "widest-line": "^3.1.0", + "wordwrap": "^1.0.0", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@oclif/plugin-autocomplete": { + "version": "3.2.40", + "resolved": "https://registry.npmjs.org/@oclif/plugin-autocomplete/-/plugin-autocomplete-3.2.40.tgz", + "integrity": "sha512-HCfDuUV3l5F5Wz7SKkaoFb+OMQ5vKul8zvsPNgI0QbZcQuGHmn3svk+392wSfXboyA1gq8kzEmKPAoQK6r6UNw==", + "license": "MIT", + "dependencies": { + "@oclif/core": "^4", + "ansis": "^3.16.0", + "debug": "^4.4.1", + "ejs": "^3.1.10" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@oclif/plugin-not-found": { + "version": "3.2.74", + "resolved": "https://registry.npmjs.org/@oclif/plugin-not-found/-/plugin-not-found-3.2.74.tgz", + "integrity": "sha512-6RD/EuIUGxAYR45nMQg+nw+PqwCXUxkR6Eyn+1fvbVjtb9d+60OPwB77LCRUI4zKNI+n0LOFaMniEdSpb+A7kQ==", + "license": "MIT", + "dependencies": { + "@inquirer/prompts": "^7.10.1", + "@oclif/core": "^4.8.0", + "ansis": "^3.17.0", + "fast-levenshtein": "^3.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@oclif/plugin-not-found/node_modules/@oclif/core": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/@oclif/core/-/core-4.8.0.tgz", + "integrity": "sha512-jteNUQKgJHLHFbbz806aGZqf+RJJ7t4gwF4MYa8fCwCxQ8/klJNWc0MvaJiBebk7Mc+J39mdlsB4XraaCKznFw==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^4.3.2", + "ansis": "^3.17.0", + "clean-stack": "^3.0.1", + "cli-spinners": "^2.9.2", + "debug": "^4.4.3", + "ejs": "^3.1.10", + "get-package-type": "^0.1.0", + "indent-string": "^4.0.0", + "is-wsl": "^2.2.0", + "lilconfig": "^3.1.3", + "minimatch": "^9.0.5", + "semver": "^7.7.3", + "string-width": "^4.2.3", + "supports-color": "^8", + "tinyglobby": "^0.2.14", + "widest-line": "^3.1.0", + "wordwrap": "^1.0.0", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@oclif/plugin-warn-if-update-available": { + "version": "3.1.55", + "resolved": "https://registry.npmjs.org/@oclif/plugin-warn-if-update-available/-/plugin-warn-if-update-available-3.1.55.tgz", + "integrity": "sha512-VIEBoaoMOCjl3y+w/kdfZMODi0mVMnDuM0vkBf3nqeidhRXVXq87hBqYDdRwN1XoD+eDfE8tBbOP7qtSOONztQ==", + "license": "MIT", + "dependencies": { + "@oclif/core": "^4", + "ansis": "^3.17.0", + "debug": "^4.4.3", + "http-call": "^5.2.2", + "lodash": "^4.17.23", + "registry-auth-token": "^5.1.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@pinax/graph-networks-registry": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@pinax/graph-networks-registry/-/graph-networks-registry-0.7.1.tgz", + "integrity": "sha512-Gn2kXRiEd5COAaMY/aDCRO0V+zfb1uQKCu5HFPoWka+EsZW27AlTINA7JctYYYEMuCbjMia5FBOzskjgEvj6LA==", + "license": "MIT" + }, + "node_modules/@pnpm/config.env-replace": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", + "integrity": "sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==", + "license": "MIT", + "engines": { + "node": ">=12.22.0" + } + }, + "node_modules/@pnpm/network.ca-file": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@pnpm/network.ca-file/-/network.ca-file-1.0.2.tgz", + "integrity": "sha512-YcPQ8a0jwYU9bTdJDpXjMi7Brhkr1mXsXrUJvjqM2mQDgkRiz8jFaQGOdaLxgjtUfQgZhKy/O3cG/YwmgKaxLA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "4.2.10" + }, + "engines": { + "node": ">=12.22.0" + } + }, + "node_modules/@pnpm/network.ca-file/node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "license": "ISC" + }, + "node_modules/@pnpm/npm-conf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@pnpm/npm-conf/-/npm-conf-3.0.2.tgz", + "integrity": "sha512-h104Kh26rR8tm+a3Qkc5S4VLYint3FE48as7+/5oCEcKR2idC/pF1G6AhIXKI+eHPJa/3J9i5z0Al47IeGHPkA==", + "license": "MIT", + "dependencies": { + "@pnpm/config.env-replace": "^1.1.0", + "@pnpm/network.ca-file": "^1.0.1", + "config-chain": "^1.1.11" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@rescript/std": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@rescript/std/-/std-9.0.0.tgz", + "integrity": "sha512-zGzFsgtZ44mgL4Xef2gOy1hrRVdrs9mcxCOOKZrIPsmbZW14yTkaF591GXxpQvjXiHtgZ/iA9qLyWH6oSReIxQ==", + "license": "SEE LICENSE IN LICENSE" + }, + "node_modules/@scure/base": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.9.tgz", + "integrity": "sha512-8YKhl8GHiNI/pU2VMaofa2Tor7PJRAjwQLBBuilkJ9L5+13yVbC7JO/wS7piioAvPSwR3JKM1IJ/u4xQzbcXKg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.4.0.tgz", + "integrity": "sha512-sVUpc0Vq3tXCkDGYVWGIZTRfnvu8LoTDaev7vbwh0omSvVORONr960MQWdKqJDCReIEmTj3PAr73O3aoxz7OPg==", + "license": "MIT", + "dependencies": { + "@noble/curves": "~1.4.0", + "@noble/hashes": "~1.4.0", + "@scure/base": "~1.1.6" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32/node_modules/@noble/curves": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.4.2.tgz", + "integrity": "sha512-TavHr8qycMChk8UwMld0ZDRvatedkzWfH8IiaeGCfymOP5i0hSCozz9vHOL0nkwk7HRMlFnAiKpS2jrUmSybcw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.4.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32/node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.3.0.tgz", + "integrity": "sha512-disdg7gHuTDZtY+ZdkmLpPCk7fxZSu3gBiEGuoC1XYxv9cGx3Z6cpTggCgW6odSOOIXCiDjuGejW+aJKCY/pIQ==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.4.0", + "@scure/base": "~1.1.6" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39/node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@types/parse-json": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", + "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", + "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@whatwg-node/disposablestack": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@whatwg-node/disposablestack/-/disposablestack-0.0.6.tgz", + "integrity": "sha512-LOtTn+JgJvX8WfBVJtF08TGrdjuFzGJc4mkP8EdDI8ADbvO7kiexYep1o8dwnt0okb0jYclCDXF13xU7Ge4zSw==", + "license": "MIT", + "dependencies": { + "@whatwg-node/promise-helpers": "^1.0.0", + "tslib": "^2.6.3" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@whatwg-node/fetch": { + "version": "0.10.13", + "resolved": "https://registry.npmjs.org/@whatwg-node/fetch/-/fetch-0.10.13.tgz", + "integrity": "sha512-b4PhJ+zYj4357zwk4TTuF2nEe0vVtOrwdsrNo5hL+u1ojXNhh1FgJ6pg1jzDlwlT4oBdzfSwaBwMCtFCsIWg8Q==", + "license": "MIT", + "dependencies": { + "@whatwg-node/node-fetch": "^0.8.3", + "urlpattern-polyfill": "^10.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@whatwg-node/node-fetch": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/@whatwg-node/node-fetch/-/node-fetch-0.8.5.tgz", + "integrity": "sha512-4xzCl/zphPqlp9tASLVeUhB5+WJHbuWGYpfoC2q1qh5dw0AqZBW7L27V5roxYWijPxj4sspRAAoOH3d2ztaHUQ==", + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^3.1.1", + "@whatwg-node/disposablestack": "^0.0.6", + "@whatwg-node/promise-helpers": "^1.3.2", + "tslib": "^2.6.3" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@whatwg-node/promise-helpers": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@whatwg-node/promise-helpers/-/promise-helpers-1.3.2.tgz", + "integrity": "sha512-Nst5JdK47VIl9UcGwtv2Rcgyn5lWtZ0/mhRQ4G8NN2isxpq2TO30iqHzmwoJycjWuyUfg3GFXqP/gFHXeV57IA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.3" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/abitype": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-0.7.1.tgz", + "integrity": "sha512-VBkRHTDZf9Myaek/dO3yMmOzB/y2s3Zo6nVU7yaw1G+TvCHAjwaJzNGN9yo4K5D8bU/VZXKP1EJpRhFr862PlQ==", + "license": "MIT", + "peerDependencies": { + "typescript": ">=4.9.4", + "zod": "^3 >=3.19.1" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/abort-error": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/abort-error/-/abort-error-1.0.1.tgz", + "integrity": "sha512-fxqCblJiIPdSXIUrxI0PL+eJG49QdP9SQ70qtB65MVAoMr2rASlOyAbJFOylfB467F/f+5BCLJJq58RYi7mGfg==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/ansi-colors": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ansis": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/ansis/-/ansis-3.17.0.tgz", + "integrity": "sha512-0qWUglt9JEqLFr3w1I1pbrChn1grhaiAR2ocX1PP/flRmxgtwTzPFFFnfIlD6aMOLQZgSuCRlidD70lvx8yhzg==", + "license": "ISC", + "engines": { + "node": ">=14" + } + }, + "node_modules/any-signal": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/any-signal/-/any-signal-4.2.0.tgz", + "integrity": "sha512-LndMvYuAPf4rC195lk7oSFuHOYFpOszIYrNYv0gHAvz+aEhE9qPZLhmrIz5pXP2BSsPOXvsuHDXEGaiQhIh9wA==", + "license": "Apache-2.0 OR MIT", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/apisauce": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/apisauce/-/apisauce-2.1.6.tgz", + "integrity": "sha512-MdxR391op/FucS2YQRfB/NMRyCnHEPDd4h17LRIuVYi0BpGmMhpxc0shbOpfs5ahABuBEffNCGal5EcsydbBWg==", + "license": "MIT", + "dependencies": { + "axios": "^0.21.4" + } + }, + "node_modules/app-module-path": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/app-module-path/-/app-module-path-2.2.0.tgz", + "integrity": "sha512-gkco+qxENJV+8vFcDiiFhuoSvRXb2a/QPqpSoWhVz829VNJfOTnELbBmPmNKFxf3xdNnw4DWCkzkDaavcX/1YQ==", + "license": "BSD-2-Clause" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/assemblyscript": { + "version": "0.19.23", + "resolved": "https://registry.npmjs.org/assemblyscript/-/assemblyscript-0.19.23.tgz", + "integrity": "sha512-fwOQNZVTMga5KRsfY80g7cpOl4PsFQczMwHzdtgoqLXaYhkhavufKb0sB0l3T1DUxpAufA0KNhlbpuuhZUwxMA==", + "license": "Apache-2.0", + "dependencies": { + "binaryen": "102.0.0-nightly.20211028", + "long": "^5.2.0", + "source-map-support": "^0.5.20" + }, + "bin": { + "asc": "bin/asc", + "asinit": "bin/asinit" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/assemblyscript" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/axios": { + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.14.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/binaryen": { + "version": "102.0.0-nightly.20211028", + "resolved": "https://registry.npmjs.org/binaryen/-/binaryen-102.0.0-nightly.20211028.tgz", + "integrity": "sha512-GCJBVB5exbxzzvyt8MGDv/MeUjs6gkXDvf4xOIItRBptYl0Tz5sm1o/uG95YK0L0VeG5ajDu3hRtkBP2kzqC5w==", + "license": "Apache-2.0", + "bin": { + "wasm-opt": "bin/wasm-opt" + } + }, + "node_modules/bl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/bl/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/bl/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/bl/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/blob-to-it": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/blob-to-it/-/blob-to-it-2.0.10.tgz", + "integrity": "sha512-I39vO57y+LBEIcAV7fif0sn96fYOYVqrPiOD+53MxQGv4DBgt1/HHZh0BHheWx2hVe24q5LTSXxqeV1Y3Nzkgg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "browser-readablestream-to-it": "^2.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browser-readablestream-to-it": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/browser-readablestream-to-it/-/browser-readablestream-to-it-2.0.10.tgz", + "integrity": "sha512-I/9hEcRtjct8CzD9sVo9Mm4ntn0D+7tOVrjbPl69XAoOfgJ8NBdOQU+WX+5SHhcELJDb14mWt7zuvyqha+MEAQ==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "license": "MIT", + "dependencies": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "node_modules/buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "license": "MIT" + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", + "license": "MIT" + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/call-bind": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/cborg": { + "version": "4.5.8", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.5.8.tgz", + "integrity": "sha512-6/viltD51JklRhq4L7jC3zgy6gryuG5xfZ3kzpE+PravtyeQLeQmCYLREhQH7pWENg5pY4Yu/XCd6a7dKScVlw==", + "license": "Apache-2.0", + "bin": { + "cborg": "lib/bin.js" + } + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/chalk/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/clean-stack": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-3.0.1.tgz", + "integrity": "sha512-lR9wNiMRcVQjSB3a7xXGLuz4cr4wJuuXlaAEbRutGowQTmlp7R72/DOgN21e8jdwblMWl9UOJMJXarX94pzKdg==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-table3": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.0.tgz", + "integrity": "sha512-gnB85c3MGC7Nm9I/FkiasNBOKjOiO1RNuXXarQms37q4QMpWdlbBgD/VnOStA2faG1dpXMv31RFApjX1/QdgWQ==", + "license": "MIT", + "dependencies": { + "object-assign": "^4.1.0", + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "colors": "^1.1.2" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "license": "MIT" + }, + "node_modules/colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "license": "MIT", + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.1.tgz", + "integrity": "sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ==", + "license": "MIT", + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cosmiconfig/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/dag-jose": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/dag-jose/-/dag-jose-5.1.1.tgz", + "integrity": "sha512-9alfZ8Wh1XOOMel8bMpDqWsDT72ojFQCJPtwZSev9qh4f8GoCV9qrJW8jcOUhcstO8Kfm09FHGo//jqiZq3z9w==", + "license": "(Apache-2.0 OR MIT)", + "dependencies": { + "@ipld/dag-cbor": "^9.0.0", + "multiformats": "~13.1.3" + } + }, + "node_modules/dag-jose/node_modules/multiformats": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.3.tgz", + "integrity": "sha512-CZPi9lFZCM/+7oRolWYsvalsyWQGFo+GpdaTmjxXXomC+nP/W1Rnxb9sUgjvmNmRZ5bOPqRAl4nuK+Ydw/4tGw==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress/-/decompress-4.2.1.tgz", + "integrity": "sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==", + "license": "MIT", + "dependencies": { + "decompress-tar": "^4.0.0", + "decompress-tarbz2": "^4.0.0", + "decompress-targz": "^4.0.0", + "decompress-unzip": "^4.0.1", + "graceful-fs": "^4.1.10", + "make-dir": "^1.0.0", + "pify": "^2.3.0", + "strip-dirs": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-tar": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/decompress-tar/-/decompress-tar-4.1.1.tgz", + "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", + "license": "MIT", + "dependencies": { + "file-type": "^5.2.0", + "is-stream": "^1.1.0", + "tar-stream": "^1.5.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-tarbz2": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/decompress-tarbz2/-/decompress-tarbz2-4.1.1.tgz", + "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", + "license": "MIT", + "dependencies": { + "decompress-tar": "^4.1.0", + "file-type": "^6.1.0", + "is-stream": "^1.1.0", + "seek-bzip": "^1.0.5", + "unbzip2-stream": "^1.0.9" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-tarbz2/node_modules/file-type": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-6.2.0.tgz", + "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-targz": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/decompress-targz/-/decompress-targz-4.1.1.tgz", + "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", + "license": "MIT", + "dependencies": { + "decompress-tar": "^4.1.1", + "file-type": "^5.2.0", + "is-stream": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-unzip": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/decompress-unzip/-/decompress-unzip-4.0.1.tgz", + "integrity": "sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==", + "license": "MIT", + "dependencies": { + "file-type": "^3.8.0", + "get-stream": "^2.2.0", + "pify": "^2.3.0", + "yauzl": "^2.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/decompress-unzip/node_modules/file-type": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-browser": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.4.0.tgz", + "integrity": "sha512-XDuvSq38Hr1MdN47EDvYtx3U0MTqpCEn+F6ft8z2vYDzMrvQhVp0ui9oQdqW3MvK3vqUETglt1tVGgjLuJ5izg==", + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/delay": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", + "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/docker-compose": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-1.3.0.tgz", + "integrity": "sha512-7Gevk/5eGD50+eMD+XDnFnOrruFkL0kSd7jEG4cjmqweDSUhB7i0g8is/nBdVpl+Bx338SqIB2GLKm32M+Vs6g==", + "license": "MIT", + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron-fetch": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/electron-fetch/-/electron-fetch-1.9.1.tgz", + "integrity": "sha512-M9qw6oUILGVrcENMSRRefE1MbHPIz0h79EKIeJWK9v563aT9Qkh8aEHPO1H5vi970wPirNY+jO9OpFoLiMsMGA==", + "license": "MIT", + "dependencies": { + "encoding": "^0.1.13" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/err-code": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", + "integrity": "sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==", + "license": "MIT" + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-promise": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==", + "license": "MIT" + }, + "node_modules/es6-promisify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha512-C+d6UdsYDk0lMebHNR4S2NybQMMngAOnOwYBQjTOiv0MkoJMP0Myw2mgpDLBcpfCmRLxyFqYhS/CfOENq4SJhQ==", + "license": "MIT", + "dependencies": { + "es6-promise": "^4.0.3" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ethereum-cryptography": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ethereum-cryptography/-/ethereum-cryptography-2.2.1.tgz", + "integrity": "sha512-r/W8lkHSiTLxUxW8Rf3u4HGB0xQweG2RyETjywylKZSzLWoWAijRz8WCuOtJ6wah+avllXBqZuk29HCCvhEIRg==", + "license": "MIT", + "dependencies": { + "@noble/curves": "1.4.2", + "@noble/hashes": "1.4.0", + "@scure/bip32": "1.4.0", + "@scure/bip39": "1.3.0" + } + }, + "node_modules/ethereum-cryptography/node_modules/@noble/curves": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.4.2.tgz", + "integrity": "sha512-TavHr8qycMChk8UwMld0ZDRvatedkzWfH8IiaeGCfymOP5i0hSCozz9vHOL0nkwk7HRMlFnAiKpS2jrUmSybcw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.4.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/ethereum-cryptography/node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/execa/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==", + "engines": { + "node": "> 0.1.90" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-levenshtein": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-3.0.0.tgz", + "integrity": "sha512-hKKNajm46uNmTlhHSyZkmToAc56uZJwYq7yrciZjqOxnlfQwERDQJmHPUp7m1m9wx8vgOe8IaCKZ5Kv2k1DdCQ==", + "license": "MIT", + "dependencies": { + "fastest-levenshtein": "^1.0.7" + } + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "license": "MIT", + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/file-type": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz", + "integrity": "sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-each": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs-extra": { + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.2.tgz", + "integrity": "sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fs-jetpack": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/fs-jetpack/-/fs-jetpack-4.3.1.tgz", + "integrity": "sha512-dbeOK84F6BiQzk2yqqCVwCPWTxAvVGJ3fMQc6E2wuEohS28mR6yHngbrKuVCK1KHRx/ccByDylqu4H5PCP2urQ==", + "license": "MIT", + "dependencies": { + "minimatch": "^3.0.2", + "rimraf": "^2.6.3" + } + }, + "node_modules/fs-jetpack/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/fs-jetpack/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-iterator": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/get-iterator/-/get-iterator-1.0.2.tgz", + "integrity": "sha512-v+dm9bNVfOYsY1OrhaCrmyOcYoSeVvbt+hHZ0Au+T+p1y+0Uyj9aMaGIeUTT6xdpRbWzDeYKvfOslPhggQMcsg==", + "license": "MIT" + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", + "integrity": "sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==", + "license": "MIT", + "dependencies": { + "object-assign": "^4.0.1", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/glob": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.3.tgz", + "integrity": "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.0.3", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/gluegun": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/gluegun/-/gluegun-5.2.0.tgz", + "integrity": "sha512-jSUM5xUy2ztYFQANne17OUm/oAd7qSX7EBksS9bQDt9UvLPqcEkeWUebmaposb8Tx7eTTD8uJVWGRe6PYSsYkg==", + "license": "MIT", + "dependencies": { + "apisauce": "^2.1.5", + "app-module-path": "^2.2.0", + "cli-table3": "0.6.0", + "colors": "1.4.0", + "cosmiconfig": "7.0.1", + "cross-spawn": "7.0.3", + "ejs": "3.1.8", + "enquirer": "2.3.6", + "execa": "5.1.1", + "fs-jetpack": "4.3.1", + "lodash.camelcase": "^4.3.0", + "lodash.kebabcase": "^4.1.1", + "lodash.lowercase": "^4.3.0", + "lodash.lowerfirst": "^4.3.1", + "lodash.pad": "^4.5.1", + "lodash.padend": "^4.6.1", + "lodash.padstart": "^4.6.1", + "lodash.repeat": "^4.1.0", + "lodash.snakecase": "^4.1.1", + "lodash.startcase": "^4.4.0", + "lodash.trim": "^4.5.1", + "lodash.trimend": "^4.5.1", + "lodash.trimstart": "^4.5.1", + "lodash.uppercase": "^4.3.0", + "lodash.upperfirst": "^4.3.1", + "ora": "4.0.2", + "pluralize": "^8.0.0", + "semver": "7.3.5", + "which": "2.0.2", + "yargs-parser": "^21.0.0" + }, + "bin": { + "gluegun": "bin/gluegun" + } + }, + "node_modules/gluegun/node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/gluegun/node_modules/ejs": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", + "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gluegun/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/gluegun/node_modules/semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/graphql": { + "version": "16.11.0", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz", + "integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==", + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, + "node_modules/graphql-import-node": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/graphql-import-node/-/graphql-import-node-0.0.5.tgz", + "integrity": "sha512-OXbou9fqh9/Lm7vwXT0XoRN9J5+WCYKnbiTalgFDvkQERITRmcfncZs6aVABedd5B85yQU5EULS4a5pnbpuI0Q==", + "license": "MIT", + "peerDependencies": { + "graphql": "*" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hashlru": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz", + "integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A==", + "license": "MIT" + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-call": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/http-call/-/http-call-5.3.0.tgz", + "integrity": "sha512-ahwimsC23ICE4kPl9xTBjKB4inbRaeLyZeRunC/1Jy/Z6X8tv22MEAjK+KBOMSVLaqXPTTmd8638waVIKLGx2w==", + "license": "ISC", + "dependencies": { + "content-type": "^1.0.4", + "debug": "^4.1.1", + "is-retry-allowed": "^1.1.0", + "is-stream": "^2.0.0", + "parse-json": "^4.0.0", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-call/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/http-call/node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", + "license": "MIT", + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/immutable": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.4.tgz", + "integrity": "sha512-p6u1bG3YSnINT5RQmx/yRZBpenIl30kVxkTLDyHLIMk0gict704Q9n+thfDI7lTRm9vXdDYutVzXhzcThxTnXA==", + "license": "MIT" + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/interface-datastore": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/interface-datastore/-/interface-datastore-8.3.2.tgz", + "integrity": "sha512-R3NLts7pRbJKc3qFdQf+u40hK8XWc0w4Qkx3OFEstC80VoaDUABY/dXA2EJPhtNC+bsrf41Ehvqb6+pnIclyRA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "interface-store": "^6.0.0", + "uint8arrays": "^5.1.0" + } + }, + "node_modules/interface-store": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-6.0.3.tgz", + "integrity": "sha512-+WvfEZnFUhRwFxgz+QCQi7UC6o9AM0EHM9bpIe2Nhqb100NHCsTvNAn4eJgvgV2/tmLo1MP9nGxQKEcZTAueLA==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/ipfs-unixfs": { + "version": "11.2.5", + "resolved": "https://registry.npmjs.org/ipfs-unixfs/-/ipfs-unixfs-11.2.5.tgz", + "integrity": "sha512-uasYJ0GLPbViaTFsOLnL9YPjX5VmhnqtWRriogAHOe4ApmIi9VAOFBzgDHsUW2ub4pEa/EysbtWk126g2vkU/g==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "protons-runtime": "^5.5.0", + "uint8arraylist": "^2.4.8" + } + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-electron": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", + "integrity": "sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==", + "license": "MIT" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-inside-container/node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-natural-number": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-natural-number/-/is-natural-number-4.0.1.tgz", + "integrity": "sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==", + "license": "MIT" + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/iso-url": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/iso-url/-/iso-url-1.2.1.tgz", + "integrity": "sha512-9JPDgCN4B7QPkLtYAAOrEuAWvP9rWvR5offAr0/SeF046wIkglqH3VXgYYP6NcsKslH80UIVgmPqNe3j7tG2ng==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/isomorphic-ws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", + "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==", + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, + "node_modules/it-all": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/it-all/-/it-all-3.0.9.tgz", + "integrity": "sha512-fz1oJJ36ciGnu2LntAlE6SA97bFZpW7Rnt0uEc1yazzR2nKokZLr8lIRtgnpex4NsmaBcvHF+Z9krljWFy/mmg==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/it-first": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/it-first/-/it-first-3.0.9.tgz", + "integrity": "sha512-ZWYun273Gbl7CwiF6kK5xBtIKR56H1NoRaiJek2QzDirgen24u8XZ0Nk+jdnJSuCTPxC2ul1TuXKxu/7eK6NuA==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/it-glob": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/it-glob/-/it-glob-3.0.4.tgz", + "integrity": "sha512-73PbGBTK/dHp5PX4l8pkQH1ozCONP0U+PB3qMqltxPonRJQNomINE3Hn9p02m2GOu95VoeVvSZdHI2N+qub0pw==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "fast-glob": "^3.3.3" + } + }, + "node_modules/it-last": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/it-last/-/it-last-3.0.9.tgz", + "integrity": "sha512-AtfUEnGDBHBEwa1LjrpGHsJMzJAWDipD6zilvhakzJcm+BCvNX8zlX2BsHClHJLLTrsY4lY9JUjc+TQV4W7m1w==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/it-map": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/it-map/-/it-map-3.1.4.tgz", + "integrity": "sha512-QB9PYQdE9fUfpVFYfSxBIyvKynUCgblb143c+ktTK6ZuKSKkp7iH58uYFzagqcJ5HcqIfn1xbfaralHWam+3fg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "it-peekable": "^3.0.0" + } + }, + "node_modules/it-peekable": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/it-peekable/-/it-peekable-3.0.8.tgz", + "integrity": "sha512-7IDBQKSp/dtBxXV3Fj0v3qM1jftJ9y9XrWLRIuU1X6RdKqWiN60syNwP0fiDxZD97b8SYM58dD3uklIk1TTQAw==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/it-pushable": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/it-pushable/-/it-pushable-3.2.3.tgz", + "integrity": "sha512-gzYnXYK8Y5t5b/BnJUr7glfQLO4U5vyb05gPx/TyTw+4Bv1zM9gFk4YsOrnulWefMewlphCjKkakFvj1y99Tcg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "p-defer": "^4.0.0" + } + }, + "node_modules/it-stream-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/it-stream-types/-/it-stream-types-2.0.2.tgz", + "integrity": "sha512-Rz/DEZ6Byn/r9+/SBCuJhpPATDF9D+dz5pbgSUyBsCDtza6wtNATrz/jz1gDyNanC3XdLboriHnOC925bZRBww==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/it-to-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/it-to-stream/-/it-to-stream-1.0.0.tgz", + "integrity": "sha512-pLULMZMAB/+vbdvbZtebC0nWBTbG581lk6w8P7DfIIIKUfa8FbY7Oi0FxZcFPbxvISs7A9E+cMpLDBc1XhpAOA==", + "license": "MIT", + "dependencies": { + "buffer": "^6.0.3", + "fast-fifo": "^1.0.0", + "get-iterator": "^1.0.2", + "p-defer": "^3.0.0", + "p-fifo": "^1.0.0", + "readable-stream": "^3.6.0" + } + }, + "node_modules/it-to-stream/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jackspeak": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", + "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jake": { + "version": "10.9.4", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", + "integrity": "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.6", + "filelist": "^1.0.4", + "picocolors": "^1.1.1" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jayson": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/jayson/-/jayson-4.2.0.tgz", + "integrity": "sha512-VfJ9t1YLwacIubLhONk0KFeosUBwstRWQ0IRT1KDjEjnVnSOVHC3uwugyV7L0c7R9lpVyrUGT2XWiBA1UTtpyg==", + "license": "MIT", + "dependencies": { + "@types/connect": "^3.4.33", + "@types/node": "^12.12.54", + "@types/ws": "^7.4.4", + "commander": "^2.20.3", + "delay": "^5.0.0", + "es6-promisify": "^5.0.0", + "eyes": "^0.1.8", + "isomorphic-ws": "^4.0.1", + "json-stringify-safe": "^5.0.1", + "stream-json": "^1.9.1", + "uuid": "^8.3.2", + "ws": "^7.5.10" + }, + "bin": { + "jayson": "bin/jayson.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jayson/node_modules/@types/node": { + "version": "12.20.55", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz", + "integrity": "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==", + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kubo-rpc-client": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/kubo-rpc-client/-/kubo-rpc-client-5.4.1.tgz", + "integrity": "sha512-v86bQWtyA//pXTrt9y4iEwjW6pt1gA18Z1famWXIR/HN5TFdYwQ3yHOlRE6JSWBDQ0rR6FOMyrrGy8To78mXow==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "@ipld/dag-cbor": "^9.0.0", + "@ipld/dag-json": "^10.0.0", + "@ipld/dag-pb": "^4.0.0", + "@libp2p/crypto": "^5.0.0", + "@libp2p/interface": "^2.0.0", + "@libp2p/logger": "^5.0.0", + "@libp2p/peer-id": "^5.0.0", + "@multiformats/multiaddr": "^12.2.1", + "@multiformats/multiaddr-to-uri": "^11.0.0", + "any-signal": "^4.1.1", + "blob-to-it": "^2.0.5", + "browser-readablestream-to-it": "^2.0.5", + "dag-jose": "^5.0.0", + "electron-fetch": "^1.9.1", + "err-code": "^3.0.1", + "ipfs-unixfs": "^11.1.4", + "iso-url": "^1.2.1", + "it-all": "^3.0.4", + "it-first": "^3.0.4", + "it-glob": "^3.0.1", + "it-last": "^3.0.4", + "it-map": "^3.0.5", + "it-peekable": "^3.0.3", + "it-to-stream": "^1.0.0", + "merge-options": "^3.0.4", + "multiformats": "^13.1.0", + "nanoid": "^5.0.7", + "native-fetch": "^4.0.2", + "parse-duration": "^2.1.2", + "react-native-fetch-api": "^3.0.0", + "stream-to-it": "^1.0.1", + "uint8arrays": "^5.0.3", + "wherearewe": "^2.0.1" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/lodash": { + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", + "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/lodash.kebabcase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.kebabcase/-/lodash.kebabcase-4.1.1.tgz", + "integrity": "sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==", + "license": "MIT" + }, + "node_modules/lodash.lowercase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.lowercase/-/lodash.lowercase-4.3.0.tgz", + "integrity": "sha512-UcvP1IZYyDKyEL64mmrwoA1AbFu5ahojhTtkOUr1K9dbuxzS9ev8i4TxMMGCqRC9TE8uDaSoufNAXxRPNTseVA==", + "license": "MIT" + }, + "node_modules/lodash.lowerfirst": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/lodash.lowerfirst/-/lodash.lowerfirst-4.3.1.tgz", + "integrity": "sha512-UUKX7VhP1/JL54NXg2aq/E1Sfnjjes8fNYTNkPU8ZmsaVeBvPHKdbNaN79Re5XRL01u6wbq3j0cbYZj71Fcu5w==", + "license": "MIT" + }, + "node_modules/lodash.pad": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.5.1.tgz", + "integrity": "sha512-mvUHifnLqM+03YNzeTBS1/Gr6JRFjd3rRx88FHWUvamVaT9k2O/kXha3yBSOwB9/DTQrSTLJNHvLBBt2FdX7Mg==", + "license": "MIT" + }, + "node_modules/lodash.padend": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz", + "integrity": "sha512-sOQs2aqGpbl27tmCS1QNZA09Uqp01ZzWfDUoD+xzTii0E7dSQfRKcRetFwa+uXaxaqL+TKm7CgD2JdKP7aZBSw==", + "license": "MIT" + }, + "node_modules/lodash.padstart": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.6.1.tgz", + "integrity": "sha512-sW73O6S8+Tg66eY56DBk85aQzzUJDtpoXFBgELMd5P/SotAguo+1kYO6RuYgXxA4HJH3LFTFPASX6ET6bjfriw==", + "license": "MIT" + }, + "node_modules/lodash.repeat": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.1.0.tgz", + "integrity": "sha512-eWsgQW89IewS95ZOcr15HHCX6FVDxq3f2PNUIng3fyzsPev9imFQxIYdFZ6crl8L56UR6ZlGDLcEb3RZsCSSqw==", + "license": "MIT" + }, + "node_modules/lodash.snakecase": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", + "integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==", + "license": "MIT" + }, + "node_modules/lodash.startcase": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.startcase/-/lodash.startcase-4.4.0.tgz", + "integrity": "sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==", + "license": "MIT" + }, + "node_modules/lodash.trim": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/lodash.trim/-/lodash.trim-4.5.1.tgz", + "integrity": "sha512-nJAlRl/K+eiOehWKDzoBVrSMhK0K3A3YQsUNXHQa5yIrKBAhsZgSu3KoAFoFT+mEgiyBHddZ0pRk1ITpIp90Wg==", + "license": "MIT" + }, + "node_modules/lodash.trimend": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/lodash.trimend/-/lodash.trimend-4.5.1.tgz", + "integrity": "sha512-lsD+k73XztDsMBKPKvzHXRKFNMohTjoTKIIo4ADLn5dA65LZ1BqlAvSXhR2rPEC3BgAUQnzMnorqDtqn2z4IHA==", + "license": "MIT" + }, + "node_modules/lodash.trimstart": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/lodash.trimstart/-/lodash.trimstart-4.5.1.tgz", + "integrity": "sha512-b/+D6La8tU76L/61/aN0jULWHkT0EeJCmVstPBn/K9MtD2qBW83AsBNrr63dKuWYwVMO7ucv13QNO/Ek/2RKaQ==", + "license": "MIT" + }, + "node_modules/lodash.uppercase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.uppercase/-/lodash.uppercase-4.3.0.tgz", + "integrity": "sha512-+Nbnxkj7s8K5U8z6KnEYPGUOGp3woZbB7Ecs7v3LkkjLQSm2kP9SKIILitN1ktn2mB/tmM9oSlku06I+/lH7QA==", + "license": "MIT" + }, + "node_modules/lodash.upperfirst": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/lodash.upperfirst/-/lodash.upperfirst-4.3.1.tgz", + "integrity": "sha512-sReKOYJIJf74dhJONhU4e0/shzi1trVbSWDOhKYE5XV2O+H7Sb2Dihwuc7xWxVl+DgFPyTqIN3zMfT9cq5iWDg==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-3.0.0.tgz", + "integrity": "sha512-dSkNGuI7iG3mfvDzUuYZyvk5dD9ocYCYzNU6CYDE6+Xqd+gwme6Z00NS3dUh8mq/73HaEtT7m6W+yUPtU6BZnQ==", + "license": "MIT", + "dependencies": { + "chalk": "^2.4.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "11.2.4", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.4.tgz", + "integrity": "sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/main-event": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/main-event/-/main-event-1.0.1.tgz", + "integrity": "sha512-NWtdGrAca/69fm6DIVd8T9rtfDII4Q8NQbIbsKQq2VzS9eqOGYs8uaNQjcuaCq/d9H/o625aOTJX2Qoxzqw0Pw==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "license": "MIT", + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/make-dir/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/matchstick-as": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/matchstick-as/-/matchstick-as-0.6.0.tgz", + "integrity": "sha512-E36fWsC1AbCkBFt05VsDDRoFvGSdcZg6oZJrtIe/YDBbuFh8SKbR5FcoqDhNWqSN+F7bN/iS2u8Md0SM+4pUpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "wabt": "1.0.24" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/merge-options": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/merge-options/-/merge-options-3.0.4.tgz", + "integrity": "sha512-2Sug1+knBjkaMsMgf1ctR1Ujx+Ayku4EdJN4Z+C2+JzoeF7A3OZ9KM2GY0CpQS51NR61LTurMJrRKPhSs3ZRTQ==", + "license": "MIT", + "dependencies": { + "is-plain-obj": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/multiformats": { + "version": "13.4.2", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.4.2.tgz", + "integrity": "sha512-eh6eHCrRi1+POZ3dA+Dq1C6jhP1GNtr9CRINMb67OKzqW9I5DUuZM/3jLPlzhgpGeiNUlEGEbkCYChXMCc/8DQ==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/mute-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", + "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/nanoid": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.6.tgz", + "integrity": "sha512-c7+7RQ+dMB5dPwwCp4ee1/iV/q2P6aK1mTZcfr1BTuVlyW9hJYiMPybJCcnBlQtuSmTIWNeazm/zqNoZSSElBg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^18 || >=20" + } + }, + "node_modules/native-fetch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/native-fetch/-/native-fetch-4.0.2.tgz", + "integrity": "sha512-4QcVlKFtv2EYVS5MBgsGX5+NWKtbDbIECdUXDBGDMAZXq3Jkv9zf+y8iS7Ub8fEdga3GpYeazp9gauNqXHJOCg==", + "license": "MIT", + "peerDependencies": { + "undici": "*" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", + "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "wsl-utils": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/ora/-/ora-4.0.2.tgz", + "integrity": "sha512-YUOZbamht5mfLxPmk4M35CD/5DuOkAacxlEUbStVXpBAt4fyhBf+vZHI/HRkI++QUp3sNoeA2Gw4C+hi4eGSig==", + "license": "MIT", + "dependencies": { + "chalk": "^2.4.2", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.2.0", + "is-interactive": "^1.0.0", + "log-symbols": "^3.0.0", + "strip-ansi": "^5.2.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ora/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ora/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-defer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-4.0.1.tgz", + "integrity": "sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-fifo": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-fifo/-/p-fifo-1.0.0.tgz", + "integrity": "sha512-IjoCxXW48tqdtDFz6fqo5q1UfFVjjVZe8TC1QRflvNUJtNfCUhxOUw6MOVZhDPjqhSzc26xKdugsO17gmzd5+A==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.0.0", + "p-defer": "^3.0.0" + } + }, + "node_modules/p-fifo/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/p-queue": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.1.0.tgz", + "integrity": "sha512-O/ZPaXuQV29uSLbxWBGGZO1mCQXV2BLIwUr59JUU9SoH76mnYvtms7aafH/isNSNGwuEfP6W/4xD0/TJXxrizw==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-duration": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-2.1.5.tgz", + "integrity": "sha512-/IX1KRw6zHDOOJrgIz++gvFASbFl7nc8GEXaLdD7d1t1x/GnrK6hh5Fgk8G3RLpkIEi4tsGj9pupGLWNg0EiJA==", + "license": "MIT" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", + "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "license": "MIT", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pluralize": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", + "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/progress-events": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/progress-events/-/progress-events-1.0.1.tgz", + "integrity": "sha512-MOzLIwhpt64KIVN64h1MwdKWiyKFNc/S6BoYKPIVUHFg0/eIEyBulhWCgn678v/4c0ri3FdGuzXymNCv02MUIw==", + "license": "Apache-2.0 OR MIT" + }, + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "license": "ISC" + }, + "node_modules/protons-runtime": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/protons-runtime/-/protons-runtime-5.6.0.tgz", + "integrity": "sha512-/Kde+sB9DsMFrddJT/UZWe6XqvL7SL5dbag/DBCElFKhkwDj7XKt53S+mzLyaDP5OqS0wXjV5SA572uWDaT0Hg==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "uint8-varint": "^2.0.2", + "uint8arraylist": "^2.4.3", + "uint8arrays": "^5.0.1" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/react-native-fetch-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-native-fetch-api/-/react-native-fetch-api-3.0.0.tgz", + "integrity": "sha512-g2rtqPjdroaboDKTsJCTlcmtw54E25OjyaunUP0anOZn4Fuo2IKs8BVfe02zVggA/UysbmfSnRJIqtNkAgggNA==", + "license": "MIT", + "dependencies": { + "p-defer": "^3.0.0" + } + }, + "node_modules/react-native-fetch-api/node_modules/p-defer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-3.0.0.tgz", + "integrity": "sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/registry-auth-token": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-5.1.1.tgz", + "integrity": "sha512-P7B4+jq8DeD2nMsAcdfaqHbssgHtZ7Z5+++a5ask90fvmJ8p5je4mOa+wzu+DB4vQ5tdJV/xywY+UnVFeQLV5Q==", + "license": "MIT", + "dependencies": { + "@pnpm/npm-conf": "^3.0.2" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/restore-cursor/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/rimraf/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/run-applescript": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/seek-bzip": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.6.tgz", + "integrity": "sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==", + "license": "MIT", + "dependencies": { + "commander": "^2.8.1" + }, + "bin": { + "seek-bunzip": "bin/seek-bunzip", + "seek-table": "bin/seek-bzip-table" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, + "node_modules/stream-to-it": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-to-it/-/stream-to-it-1.0.1.tgz", + "integrity": "sha512-AqHYAYPHcmvMrcLNgncE/q0Aj/ajP6A4qGhxP6EVn7K3YTNs0bJpJyk57wc2Heb7MUL64jurvmnmui8D9kjZgA==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "it-stream-types": "^2.0.1" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-dirs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", + "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", + "license": "MIT", + "dependencies": { + "is-natural-number": "^4.0.1" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "license": "MIT", + "dependencies": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/tar-stream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/tar-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/tar-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "license": "MIT", + "dependencies": { + "tmp": "^0.2.0" + } + }, + "node_modules/to-buffer": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", + "integrity": "sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==", + "license": "MIT", + "dependencies": { + "isarray": "^2.0.5", + "safe-buffer": "^5.2.1", + "typed-array-buffer": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typed-array-buffer": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "is-typed-array": "^1.1.14" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uint8-varint": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", + "integrity": "sha512-FwpTa7ZGA/f/EssWAb5/YV6pHgVF1fViKdW8cWaEarjB8t7NyofSWBdOTyFPaGuUG4gx3v1O3PQ8etsiOs3lcw==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "uint8arraylist": "^2.0.0", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/uint8arraylist": { + "version": "2.4.8", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.8.tgz", + "integrity": "sha512-vc1PlGOzglLF0eae1M8mLRTBivsvrGsdmJ5RbK3e+QRvRLOZfZhQROTwH/OfyF3+ZVUg9/8hE8bmKP2CvP9quQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "uint8arrays": "^5.0.1" + } + }, + "node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "license": "MIT", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, + "node_modules/unbzip2-stream/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/undici": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", + "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/urlpattern-polyfill": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.1.0.tgz", + "integrity": "sha512-IGjKp/o0NL3Bso1PymYURCJxMPNAf/ILOpendP9f5B6e1rTJgdgiOvgfoT8VxCAdY+Wisb9uhGaJJf3yZ2V9nw==", + "license": "MIT" + }, + "node_modules/utf8-codec": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/utf8-codec/-/utf8-codec-1.0.0.tgz", + "integrity": "sha512-S/QSLezp3qvG4ld5PUfXiH7mCFxLKjSVZRFkB3DOjgwHuJPFDkInAXc/anf7BAbHt/D38ozDzL+QMZ6/7gsI6w==", + "license": "MIT" + }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/wabt": { + "version": "1.0.24", + "resolved": "https://registry.npmjs.org/wabt/-/wabt-1.0.24.tgz", + "integrity": "sha512-8l7sIOd3i5GWfTWciPL0+ff/FK/deVK2Q6FN+MPz4vfUcD78i2M/49XJTwF6aml91uIiuXJEsLKWMB2cw/mtKg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "wasm-decompile": "bin/wasm-decompile", + "wasm-interp": "bin/wasm-interp", + "wasm-objdump": "bin/wasm-objdump", + "wasm-opcodecnt": "bin/wasm-opcodecnt", + "wasm-strip": "bin/wasm-strip", + "wasm-validate": "bin/wasm-validate", + "wasm2c": "bin/wasm2c", + "wasm2wat": "bin/wasm2wat", + "wat2wasm": "bin/wat2wasm" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/weald": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/weald/-/weald-1.1.1.tgz", + "integrity": "sha512-PaEQShzMCz8J/AD2N3dJMc1hTZWkJeLKS2NMeiVkV5KDHwgZe7qXLEzyodsT/SODxWDdXJJqocuwf3kHzcXhSQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "ms": "^3.0.0-canary.1", + "supports-color": "^10.0.0" + } + }, + "node_modules/weald/node_modules/ms": { + "version": "3.0.0-canary.202508261828", + "resolved": "https://registry.npmjs.org/ms/-/ms-3.0.0-canary.202508261828.tgz", + "integrity": "sha512-NotsCoUCIUkojWCzQff4ttdCfIPoA1UGZsyQbi7KmqkNRfKCrvga8JJi2PknHymHOuor0cJSn/ylj52Cbt2IrQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/weald/node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/web3-errors": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/web3-errors/-/web3-errors-1.3.1.tgz", + "integrity": "sha512-w3NMJujH+ZSW4ltIZZKtdbkbyQEvBzyp3JRn59Ckli0Nz4VMsVq8aF1bLWM7A2kuQ+yVEm3ySeNU+7mSRwx7RQ==", + "license": "LGPL-3.0", + "dependencies": { + "web3-types": "^1.10.0" + }, + "engines": { + "node": ">=14", + "npm": ">=6.12.0" + } + }, + "node_modules/web3-eth-abi": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/web3-eth-abi/-/web3-eth-abi-4.4.1.tgz", + "integrity": "sha512-60ecEkF6kQ9zAfbTY04Nc9q4eEYM0++BySpGi8wZ2PD1tw/c0SDvsKhV6IKURxLJhsDlb08dATc3iD6IbtWJmg==", + "license": "LGPL-3.0", + "dependencies": { + "abitype": "0.7.1", + "web3-errors": "^1.3.1", + "web3-types": "^1.10.0", + "web3-utils": "^4.3.3", + "web3-validator": "^2.0.6" + }, + "engines": { + "node": ">=14", + "npm": ">=6.12.0" + } + }, + "node_modules/web3-types": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/web3-types/-/web3-types-1.10.0.tgz", + "integrity": "sha512-0IXoaAFtFc8Yin7cCdQfB9ZmjafrbP6BO0f0KT/khMhXKUpoJ6yShrVhiNpyRBo8QQjuOagsWzwSK2H49I7sbw==", + "license": "LGPL-3.0", + "engines": { + "node": ">=14", + "npm": ">=6.12.0" + } + }, + "node_modules/web3-utils": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/web3-utils/-/web3-utils-4.3.3.tgz", + "integrity": "sha512-kZUeCwaQm+RNc2Bf1V3BYbF29lQQKz28L0y+FA4G0lS8IxtJVGi5SeDTUkpwqqkdHHC7JcapPDnyyzJ1lfWlOw==", + "license": "LGPL-3.0", + "dependencies": { + "ethereum-cryptography": "^2.0.0", + "eventemitter3": "^5.0.1", + "web3-errors": "^1.3.1", + "web3-types": "^1.10.0", + "web3-validator": "^2.0.6" + }, + "engines": { + "node": ">=14", + "npm": ">=6.12.0" + } + }, + "node_modules/web3-validator": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/web3-validator/-/web3-validator-2.0.6.tgz", + "integrity": "sha512-qn9id0/l1bWmvH4XfnG/JtGKKwut2Vokl6YXP5Kfg424npysmtRLe9DgiNBM9Op7QL/aSiaA0TVXibuIuWcizg==", + "license": "LGPL-3.0", + "dependencies": { + "ethereum-cryptography": "^2.0.0", + "util": "^0.12.5", + "web3-errors": "^1.2.0", + "web3-types": "^1.6.0", + "zod": "^3.21.4" + }, + "engines": { + "node": ">=14", + "npm": ">=6.12.0" + } + }, + "node_modules/wherearewe": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/wherearewe/-/wherearewe-2.0.1.tgz", + "integrity": "sha512-XUguZbDxCA2wBn2LoFtcEhXL6AXo+hVjGonwhSTTTU9SzbWG8Xu3onNIpzf9j/mYUcJQ0f+m37SzG77G851uFw==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "is-electron": "^2.2.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-typed-array": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.20.tgz", + "integrity": "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "license": "MIT", + "dependencies": { + "string-width": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/wsl-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", + "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", + "license": "MIT", + "dependencies": { + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wsl-utils/node_modules/is-wsl": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", + "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/yaml": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", + "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/localnet/subgraph/gateway/package.json b/localnet/subgraph/gateway/package.json new file mode 100644 index 000000000..2e1149994 --- /dev/null +++ b/localnet/subgraph/gateway/package.json @@ -0,0 +1,18 @@ +{ + "name": "gateway_graph", + "license": "UNLICENSED", + "scripts": { + "codegen": "graph codegen", + "build": "graph build", + "deploy": "graph deploy --node https://api.studio.thegraph.com/deploy/ gateway_graph", + "create-local": "graph create --node http://localhost:8020/ gateway_graph", + "remove-local": "graph remove --node http://localhost:8020/ gateway_graph", + "deploy-local": "graph deploy --node http://localhost:8020/ --ipfs http://localhost:5001 gateway_graph", + "test": "graph test" + }, + "dependencies": { + "@graphprotocol/graph-cli": "0.98.1", + "@graphprotocol/graph-ts": "0.37.0" + }, + "devDependencies": { "matchstick-as": "0.6.0" } +} diff --git a/localnet/subgraph/gateway/schema.graphql b/localnet/subgraph/gateway/schema.graphql new file mode 100644 index 000000000..b89532d7f --- /dev/null +++ b/localnet/subgraph/gateway/schema.graphql @@ -0,0 +1,124 @@ +type BridgeIn @entity(immutable: true) { + id: Bytes! + depositorAddress: Bytes! # address + instanceId: Bytes! # bytes16 + peginAmountSats: BigInt! # uint64 + feeAmountSats: BigInt! # uint64 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type BridgeInRequest @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + depositorAddress: Bytes! # address + peginAmountSats: BigInt! # uint64 + txnFees: [BigInt!]! # uint64[3] + userInputs: [Bytes!]! # tuple[] + userXonlyPubkey: Bytes! # bytes32 + userChangeAddress: String! # string + userRefundAddress: String! # string + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type CancelWithdraw @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + triggerAddress: Bytes! # address + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type CommitteeResponse @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + committeeAddress: Bytes! # address + committeePubkey: Bytes! # bytes + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type InitWithdraw @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + operatorAddress: Bytes! # address + withdrawAmountSats: BigInt! # uint64 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type Initialized @entity(immutable: true) { + id: Bytes! + version: BigInt! # uint64 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type PostGraphData @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type ProceedWithdraw @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + kickoffTxid: Bytes! # bytes32 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type WithdrawDisproved @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + disproveTxType: Int! # uint8 + txnIndex: BigInt! # uint256 + challengeStartTxid: Bytes! # bytes32 + challengeFinishTxid: Bytes! # bytes32 + challengerAddress: Bytes! # address + disproverAddress: Bytes! # address + challengerRewardAmount: BigInt! # uint256 + disproverRewardAmount: BigInt! # uint256 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type WithdrawHappyPath @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + take1Txid: Bytes! # bytes32 + operatorAddress: Bytes! # address + rewardAmountSats: BigInt! # uint64 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} + +type WithdrawUnhappyPath @entity(immutable: true) { + id: Bytes! + instanceId: Bytes! # bytes16 + graphId: Bytes! # bytes16 + take2Txid: Bytes! # bytes32 + operatorAddress: Bytes! # address + rewardAmountSats: BigInt! # uint64 + blockNumber: BigInt! + blockTimestamp: BigInt! + transactionHash: Bytes! +} diff --git a/localnet/subgraph/gateway/src/gateway-upgradeable.ts b/localnet/subgraph/gateway/src/gateway-upgradeable.ts new file mode 100644 index 000000000..b4dd32103 --- /dev/null +++ b/localnet/subgraph/gateway/src/gateway-upgradeable.ts @@ -0,0 +1,209 @@ +import { + BridgeIn as BridgeInEvent, + BridgeInRequest as BridgeInRequestEvent, + CancelWithdraw as CancelWithdrawEvent, + CommitteeResponse as CommitteeResponseEvent, + InitWithdraw as InitWithdrawEvent, + Initialized as InitializedEvent, + PostGraphData as PostGraphDataEvent, + ProceedWithdraw as ProceedWithdrawEvent, + WithdrawDisproved as WithdrawDisprovedEvent, + WithdrawHappyPath as WithdrawHappyPathEvent, + WithdrawUnhappyPath as WithdrawUnhappyPathEvent +} from "../generated/GatewayUpgradeable/GatewayUpgradeable" +import { + BridgeIn, + BridgeInRequest, + CancelWithdraw, + CommitteeResponse, + InitWithdraw, + Initialized, + PostGraphData, + ProceedWithdraw, + WithdrawDisproved, + WithdrawHappyPath, + WithdrawUnhappyPath +} from "../generated/schema" +import { Bytes } from "@graphprotocol/graph-ts" + +export function handleBridgeIn(event: BridgeInEvent): void { + let entity = new BridgeIn( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.depositorAddress = event.params.depositorAddress + entity.instanceId = event.params.instanceId + entity.peginAmountSats = event.params.peginAmountSats + entity.feeAmountSats = event.params.feeAmountSats + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleBridgeInRequest(event: BridgeInRequestEvent): void { + let entity = new BridgeInRequest( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.depositorAddress = event.params.depositorAddress + entity.peginAmountSats = event.params.peginAmountSats + entity.txnFees = event.params.txnFees + entity.userInputs = changetype(event.params.userInputs) + entity.userXonlyPubkey = event.params.userXonlyPubkey + entity.userChangeAddress = event.params.userChangeAddress + entity.userRefundAddress = event.params.userRefundAddress + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleCancelWithdraw(event: CancelWithdrawEvent): void { + let entity = new CancelWithdraw( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + entity.triggerAddress = event.params.triggerAddress + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleCommitteeResponse(event: CommitteeResponseEvent): void { + let entity = new CommitteeResponse( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.committeeAddress = event.params.committeeAddress + entity.committeePubkey = event.params.committeePubkey + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleInitWithdraw(event: InitWithdrawEvent): void { + let entity = new InitWithdraw( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + entity.operatorAddress = event.params.operatorAddress + entity.withdrawAmountSats = event.params.withdrawAmountSats + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleInitialized(event: InitializedEvent): void { + let entity = new Initialized( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.version = event.params.version + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handlePostGraphData(event: PostGraphDataEvent): void { + let entity = new PostGraphData( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleProceedWithdraw(event: ProceedWithdrawEvent): void { + let entity = new ProceedWithdraw( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + entity.kickoffTxid = event.params.kickoffTxid + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleWithdrawDisproved(event: WithdrawDisprovedEvent): void { + let entity = new WithdrawDisproved( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + entity.disproveTxType = event.params.disproveTxType + entity.txnIndex = event.params.txnIndex + entity.challengeStartTxid = event.params.challengeStartTxid + entity.challengeFinishTxid = event.params.challengeFinishTxid + entity.challengerAddress = event.params.challengerAddress + entity.disproverAddress = event.params.disproverAddress + entity.challengerRewardAmount = event.params.challengerRewardAmount + entity.disproverRewardAmount = event.params.disproverRewardAmount + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleWithdrawHappyPath(event: WithdrawHappyPathEvent): void { + let entity = new WithdrawHappyPath( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + entity.take1Txid = event.params.take1Txid + entity.operatorAddress = event.params.operatorAddress + entity.rewardAmountSats = event.params.rewardAmountSats + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} + +export function handleWithdrawUnhappyPath( + event: WithdrawUnhappyPathEvent +): void { + let entity = new WithdrawUnhappyPath( + event.transaction.hash.concatI32(event.logIndex.toI32()) + ) + entity.instanceId = event.params.instanceId + entity.graphId = event.params.graphId + entity.take2Txid = event.params.take2Txid + entity.operatorAddress = event.params.operatorAddress + entity.rewardAmountSats = event.params.rewardAmountSats + + entity.blockNumber = event.block.number + entity.blockTimestamp = event.block.timestamp + entity.transactionHash = event.transaction.hash + + entity.save() +} diff --git a/localnet/subgraph/gateway/subgraph.yaml b/localnet/subgraph/gateway/subgraph.yaml new file mode 100644 index 000000000..cc54b746b --- /dev/null +++ b/localnet/subgraph/gateway/subgraph.yaml @@ -0,0 +1,56 @@ +specVersion: 1.3.0 +indexerHints: + prune: auto +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum + name: GatewayUpgradeable + network: goat-local + source: + address: "0x0000000000000000000000000000000000000000" + abi: GatewayUpgradeable + startBlock: 0 + mapping: + kind: ethereum/events + apiVersion: 0.0.9 + language: wasm/assemblyscript + entities: + - BridgeIn + - BridgeInRequest + - CancelWithdraw + - CommitteeResponse + - InitWithdraw + - Initialized + - PostGraphData + - ProceedWithdraw + - WithdrawDisproved + - WithdrawHappyPath + - WithdrawUnhappyPath + abis: + - name: GatewayUpgradeable + file: ./abis/GatewayUpgradeable.json + eventHandlers: + - event: BridgeIn(indexed address,indexed bytes16,indexed uint64,uint64) + handler: handleBridgeIn + - event: BridgeInRequest(indexed bytes16,indexed address,uint64,uint64[3],(bytes32,uint32,uint64)[],bytes32,string,string) + handler: handleBridgeInRequest + - event: CancelWithdraw(indexed bytes16,indexed bytes16,indexed address) + handler: handleCancelWithdraw + - event: CommitteeResponse(indexed bytes16,indexed address,bytes) + handler: handleCommitteeResponse + - event: InitWithdraw(indexed bytes16,indexed bytes16,indexed address,uint64) + handler: handleInitWithdraw + - event: Initialized(uint64) + handler: handleInitialized + - event: PostGraphData(indexed bytes16,indexed bytes16) + handler: handlePostGraphData + - event: ProceedWithdraw(indexed bytes16,indexed bytes16,bytes32) + handler: handleProceedWithdraw + - event: WithdrawDisproved(indexed bytes16,indexed bytes16,uint8,uint256,bytes32,bytes32,address,address,uint256,uint256) + handler: handleWithdrawDisproved + - event: WithdrawHappyPath(indexed bytes16,indexed bytes16,bytes32,indexed address,uint64) + handler: handleWithdrawHappyPath + - event: WithdrawUnhappyPath(indexed bytes16,indexed bytes16,bytes32,indexed address,uint64) + handler: handleWithdrawUnhappyPath + file: ./src/gateway-upgradeable.ts diff --git a/localnet/subgraph/gateway/tests/gateway-upgradeable-utils.ts b/localnet/subgraph/gateway/tests/gateway-upgradeable-utils.ts new file mode 100644 index 000000000..572fbae0f --- /dev/null +++ b/localnet/subgraph/gateway/tests/gateway-upgradeable-utils.ts @@ -0,0 +1,438 @@ +import { newMockEvent } from "matchstick-as" +import { ethereum, Address, Bytes, BigInt } from "@graphprotocol/graph-ts" +import { + BridgeIn, + BridgeInRequest, + CancelWithdraw, + CommitteeResponse, + InitWithdraw, + Initialized, + PostGraphData, + ProceedWithdraw, + WithdrawDisproved, + WithdrawHappyPath, + WithdrawUnhappyPath +} from "../generated/GatewayUpgradeable/GatewayUpgradeable" + +export function createBridgeInEvent( + depositorAddress: Address, + instanceId: Bytes, + peginAmountSats: BigInt, + feeAmountSats: BigInt +): BridgeIn { + let bridgeInEvent = changetype(newMockEvent()) + + bridgeInEvent.parameters = new Array() + + bridgeInEvent.parameters.push( + new ethereum.EventParam( + "depositorAddress", + ethereum.Value.fromAddress(depositorAddress) + ) + ) + bridgeInEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + bridgeInEvent.parameters.push( + new ethereum.EventParam( + "peginAmountSats", + ethereum.Value.fromUnsignedBigInt(peginAmountSats) + ) + ) + bridgeInEvent.parameters.push( + new ethereum.EventParam( + "feeAmountSats", + ethereum.Value.fromUnsignedBigInt(feeAmountSats) + ) + ) + + return bridgeInEvent +} + +export function createBridgeInRequestEvent( + instanceId: Bytes, + depositorAddress: Address, + peginAmountSats: BigInt, + txnFees: Array, + userInputs: Array, + userXonlyPubkey: Bytes, + userChangeAddress: string, + userRefundAddress: string +): BridgeInRequest { + let bridgeInRequestEvent = changetype(newMockEvent()) + + bridgeInRequestEvent.parameters = new Array() + + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "depositorAddress", + ethereum.Value.fromAddress(depositorAddress) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "peginAmountSats", + ethereum.Value.fromUnsignedBigInt(peginAmountSats) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "txnFees", + ethereum.Value.fromUnsignedBigIntArray(txnFees) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "userInputs", + ethereum.Value.fromTupleArray(userInputs) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "userXonlyPubkey", + ethereum.Value.fromFixedBytes(userXonlyPubkey) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "userChangeAddress", + ethereum.Value.fromString(userChangeAddress) + ) + ) + bridgeInRequestEvent.parameters.push( + new ethereum.EventParam( + "userRefundAddress", + ethereum.Value.fromString(userRefundAddress) + ) + ) + + return bridgeInRequestEvent +} + +export function createCancelWithdrawEvent( + instanceId: Bytes, + graphId: Bytes, + triggerAddress: Address +): CancelWithdraw { + let cancelWithdrawEvent = changetype(newMockEvent()) + + cancelWithdrawEvent.parameters = new Array() + + cancelWithdrawEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + cancelWithdrawEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + cancelWithdrawEvent.parameters.push( + new ethereum.EventParam( + "triggerAddress", + ethereum.Value.fromAddress(triggerAddress) + ) + ) + + return cancelWithdrawEvent +} + +export function createCommitteeResponseEvent( + instanceId: Bytes, + committeeAddress: Address, + committeePubkey: Bytes +): CommitteeResponse { + let committeeResponseEvent = changetype(newMockEvent()) + + committeeResponseEvent.parameters = new Array() + + committeeResponseEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + committeeResponseEvent.parameters.push( + new ethereum.EventParam( + "committeeAddress", + ethereum.Value.fromAddress(committeeAddress) + ) + ) + committeeResponseEvent.parameters.push( + new ethereum.EventParam( + "committeePubkey", + ethereum.Value.fromBytes(committeePubkey) + ) + ) + + return committeeResponseEvent +} + +export function createInitWithdrawEvent( + instanceId: Bytes, + graphId: Bytes, + operatorAddress: Address, + withdrawAmountSats: BigInt +): InitWithdraw { + let initWithdrawEvent = changetype(newMockEvent()) + + initWithdrawEvent.parameters = new Array() + + initWithdrawEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + initWithdrawEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + initWithdrawEvent.parameters.push( + new ethereum.EventParam( + "operatorAddress", + ethereum.Value.fromAddress(operatorAddress) + ) + ) + initWithdrawEvent.parameters.push( + new ethereum.EventParam( + "withdrawAmountSats", + ethereum.Value.fromUnsignedBigInt(withdrawAmountSats) + ) + ) + + return initWithdrawEvent +} + +export function createInitializedEvent(version: BigInt): Initialized { + let initializedEvent = changetype(newMockEvent()) + + initializedEvent.parameters = new Array() + + initializedEvent.parameters.push( + new ethereum.EventParam( + "version", + ethereum.Value.fromUnsignedBigInt(version) + ) + ) + + return initializedEvent +} + +export function createPostGraphDataEvent( + instanceId: Bytes, + graphId: Bytes +): PostGraphData { + let postGraphDataEvent = changetype(newMockEvent()) + + postGraphDataEvent.parameters = new Array() + + postGraphDataEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + postGraphDataEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + + return postGraphDataEvent +} + +export function createProceedWithdrawEvent( + instanceId: Bytes, + graphId: Bytes, + kickoffTxid: Bytes +): ProceedWithdraw { + let proceedWithdrawEvent = changetype(newMockEvent()) + + proceedWithdrawEvent.parameters = new Array() + + proceedWithdrawEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + proceedWithdrawEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + proceedWithdrawEvent.parameters.push( + new ethereum.EventParam( + "kickoffTxid", + ethereum.Value.fromFixedBytes(kickoffTxid) + ) + ) + + return proceedWithdrawEvent +} + +export function createWithdrawDisprovedEvent( + instanceId: Bytes, + graphId: Bytes, + disproveTxType: i32, + txnIndex: BigInt, + challengeStartTxid: Bytes, + challengeFinishTxid: Bytes, + challengerAddress: Address, + disproverAddress: Address, + challengerRewardAmount: BigInt, + disproverRewardAmount: BigInt +): WithdrawDisproved { + let withdrawDisprovedEvent = changetype(newMockEvent()) + + withdrawDisprovedEvent.parameters = new Array() + + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "disproveTxType", + ethereum.Value.fromUnsignedBigInt(BigInt.fromI32(disproveTxType)) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "txnIndex", + ethereum.Value.fromUnsignedBigInt(txnIndex) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "challengeStartTxid", + ethereum.Value.fromFixedBytes(challengeStartTxid) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "challengeFinishTxid", + ethereum.Value.fromFixedBytes(challengeFinishTxid) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "challengerAddress", + ethereum.Value.fromAddress(challengerAddress) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "disproverAddress", + ethereum.Value.fromAddress(disproverAddress) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "challengerRewardAmount", + ethereum.Value.fromUnsignedBigInt(challengerRewardAmount) + ) + ) + withdrawDisprovedEvent.parameters.push( + new ethereum.EventParam( + "disproverRewardAmount", + ethereum.Value.fromUnsignedBigInt(disproverRewardAmount) + ) + ) + + return withdrawDisprovedEvent +} + +export function createWithdrawHappyPathEvent( + instanceId: Bytes, + graphId: Bytes, + take1Txid: Bytes, + operatorAddress: Address, + rewardAmountSats: BigInt +): WithdrawHappyPath { + let withdrawHappyPathEvent = changetype(newMockEvent()) + + withdrawHappyPathEvent.parameters = new Array() + + withdrawHappyPathEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + withdrawHappyPathEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + withdrawHappyPathEvent.parameters.push( + new ethereum.EventParam( + "take1Txid", + ethereum.Value.fromFixedBytes(take1Txid) + ) + ) + withdrawHappyPathEvent.parameters.push( + new ethereum.EventParam( + "operatorAddress", + ethereum.Value.fromAddress(operatorAddress) + ) + ) + withdrawHappyPathEvent.parameters.push( + new ethereum.EventParam( + "rewardAmountSats", + ethereum.Value.fromUnsignedBigInt(rewardAmountSats) + ) + ) + + return withdrawHappyPathEvent +} + +export function createWithdrawUnhappyPathEvent( + instanceId: Bytes, + graphId: Bytes, + take2Txid: Bytes, + operatorAddress: Address, + rewardAmountSats: BigInt +): WithdrawUnhappyPath { + let withdrawUnhappyPathEvent = changetype(newMockEvent()) + + withdrawUnhappyPathEvent.parameters = new Array() + + withdrawUnhappyPathEvent.parameters.push( + new ethereum.EventParam( + "instanceId", + ethereum.Value.fromFixedBytes(instanceId) + ) + ) + withdrawUnhappyPathEvent.parameters.push( + new ethereum.EventParam("graphId", ethereum.Value.fromFixedBytes(graphId)) + ) + withdrawUnhappyPathEvent.parameters.push( + new ethereum.EventParam( + "take2Txid", + ethereum.Value.fromFixedBytes(take2Txid) + ) + ) + withdrawUnhappyPathEvent.parameters.push( + new ethereum.EventParam( + "operatorAddress", + ethereum.Value.fromAddress(operatorAddress) + ) + ) + withdrawUnhappyPathEvent.parameters.push( + new ethereum.EventParam( + "rewardAmountSats", + ethereum.Value.fromUnsignedBigInt(rewardAmountSats) + ) + ) + + return withdrawUnhappyPathEvent +} diff --git a/localnet/subgraph/gateway/tests/gateway-upgradeable.test.ts b/localnet/subgraph/gateway/tests/gateway-upgradeable.test.ts new file mode 100644 index 000000000..0220fd820 --- /dev/null +++ b/localnet/subgraph/gateway/tests/gateway-upgradeable.test.ts @@ -0,0 +1,74 @@ +import { + assert, + describe, + test, + clearStore, + beforeAll, + afterAll +} from "matchstick-as/assembly/index" +import { Address, Bytes, BigInt } from "@graphprotocol/graph-ts" +import { BridgeIn } from "../generated/schema" +import { BridgeIn as BridgeInEvent } from "../generated/GatewayUpgradeable/GatewayUpgradeable" +import { handleBridgeIn } from "../src/gateway-upgradeable" +import { createBridgeInEvent } from "./gateway-upgradeable-utils" + +// Tests structure (matchstick-as >=0.5.0) +// https://thegraph.com/docs/en/subgraphs/developing/creating/unit-testing-framework/#tests-structure + +describe("Describe entity assertions", () => { + beforeAll(() => { + let depositorAddress = Address.fromString( + "0x0000000000000000000000000000000000000001" + ) + let instanceId = Bytes.fromI32(1234567890) + let peginAmountSats = BigInt.fromI32(234) + let feeAmountSats = BigInt.fromI32(234) + let newBridgeInEvent = createBridgeInEvent( + depositorAddress, + instanceId, + peginAmountSats, + feeAmountSats + ) + handleBridgeIn(newBridgeInEvent) + }) + + afterAll(() => { + clearStore() + }) + + // For more test scenarios, see: + // https://thegraph.com/docs/en/subgraphs/developing/creating/unit-testing-framework/#write-a-unit-test + + test("BridgeIn created and stored", () => { + assert.entityCount("BridgeIn", 1) + + // 0xa16081f360e3847006db660bae1c6d1b2e17ec2a is the default address used in newMockEvent() function + assert.fieldEquals( + "BridgeIn", + "0xa16081f360e3847006db660bae1c6d1b2e17ec2a-1", + "depositorAddress", + "0x0000000000000000000000000000000000000001" + ) + assert.fieldEquals( + "BridgeIn", + "0xa16081f360e3847006db660bae1c6d1b2e17ec2a-1", + "instanceId", + "1234567890" + ) + assert.fieldEquals( + "BridgeIn", + "0xa16081f360e3847006db660bae1c6d1b2e17ec2a-1", + "peginAmountSats", + "234" + ) + assert.fieldEquals( + "BridgeIn", + "0xa16081f360e3847006db660bae1c6d1b2e17ec2a-1", + "feeAmountSats", + "234" + ) + + // More assert options: + // https://thegraph.com/docs/en/subgraphs/developing/creating/unit-testing-framework/#asserts + }) +}) diff --git a/localnet/subgraph/gateway/tsconfig.json b/localnet/subgraph/gateway/tsconfig.json new file mode 100644 index 000000000..4e866720d --- /dev/null +++ b/localnet/subgraph/gateway/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@graphprotocol/graph-ts/types/tsconfig.base.json", + "include": ["src", "tests"] +} diff --git a/localnet/tools/cast.sh b/localnet/tools/cast.sh new file mode 100755 index 000000000..022cb8fe6 --- /dev/null +++ b/localnet/tools/cast.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# cast inside the pinned foundry container (fallback when the host has no cast). Loopback +# RPC URLs work through --network host. Each call starts a container (~0.3 s). +set -euo pipefail +source "$(cd "$(dirname "$0")/.." && pwd)/scripts/localnet.env" +exec docker run --rm --network host -u "$LOCALNET_UID:$LOCALNET_GID" -e HOME=/tmp -e FOUNDRY_DISABLE_NIGHTLY_WARNING=1 \ + --entrypoint cast "$FOUNDRY_IMAGE" "$@" diff --git a/localnet/tools/faultproxy.py b/localnet/tools/faultproxy.py new file mode 100644 index 000000000..a19291b4e --- /dev/null +++ b/localnet/tools/faultproxy.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""Fault-injecting HTTP proxy for the localnet (stdlib only). + +All ports derive from PORT_BASE (env, default 22000, same offsets as scripts/localnet.env). +One listener per (role, backend): port = PORT_BASE+960 + role_idx*4 + backend_idx. + backends: btc -> 127.0.0.1:PORT_BASE+2 (esplora) goat -> +545 (anvil) + graph -> +800 (graph-node) proof -> +950 (mock proof builder) +Control API on 127.0.0.1:PORT_BASE+989 (JSON): + GET /health | /faults | /stats + POST /fault {"role": "all"|, "backend": , "type": , "params": {...}, "ttl": secs} + POST /clear {"role"?: .., "backend"?: .., "type"?: ..} +Fault types (per key, stackable, every fault expires after ttl, default 600 s): + latency {ms, jitter} delay before forwarding each request + down {} listener closed -> ECONNREFUSED + blackhole {secs} accept, read, never answer (close after secs if set) + cut {p} RST the connection mid-response with probability p + http {code, p} answer instead of forwarding, probability p + slow {ms_per_kb} delay per KB of response body + stall {secs} send the response head, then hang + corrupt {p} answer 200 with a non-JSON body + flap {on, off} periodic down/up +Every forwarded request is sent with `Connection: close` (one request per client +connection), so request boundaries are unambiguous. +""" +import argparse +import asyncio +import json +import os +import random +import socket +import struct +import sys +import time + +ROLES = ["committee_0", "committee_1", "operator_0", "verifier_0", "watchtower_0"] +PORT_BASE = int(os.environ.get("PORT_BASE", "22000")) +BACKENDS = [("btc", PORT_BASE + 2), ("goat", PORT_BASE + 545), ("graph", PORT_BASE + 800), ("proof", PORT_BASE + 950)] +BASE_PORT = PORT_BASE + 960 +CTRL_PORT = PORT_BASE + 989 +DEFAULT_TTL = 600 +IDLE_SECS = 300 +READ_LIMIT = 4 << 20 + + +def log(msg): + print(time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), msg, flush=True) + + +def port_of(role, backend): + return BASE_PORT + ROLES.index(role) * 4 + [b for b, _ in BACKENDS].index(backend) + + +class Proxy: + def __init__(self): + self.faults = {} # key -> {type: {"params":..., "expires": t}} + self.servers = {} # key -> asyncio.Server | None + self.flaps = {} # key -> task + self.stats = {} + for r in ROLES: + for b, _ in BACKENDS: + self.stats[(r, b)] = {"requests": 0, "forwarded": 0, "errors": 0, "active": 0, "faulted": {}} + + # ---- fault bookkeeping ------------------------------------------------- + def active(self, key): + now = time.time() + f = self.faults.get(key, {}) + return {t: v["params"] for t, v in f.items() if v["expires"] > now} + + def faulted(self, key, t): + d = self.stats[key]["faulted"] + d[t] = d.get(t, 0) + 1 + + async def set_fault(self, role, backend, ftype, params, ttl): + keys = [(r, backend) for r in (ROLES if role == "all" else [role])] + for key in keys: + self.faults.setdefault(key, {})[ftype] = {"params": params, "expires": time.time() + ttl} + if ftype == "down": + await self.stop_listener(key) + if ftype == "flap": + self.cancel_flap(key) + self.flaps[key] = asyncio.create_task(self.flap_loop(key, params)) + log(f"FAULT set {key[0]}/{key[1]} {ftype} {params} ttl={ttl}") + + async def clear(self, role=None, backend=None, ftype=None): + n = 0 + for key in list(self.faults): + if role and role != "all" and key[0] != role: + continue + if backend and key[1] != backend: + continue + for t in list(self.faults[key]): + if ftype and t != ftype: + continue + await self.drop_fault(key, t) + n += 1 + return n + + async def drop_fault(self, key, t): + self.faults.get(key, {}).pop(t, None) + if t == "flap": + self.cancel_flap(key) + if t in ("down", "flap") and "down" not in self.faults.get(key, {}): + await self.start_listener(key) + log(f"FAULT cleared {key[0]}/{key[1]} {t}") + + def cancel_flap(self, key): + task = self.flaps.pop(key, None) + if task: + task.cancel() + + async def flap_loop(self, key, params): + on, off = float(params.get("on", 20)), float(params.get("off", 10)) + try: + while True: + await asyncio.sleep(on) + await self.stop_listener(key) + log(f"FLAP {key[0]}/{key[1]} down") + await asyncio.sleep(off) + if "down" not in self.faults.get(key, {}): + await self.start_listener(key) + log(f"FLAP {key[0]}/{key[1]} up") + except asyncio.CancelledError: + pass + + async def sweeper(self): + while True: + await asyncio.sleep(1) + now = time.time() + for key in list(self.faults): + for t, v in list(self.faults[key].items()): + if v["expires"] <= now: + await self.drop_fault(key, t) + log(f"FAULT expired {key[0]}/{key[1]} {t}") + + # ---- listeners ----------------------------------------------------------- + async def start_listener(self, key): + if self.servers.get(key): + return + role, backend = key + upstream = dict(BACKENDS)[backend] + srv = await asyncio.start_server( + lambda r, w: self.handle(key, upstream, r, w), "127.0.0.1", port_of(role, backend), limit=READ_LIMIT + ) + self.servers[key] = srv + + async def stop_listener(self, key): + srv = self.servers.pop(key, None) + if srv: + srv.close() + await srv.wait_closed() + + async def start_all(self): + for key in self.stats: + await self.start_listener(key) + + # ---- data path ------------------------------------------------------------- + async def handle(self, key, uport, reader, writer): + st = self.stats[key] + st["active"] += 1 + try: + await self.handle_inner(key, uport, reader, writer, st) + except (ConnectionError, asyncio.IncompleteReadError, asyncio.TimeoutError, asyncio.LimitOverrunError): + st["errors"] += 1 + except Exception as e: # noqa: BLE001 + st["errors"] += 1 + log(f"ERROR {key[0]}/{key[1]}: {e!r}") + finally: + st["active"] -= 1 + try: + writer.close() + except Exception: # noqa: BLE001 + pass + + async def handle_inner(self, key, uport, reader, writer, st): + f = self.active(key) + if "blackhole" in f: + self.faulted(key, "blackhole") + secs = f["blackhole"].get("secs") + try: + await asyncio.wait_for(self.drain(reader), timeout=secs) + except asyncio.TimeoutError: + pass + return + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=IDLE_SECS) + lines = head.decode("latin-1").split("\r\n") + request_line = lines[0] + headers = {} + for ln in lines[1:]: + if ":" in ln: + k, v = ln.split(":", 1) + headers[k.strip().lower()] = v.strip() + if "chunked" in headers.get("transfer-encoding", "").lower(): + body = await self.read_chunked(reader) + else: + body = await reader.readexactly(int(headers.get("content-length", "0") or 0)) + st["requests"] += 1 + f = self.active(key) + if "latency" in f: + ms = float(f["latency"].get("ms", 0)) + jit = float(f["latency"].get("jitter", 0)) + await asyncio.sleep(max(0.0, ms + random.uniform(-jit, jit)) / 1000) + if "http" in f and random.random() < float(f["http"].get("p", 1)): + code = int(f["http"].get("code", 503)) + self.faulted(key, "http") + await self.reply(writer, code, b'{"error":"faultproxy injected"}') + return + if "corrupt" in f and random.random() < float(f["corrupt"].get("p", 1)): + self.faulted(key, "corrupt") + await self.reply(writer, 200, b"{\"result\": <<>>") + return + # forward with Connection: close + out = [request_line] + for ln in lines[1:]: + if ln and not ln.lower().startswith(("connection:", "keep-alive:")): + out.append(ln) + out.append("Connection: close") + new_head = ("\r\n".join(out) + "\r\n\r\n").encode("latin-1") + try: + ur, uw = await asyncio.wait_for(asyncio.open_connection("127.0.0.1", uport), timeout=5) + except (OSError, asyncio.TimeoutError): + st["errors"] += 1 + await self.reply(writer, 502, b'{"error":"faultproxy: upstream unreachable"}') + return + try: + uw.write(new_head + body) + await uw.drain() + resp_head = await ur.readuntil(b"\r\n\r\n") + rl = resp_head.decode("latin-1").split("\r\n") + rl = [ln for ln in rl if ln and not ln.lower().startswith(("connection:", "keep-alive:"))] + rl.append("Connection: close") + writer.write(("\r\n".join(rl) + "\r\n\r\n").encode("latin-1")) + await writer.drain() + if "stall" in f: + self.faulted(key, "stall") + await asyncio.sleep(float(f["stall"].get("secs", 3600))) + return + cut_p = float(f["cut"].get("p", 0)) if "cut" in f else 0.0 + slow = float(f["slow"].get("ms_per_kb", 0)) if "slow" in f else 0.0 + while True: + chunk = await ur.read(65536) + if not chunk: + break + if cut_p and random.random() < cut_p: + self.faulted(key, "cut") + self.abort(writer) + return + if slow: + await asyncio.sleep(slow * len(chunk) / 1024 / 1000) + writer.write(chunk) + await writer.drain() + st["forwarded"] += 1 + finally: + uw.close() + + @staticmethod + async def drain(reader): + while await reader.read(65536): + pass + + @staticmethod + async def read_chunked(reader): + out = b"" + while True: + line = await reader.readline() + out += line + size = int(line.split(b";")[0].strip() or b"0", 16) + if size == 0: + while True: + t = await reader.readline() + out += t + if t in (b"\r\n", b""): + return out + out += await reader.readexactly(size + 2) + + @staticmethod + async def reply(writer, code, body): + reason = {200: "OK", 429: "Too Many Requests", 500: "Internal Server Error", 502: "Bad Gateway", + 503: "Service Unavailable", 504: "Gateway Timeout"}.get(code, "Error") + writer.write((f"HTTP/1.1 {code} {reason}\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + body) + await writer.drain() + + @staticmethod + def abort(writer): + sock = writer.get_extra_info("socket") + if sock is not None: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + writer.close() + + # ---- control API -------------------------------------------------------- + async def control(self, reader, writer): + try: + head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout=10) + lines = head.decode().split("\r\n") + method, path, _ = lines[0].split(" ", 2) + clen = 0 + for ln in lines[1:]: + if ln.lower().startswith("content-length:"): + clen = int(ln.split(":", 1)[1]) + body = await reader.readexactly(clen) if clen else b"" + req = json.loads(body) if body else {} + code, resp = 200, {} + if path == "/health": + resp = {"ok": True, "listeners": len([s for s in self.servers.values() if s])} + elif path == "/faults": + now = time.time() + resp = {f"{k[0]}/{k[1]}": {t: {"params": v["params"], "ttl_left": round(v["expires"] - now)} + for t, v in fs.items() if v["expires"] > now} + for k, fs in self.faults.items()} + resp = {k: v for k, v in resp.items() if v} + elif path == "/stats": + resp = {f"{k[0]}/{k[1]}": v for k, v in self.stats.items()} + elif path == "/fault" and method == "POST": + role, backend, ftype = req.get("role", "all"), req["backend"], req["type"] + if role != "all" and role not in ROLES or backend not in dict(BACKENDS): + code, resp = 400, {"error": "unknown role/backend"} + else: + await self.set_fault(role, backend, ftype, req.get("params", {}), float(req.get("ttl", DEFAULT_TTL))) + resp = {"ok": True} + elif path == "/clear" and method == "POST": + resp = {"cleared": await self.clear(req.get("role"), req.get("backend"), req.get("type"))} + else: + code, resp = 404, {"error": "no such route"} + await self.reply(writer, code, json.dumps(resp).encode()) + except Exception as e: # noqa: BLE001 + try: + await self.reply(writer, 500, json.dumps({"error": repr(e)}).encode()) + except Exception: # noqa: BLE001 + pass + finally: + writer.close() + + +async def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ctrl-port", type=int, default=CTRL_PORT) + args = ap.parse_args() + p = Proxy() + await p.start_all() + ctrl = await asyncio.start_server(p.control, "127.0.0.1", args.ctrl_port) + asyncio.create_task(p.sweeper()) + log(f"faultproxy up: {len(p.servers)} listeners on {BASE_PORT}-{BASE_PORT + len(p.servers) - 1}, control {args.ctrl_port}") + async with ctrl: + await ctrl.serve_forever() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + sys.exit(0) diff --git a/localnet/tools/forge.sh b/localnet/tools/forge.sh new file mode 100755 index 000000000..9f943af36 --- /dev/null +++ b/localnet/tools/forge.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# forge inside the pinned foundry container. The localnet tree is mounted read-only at /work and +# used as the project root (config contracts-local/foundry.toml, sources contracts-local/ + the +# upstream contracts/), every artifact goes under $LOCALNET_DATA/forge (out, cache, broadcast, +# solc cache), so the checkout never changes. --network host lets `forge script --broadcast` +# reach anvil on 127.0.0.1. Usage: tools/forge.sh build --sizes | tools/forge.sh script ... +set -euo pipefail +source "$(cd "$(dirname "$0")/.." && pwd)/scripts/localnet.env" +mkdir -p "$LOCALNET_DATA/forge/home" +env_args=(); for v in PRIVATE_KEY BITCOINSPV_ADDR STAKE_TOKEN_ADDR COMMITTEE_0 COMMITTEE_1 COMMITTEE_2 COMMITTEE_3 WATCHTOWER_0 WATCHTOWER_1; do + [[ -n "${!v:-}" ]] && env_args+=(-e "$v=${!v}"); done +exec docker run --rm --network host -u "$LOCALNET_UID:$LOCALNET_GID" \ + -v "$LOCALNET_ROOT:/work:ro" -v "$LOCALNET_DATA/forge:/forge" -w /work \ + -e HOME=/forge/home -e FOUNDRY_CONFIG=/work/contracts-local/foundry.toml -e FOUNDRY_OUT=/forge/out -e FOUNDRY_CACHE_PATH=/forge/cache -e FOUNDRY_BROADCAST=/forge/broadcast \ + -e FOUNDRY_DISABLE_NIGHTLY_WARNING=1 "${env_args[@]}" \ + --entrypoint forge "$FOUNDRY_IMAGE" "$1" --root /work "${@:2}" diff --git a/localnet/tools/gen-keys.sh b/localnet/tools/gen-keys.sh new file mode 100755 index 000000000..91b4e9ae2 --- /dev/null +++ b/localnet/tools/gen-keys.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# --------------------------------------------------------------------------- +# Generate every identity of the localnet and write +# $LOCALNET_DATA/state/keys.json (machine readable, source of truth) +# $LOCALNET_DATA/state/keys.env (shell-sourceable convenience export) +# $LOCALNET_DATA/state/master-seed.txt (chmod 600) +# +# TEST-ONLY KEYS. Everything here is for an isolated regtest/anvil sandbox. +# +# Deterministic: every secret is sha256("bvlocal-localnet-keys-v1||||"). +# Needs the node binary in $BIN_DIR (scripts/install-bins.sh) and cast. +# ./gen-keys.sh -> reuse state/master-seed.txt if present, else create a new random one +# MASTER_SEED=<64hex> ./gen-keys.sh --force -> recreate an exact earlier key set +# ./gen-keys.sh --force -> overwrite state/keys.json with the same seed +# NEW_SEED=1 ./gen-keys.sh --force -> brand new random seed (all identities change) +# +# Verification (always run, the script exits non-zero on any mismatch): +# * BTC pubkey / funding address vs `bitvm-noded key funding-address` (the node's own code path) +# * BTC pubkey vs `cast wallet public-key` +# * GOAT address vs `cast wallet address` +# * PEER_KEY/PeerId encoding vs a fresh `bitvm-noded key peer` sample re-derived from its seed +# --------------------------------------------------------------------------- +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/localnet.env" +TOOLS=$ROOT/tools +STATE=$STATE_DIR +NODE_BIN=$BIN_DIR/bitvm-noded +[[ -x "$NODE_BIN" ]] || { echo "missing $NODE_BIN (run scripts/install-bins.sh)" >&2; exit 1; } +mkdir -p "$TMP_DIR" +FORCE=0 +[[ "${1:-}" == "--force" ]] && FORCE=1 + +mkdir -p "$STATE" +if [[ -f "$STATE/keys.json" && $FORCE -eq 0 ]]; then + echo "state/keys.json already exists - refusing to overwrite (use --force)" >&2 + exit 1 +fi + +# --- master seed ----------------------------------------------------------- +if [[ -n "${MASTER_SEED:-}" ]]; then + SEED="$MASTER_SEED" +elif [[ -f "$STATE/master-seed.txt" && "${NEW_SEED:-0}" != "1" ]]; then + SEED=$(tr -d '[:space:]' < "$STATE/master-seed.txt") +else + SEED=$(openssl rand -hex 32) +fi +[[ ${#SEED} -eq 64 ]] || { echo "MASTER_SEED must be 64 hex chars" >&2; exit 1; } +printf '%s\n' "$SEED" > "$STATE/master-seed.txt" +chmod 600 "$STATE/master-seed.txt" + +# --- a fresh node-generated peer key, used only to validate our encoder ---- +PEER_SAMPLE=$("$NODE_BIN" key peer) + +MASTER_SEED="$SEED" PEER_SAMPLE="$PEER_SAMPLE" TOOLS="$TOOLS" ANVIL_CHAIN_ID="$ANVIL_CHAIN_ID" python3 - "$STATE" <<'PYEOF' +import base64, datetime, json, os, sys +sys.path.insert(0, os.environ["TOOLS"]) +from keyderive import (derive_secret, evm_address, libp2p_peer_key, p2wsh_funding_address, + peer_id_to_bytes, secp_pubkey, selftest, wif) + +selftest() +state = sys.argv[1] +seed = os.environ["MASTER_SEED"] + +# sanity: our libp2p encoder must reproduce a key the node itself generated +lines = dict(l.split("=", 1) for l in os.environ["PEER_SAMPLE"].strip().splitlines()) +sample_seed = base64.b64decode(lines["PEER_KEY"])[4:36] +pk, pid, _, _ = libp2p_peer_key(sample_seed) +assert pk == lines["PEER_KEY"], "PEER_KEY encoding mismatch vs bitvm-noded key peer" +assert pid == lines["PEER_ID"], "PeerId encoding mismatch vs bitvm-noded key peer" + +# role -> (wants btc key, wants peer key, wants goat key) +ROLES = [ + ("deployer", False, False, True), + ("committee_0", True, True, True), + ("committee_1", True, True, True), + ("operator_0", True, True, True), + ("verifier_0", True, True, True), + ("watchtower_0", True, True, True), + ("user_0", True, False, True), +] + +out = { + "_note": ("TEST-ONLY KEYS for the isolated BitVM2 localnet (bitcoin regtest + local anvil, " + "all endpoints on 127.0.0.1). They control nothing of value " + "and must never be reused on any public network."), + "_generated_by": "localnet/tools/gen-keys.sh (deterministic from master_seed)", + "_generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + "master_seed": seed, + "derivation": { + "scheme": "sha256(\"bvlocal-localnet-keys-v1||||\")", + "purposes": {"bitvm": "BITVM_SECRET (secp256k1)", + "goat": "GOAT_PRIVATE_KEY (secp256k1/EVM)", + "peer": "ed25519 seed for PEER_KEY"}, + }, + "bitcoin_network": "regtest", + "goat_chain_id": int(os.environ["ANVIL_CHAIN_ID"]), + "roles": {}, +} + +for role, want_btc, want_peer, want_goat in ROLES: + entry = {} + if want_btc: + sk = derive_secret(seed, role, "bitvm") + comp, xonly, uncomp, prefix = secp_pubkey(sk) + entry["BITVM_SECRET"] = sk # 64 hex, NO 0x (node/src/env.rs get_bitvm_secret) + entry["btc_pubkey_compressed"] = comp # 33 B, node_pubkey / NodeInfo.btc_pub_key + entry["btc_pubkey_xonly"] = xonly # 32 B, no 0x + entry["btc_pubkey_xonly_0x"] = "0x" + xonly # bytes32 form for the contracts + entry["btc_pubkey_prefix"] = "0x" + prefix # registerPubkey(bytes1 pubkeyPrefix) + entry["btc_pubkey_uncompressed_xy"] = "0x" + uncomp + entry["btc_key_evm_address"] = evm_address(uncomp) # what ecrecover must return in registerPubkey + entry["btc_funding_address_regtest"] = p2wsh_funding_address(comp, "bcrt") + entry["btc_wif_regtest"] = wif(sk) + if want_peer: + pseed = derive_secret(seed, role, "peer") + peer_key, peer_id, peer_bytes, ed_pub = libp2p_peer_key(bytes.fromhex(pseed)) + entry["PEER_KEY"] = peer_key + entry["peer_id"] = peer_id + entry["peer_id_bytes"] = peer_bytes # CommitteeManagement.registerPeerId / isVerifier + entry["peer_ed25519_pubkey"] = "0x" + ed_pub + assert peer_id_to_bytes(peer_id) == peer_bytes + if want_goat: + gsk = derive_secret(seed, role, "goat") + _, _, guncomp, _ = secp_pubkey(gsk) + entry["GOAT_PRIVATE_KEY"] = "0x" + gsk # alloy PrivateKeySigner, 0x optional + entry["goat_address"] = evm_address(guncomp) # GOAT_ADDRESS / EVM address + out["roles"][role] = entry + +with open(os.path.join(state, "keys.json"), "w") as f: + json.dump(out, f, indent=2) + f.write("\n") + +# shell-sourceable form: _ +ABBR = {"deployer": "DEPLOYER", "committee_0": "C0", "committee_1": "C1", "operator_0": "OP0", + "verifier_0": "VF0", "watchtower_0": "WT0", "user_0": "USER0"} +with open(os.path.join(state, "keys.env"), "w") as f: + f.write("# TEST-ONLY keys for the localnet - generated by localnet/tools/gen-keys.sh\n") + f.write("# source this file, then use e.g. $C0_BITVM_SECRET, $OP0_GOAT_ADDRESS\n") + for role, entry in out["roles"].items(): + p = ABBR[role] + f.write(f"\n# ---- {role} ----\n") + for k, v in entry.items(): + f.write(f'export {p}_{k.upper()}="{v}"\n') +print("wrote state/keys.json and state/keys.env") +PYEOF + +chmod 600 "$STATE/keys.json" "$STATE/keys.env" + +# --- verification against the node binary and cast ------------------------- +echo "--- verification ---" +FAIL=0 +verify_role() { + local role=$1 + local sk pub_expect addr_expect comp xonly goat_sk goat_addr + sk=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role'].get('BITVM_SECRET',''))") + if [[ -n "$sk" ]]; then + comp=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role']['btc_pubkey_compressed'])") + addr_expect=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role']['btc_funding_address_regtest'])") + # (1) node's own code path: BITVM_SECRET -> get_node_pubkey -> node_p2wsh_address + got=$(cd "$TMP_DIR" && ACTOR=Verifier BITCOIN_NETWORK=regtest BITVM_SECRET="$sk" \ + "$NODE_BIN" key funding-address | sed 's/.*: //') + if [[ "$got" != "$addr_expect" ]]; then + echo "FAIL $role funding address: node=$got json=$addr_expect"; FAIL=1 + else + echo "ok $role funding-address (bitvm-noded) $got" + fi + # (2) cast: BITVM_SECRET -> secp256k1 pubkey + raw=$("$CAST" wallet public-key --raw-private-key "0x$sk") + x=${raw:2:64}; y=${raw:66:64} + if [[ $((16#${y: -1} % 2)) -eq 0 ]]; then pfx=02; else pfx=03; fi + if [[ "$pfx$x" != "$comp" ]]; then echo "FAIL $role btc pubkey vs cast"; FAIL=1; + else echo "ok $role btc pubkey (cast) $comp"; fi + fi + goat_sk=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role'].get('GOAT_PRIVATE_KEY',''))") + if [[ -n "$goat_sk" ]]; then + goat_addr=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role']['goat_address'])") + got=$("$CAST" wallet address --private-key "$goat_sk") + if [[ "$got" != "$goat_addr" ]]; then echo "FAIL $role goat address: cast=$got json=$goat_addr"; FAIL=1; + else echo "ok $role goat address (cast) $got"; fi + fi +} +for r in deployer committee_0 committee_1 operator_0 verifier_0 watchtower_0 user_0; do verify_role "$r"; done + +# (3) PEER_KEY -> PeerId through the node's own libp2p code path. +# The node is started with a dead GOAT RPC so it prints `local_peer_id` and then +# panics at env.rs `cannot get chain_id` long before it binds any port. +verify_peer_id() { + local role=$1 pk pid got tmpdb + pk=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role'].get('PEER_KEY',''))") + [[ -z "$pk" ]] && return 0 + pid=$(python3 -c "import json;d=json.load(open('$STATE/keys.json'));print(d['roles']['$role']['peer_id'])") + tmpdb=$(mktemp -p "$TMP_DIR" peerverify-XXXXXX.db) + rm -f "$tmpdb" + got=$( (cd "$TMP_DIR" && RUST_LOG=info ACTOR=Committee BITCOIN_NETWORK=regtest \ + BITVM_SECRET=$(python3 -c "print('11'*32)") PEER_KEY="$pk" GOAT_CHAIN_URL=http://127.0.0.1:1 \ + timeout 60 "$NODE_BIN" --rpc-addr 127.0.0.1:$KEYCHECK_PORT --p2p-port $KEYCHECK_PORT \ + --db-path "sqlite:$tmpdb" 2>/dev/null || true) | grep -o '"local_peer_id":"[^"]*"' | head -1 | cut -d'"' -f4) + rm -f "$tmpdb" "$tmpdb"-shm "$tmpdb"-wal + if [[ "$got" != "$pid" ]]; then echo "FAIL $role peer id: node=$got json=$pid"; FAIL=1; + else echo "ok $role peer id (bitvm-noded) $got"; fi +} +if [[ "${SKIP_PEER_ID_VERIFY:-0}" != "1" ]]; then + for r in committee_0 committee_1 operator_0 verifier_0 watchtower_0; do verify_peer_id "$r"; done +fi +if [[ $FAIL -ne 0 ]]; then echo "VERIFICATION FAILED"; exit 1; fi +echo "--- all identities verified ---" diff --git a/localnet/tools/keyderive.py b/localnet/tools/keyderive.py new file mode 100644 index 000000000..d3d48925b --- /dev/null +++ b/localnet/tools/keyderive.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +Pure-python key derivation helpers for the BitVM2 localnet. + +No third-party packages (none are installed on this host). Everything is derived +from first principles so that it can be re-checked independently: + + * secp256k1 scalar multiplication -> BTC compressed / x-only pubkeys, EVM pubkeys + * keccak-256 -> EVM (GOAT) addresses + * bech32 / segwit v0 -> P2WSH funding addresses (bcrt1q...) + * base58 / base58check -> libp2p PeerId strings, WIF + * ed25519 (RFC 8032) -> libp2p PEER_KEY / PeerId + +Every derivation here is cross-checked against the node's own binaries or cast by +tools/gen-keys.sh. +""" +import hashlib + +# --------------------------------------------------------------------------- # +# secp256k1 +# --------------------------------------------------------------------------- # +P = 2**256 - 2**32 - 977 +N = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 +G = ( + 0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798, + 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8, +) + + +def _add(a, b): + if a is None: + return b + if b is None: + return a + if a[0] == b[0] and (a[1] + b[1]) % P == 0: + return None + if a == b: + lam = (3 * a[0] * a[0]) * pow(2 * a[1], P - 2, P) % P + else: + lam = (b[1] - a[1]) * pow(b[0] - a[0], P - 2, P) % P + x = (lam * lam - a[0] - b[0]) % P + return (x, (lam * (a[0] - x) - a[1]) % P) + + +def secp256k1_mul(k, point=G): + r = None + while k: + if k & 1: + r = _add(r, point) + point = _add(point, point) + k >>= 1 + return r + + +def secp_pubkey(sk_hex): + """Return (compressed33_hex, xonly32_hex, uncompressed64_hex, prefix_hex).""" + sk = int(sk_hex, 16) + if not (0 < sk < N): + raise ValueError("secret key out of range") + x, y = secp256k1_mul(sk) + xs, ys = "%064x" % x, "%064x" % y + prefix = "02" if y % 2 == 0 else "03" + return prefix + xs, xs, xs + ys, prefix + + +# --------------------------------------------------------------------------- # +# keccak-256 (EVM addresses) +# --------------------------------------------------------------------------- # +_KECCAK_RC = [ + 0x0000000000000001, 0x0000000000008082, 0x800000000000808A, 0x8000000080008000, + 0x000000000000808B, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, + 0x000000000000008A, 0x0000000000000088, 0x0000000080008009, 0x000000008000000A, + 0x000000008000808B, 0x800000000000008B, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800A, 0x800000008000000A, + 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008, +] +_KECCAK_ROT = [ + [0, 36, 3, 41, 18], [1, 44, 10, 45, 2], [62, 6, 43, 15, 61], + [28, 55, 25, 21, 56], [27, 20, 39, 8, 14], +] + + +def _keccak_f(state): + def rol(v, n): + return ((v << n) | (v >> (64 - n))) & 0xFFFFFFFFFFFFFFFF + + for rnd in range(24): + c = [state[x][0] ^ state[x][1] ^ state[x][2] ^ state[x][3] ^ state[x][4] for x in range(5)] + d = [c[(x - 1) % 5] ^ rol(c[(x + 1) % 5], 1) for x in range(5)] + for x in range(5): + for y in range(5): + state[x][y] ^= d[x] + b = [[0] * 5 for _ in range(5)] + for x in range(5): + for y in range(5): + b[y][(2 * x + 3 * y) % 5] = rol(state[x][y], _KECCAK_ROT[x][y]) + for x in range(5): + for y in range(5): + state[x][y] = b[x][y] ^ ((~b[(x + 1) % 5][y]) & b[(x + 2) % 5][y]) & 0xFFFFFFFFFFFFFFFF + state[0][0] ^= _KECCAK_RC[rnd] + return state + + +def keccak256(data: bytes) -> bytes: + rate = 136 # 1088 bits + state = [[0] * 5 for _ in range(5)] + padded = bytearray(data) + padded.append(0x01) + while len(padded) % rate != 0: + padded.append(0x00) + padded[-1] ^= 0x80 + for off in range(0, len(padded), rate): + block = padded[off:off + rate] + for i in range(rate // 8): + lane = int.from_bytes(block[i * 8:(i + 1) * 8], "little") + state[i % 5][i // 5] ^= lane + _keccak_f(state) + out = bytearray() + for i in range(4): # 32 bytes + out += state[i % 5][i // 5].to_bytes(8, "little") + return bytes(out) + + +def evm_address(uncompressed64_hex: str) -> str: + """EIP-55 checksummed address from the 64-byte (x||y) pubkey.""" + h = keccak256(bytes.fromhex(uncompressed64_hex))[-20:].hex() + cs = keccak256(h.encode()).hex() + return "0x" + "".join(c.upper() if c.isalpha() and int(cs[i], 16) >= 8 else c + for i, c in enumerate(h)) + + +# --------------------------------------------------------------------------- # +# bech32 / segwit v0 (BIP-173) +# --------------------------------------------------------------------------- # +_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" + + +def _bech32_polymod(values): + gen = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] + chk = 1 + for v in values: + b = chk >> 25 + chk = ((chk & 0x1FFFFFF) << 5) ^ v + for i in range(5): + chk ^= gen[i] if ((b >> i) & 1) else 0 + return chk + + +def _bech32_hrp_expand(hrp): + return [ord(c) >> 5 for c in hrp] + [0] + [ord(c) & 31 for c in hrp] + + +def _convertbits(data, frombits, tobits, pad=True): + acc, bits, ret = 0, 0, [] + maxv = (1 << tobits) - 1 + for value in data: + acc = (acc << frombits) | value + bits += frombits + while bits >= tobits: + bits -= tobits + ret.append((acc >> bits) & maxv) + if pad and bits: + ret.append((acc << (tobits - bits)) & maxv) + return ret + + +def segwit_address(hrp, witver, witprog: bytes) -> str: + data = [witver] + _convertbits(list(witprog), 8, 5) + values = _bech32_hrp_expand(hrp) + data + polymod = _bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1 + checksum = [(polymod >> 5 * (5 - i)) & 31 for i in range(6)] + return hrp + "1" + "".join(_CHARSET[d] for d in data + checksum) + + +def p2wsh_funding_address(compressed_pubkey_hex: str, hrp: str = "bcrt") -> str: + """Address of the node's funding output: P2WSH(<33B pubkey> OP_CHECKSIG). + + Script is exactly `OP_PUSHBYTES_33 OP_CHECKSIG` (node/src/utils.rs + node_p2wsh_script/node_p2wsh_address). + """ + script = bytes([0x21]) + bytes.fromhex(compressed_pubkey_hex) + bytes([0xAC]) + return segwit_address(hrp, 0, hashlib.sha256(script).digest()) + + +# --------------------------------------------------------------------------- # +# base58 +# --------------------------------------------------------------------------- # +_B58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + + +def b58encode(b: bytes) -> str: + n = int.from_bytes(b, "big") + out = "" + while n: + n, r = divmod(n, 58) + out = _B58[r] + out + return "1" * (len(b) - len(b.lstrip(b"\x00"))) + out + + +def b58decode(s: str) -> bytes: + n = 0 + for c in s: + n = n * 58 + _B58.index(c) + body = n.to_bytes((n.bit_length() + 7) // 8, "big") if n else b"" + return b"\x00" * (len(s) - len(s.lstrip("1"))) + body + + +def wif(sk_hex: str, network_prefix: int = 0xEF, compressed: bool = True) -> str: + """WIF for the raw secp256k1 secret (0xEF = regtest/testnet).""" + payload = bytes([network_prefix]) + bytes.fromhex(sk_hex) + (b"\x01" if compressed else b"") + chk = hashlib.sha256(hashlib.sha256(payload).digest()).digest()[:4] + return b58encode(payload + chk) + + +# --------------------------------------------------------------------------- # +# ed25519 (RFC 8032) -> libp2p PEER_KEY / PeerId +# --------------------------------------------------------------------------- # +_Q = 2**255 - 19 +_D = -121665 * pow(121666, _Q - 2, _Q) % _Q +_I = pow(2, (_Q - 1) // 4, _Q) + + +def _ed_xrecover(y): + xx = (y * y - 1) * pow(_D * y * y + 1, _Q - 2, _Q) + x = pow(xx, (_Q + 3) // 8, _Q) + if (x * x - xx) % _Q != 0: + x = (x * _I) % _Q + if x % 2 != 0: + x = _Q - x + return x + + +_ED_BY = 4 * pow(5, _Q - 2, _Q) % _Q +_ED_B = (_ed_xrecover(_ED_BY) % _Q, _ED_BY) + + +def _ed_add(p, q): + x1, y1 = p + x2, y2 = q + t = _D * x1 * x2 * y1 * y2 + x3 = (x1 * y2 + x2 * y1) * pow(1 + t, _Q - 2, _Q) + y3 = (y1 * y2 + x1 * x2) * pow(1 - t, _Q - 2, _Q) + return (x3 % _Q, y3 % _Q) + + +def _ed_mul(p, e): + if e == 0: + return (0, 1) + q = _ed_mul(p, e // 2) + q = _ed_add(q, q) + if e & 1: + q = _ed_add(q, p) + return q + + +def ed25519_pubkey(seed32: bytes) -> bytes: + h = bytearray(hashlib.sha512(seed32).digest()[:32]) + h[0] &= 248 + h[31] &= 127 + h[31] |= 64 + a = int.from_bytes(h, "little") + x, y = _ed_mul(_ED_B, a) + enc = bytearray(y.to_bytes(32, "little")) + enc[31] |= (x & 1) << 7 + return bytes(enc) + + +def libp2p_peer_key(seed32: bytes): + """Return (PEER_KEY base64, PEER_ID string, peer-id bytes hex, pubkey hex). + + PEER_KEY = base64(protobuf PrivateKey{Type=Ed25519(1), Data=seed||pub}) + = base64(08 01 12 40 || seed32 || pub32) (as produced by + `bitvm-noded key peer`, libp2p Keypair::to_protobuf_encoding) + PeerId = base58btc( 00 24 08 01 12 20 || pub32 ) (identity multihash + over protobuf PublicKey{Type=Ed25519(1), Data=pub32}) + """ + import base64 as _b64 + pub = ed25519_pubkey(seed32) + proto_priv = bytes([0x08, 0x01, 0x12, 0x40]) + seed32 + pub + peer_key = _b64.b64encode(proto_priv).decode() + peer_id_bytes = bytes([0x00, 0x24, 0x08, 0x01, 0x12, 0x20]) + pub + return peer_key, b58encode(peer_id_bytes), "0x" + peer_id_bytes.hex(), pub.hex() + + +def peer_id_to_bytes(peer_id: str) -> str: + return "0x" + b58decode(peer_id).hex() + + +# --------------------------------------------------------------------------- # +# deterministic role key material +# --------------------------------------------------------------------------- # +DOMAIN = b"bvlocal-localnet-keys-v1" + + +def derive_secret(master_seed_hex: str, role: str, purpose: str) -> str: + """32-byte key material, valid as a secp256k1 secret key.""" + seed = bytes.fromhex(master_seed_hex) + counter = 0 + while True: + material = hashlib.sha256( + DOMAIN + b"|" + role.encode() + b"|" + purpose.encode() + b"|" + + str(counter).encode() + b"|" + seed + ).digest() + k = int.from_bytes(material, "big") + if 0 < k < N: + return material.hex() + counter += 1 + + +def selftest(): + assert keccak256(b"").hex() == \ + "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + assert keccak256(b"abc").hex() == \ + "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45" + c, x, u, pfx = secp_pubkey("%064x" % 1) + assert c == "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798" + assert evm_address(u) == "0x7E5F4552091A69125d5DfCb7b8C2659029395Bdf" + # RFC 8032 test vector 1 + assert ed25519_pubkey(bytes.fromhex( + "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60")).hex() == \ + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a" + # BIP-173 style check of our bech32 encoder via a known node output: + # BITVM_SECRET=00..01 -> bcrt1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qzf4jry + assert p2wsh_funding_address(c) == \ + "bcrt1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qzf4jry" + assert b58decode(b58encode(b"\x00\x24hello")) == b"\x00\x24hello" + return "keyderive selftest OK" + + +if __name__ == "__main__": + print(selftest()) diff --git a/node/Cargo.toml b/node/Cargo.toml index 6d5f17743..2680fae67 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -33,17 +33,24 @@ path = "src/bin/send_rbf.rs" name = "pegout" path = "src/bin/send_pegout.rs" -[[bin]] -name = "update-db" -path = "src/bin/db_inject.rs" - [[bin]] name = "mock-rpc" path = "src/bin/mock_rpc.rs" +[[bin]] +name = "rpc-sign" +path = "src/bin/rpc_sign.rs" + +[[bin]] +name = "mock-proof-builder" +path = "src/bin/mock_proof_builder.rs" +required-features = ["mock-gc"] + [features] default = [] rpc-debug-endpoints = [] +# Localnet-only: replace the real BABE/GC machinery and proof builder with deterministic mocks. +mock-gc = ["bitvm-lib/mock-gc"] [dependencies] libp2p = { workspace = true, features = [ @@ -63,6 +70,7 @@ libp2p = { workspace = true, features = [ "gossipsub", "quic", "websocket", + "request-response", ] } libp2p-swarm-derive = { workspace = true } libp2p-metrics = { workspace = true } @@ -130,6 +138,7 @@ bitvm = { workspace = true } goat = { workspace = true } bitvm-lib = { workspace = true } store = { workspace = true } +engine = { workspace = true } client = { workspace = true } bitcoin-light-client-circuit = { workspace = true } commit-chain = { workspace = true } diff --git a/node/README.md b/node/README.md index 8cff4628b..40ee24ca0 100644 --- a/node/README.md +++ b/node/README.md @@ -29,16 +29,16 @@ The BitVM Node (`bitvm-noded`) is a multi-role distributed node that participate - **Multi-signature Consensus**: Committee-based transaction presigning - **Optimistic Verification**: Watchtower monitoring with dispute resolution - **ZK Proofs**: Cryptographic verification of state transitions -- **P2P Graph Distribution**: Decentralized graph data synchronization via relayer nodes +- **P2P Graph Distribution**: graph definitions are pulled directly from relayers, committee members or the operator (`GetGraph` request-response) ### Key Capabilities | Capability | Description | |------------|-------------| -| **P2P Communication** | libp2p-based gossip network (Kademlia DHT + Gossipsub) | +| **P2P Communication** | libp2p-based gossip network (Kademlia DHT + Gossipsub) plus a direct `GetGraph` protocol | | **Chain Monitoring** | Bitcoin and GOAT L2 event watching | +| **Execution Engine** | One durable inbox, single-writer aggregates (instance, graph), one outbox, leased jobs (`crates/engine`) | | **Graph Management** | Transaction graph creation, signing, and lifecycle tracking | -| **Graph Synchronization** | P2P-based graph distribution with relayer node support | | **Challenge Processing** | Watchtower challenges and dispute resolution | | **Proof Coordination** | ZK proof generation via proof-builder-rpc | @@ -63,14 +63,15 @@ flowchart TB RPC["RPC Service (Axum)"] end - subgraph Core["Core Processing"] - MH["Message Handler"] - DISP["Role Dispatcher"] + subgraph Core["Execution Engine"] + INBOX[("engine_inbox")] + AGG["Aggregates\n(instance, graph)"] + OUTBOX[("engine_outbox")] + JOBS["Jobs\n(CPU pool, network pool)"] end subgraph Tasks["Background Tasks"] - GM["Graph Maintenance"] - IM["Instance Maintenance"] + SCAN["Chain Scans\n(instance, graph)"] NM["Node Maintenance"] SPV["SPV Maintenance"] end @@ -84,22 +85,26 @@ flowchart TB BTC --> EW GOAT --> EW - EW --> MH - P2P --> MH - RPC --> MH - MH --> DISP - DISP --> GM - DISP --> IM - GM --> DB - IM --> DB + BTC --> SCAN + GOAT --> SCAN + EW --> INBOX + P2P --> INBOX + SCAN --> INBOX + RPC --> DB + INBOX --> AGG + AGG --> OUTBOX + AGG --> JOBS + JOBS --> INBOX + OUTBOX --> P2P + AGG --> DB + JOBS --> GD + JOBS --> GM_TBL + JOBS --> PB NM --> DB SPV --> DB - GM --> GD - GM --> GM_TBL - GM --> PB ``` -**Description**: The system employs a three-layer architecture. The Input Layer receives events from external chains, P2P network messages, and RPC requests. The Core Processing Layer dispatches messages to role-specific handlers. The Background Tasks Layer manages graph state machines, instance lifecycles, node health, and SPV header chain updates. All state is persisted to SQLite database, with graph data stored in a dedicated `graph_raw_data` table. +**Description**: Every input (an admitted P2P message, a chain event, a chain scan at a block height, a job result) is recorded once in `engine_inbox` and applied by the single-writer aggregate it belongs to (`instance:` or `graph:`) inside one SQLite transaction together with its effects: outbox rows, jobs and `waiting_for` registrations. The outbox is the only sender; it re-publishes until the aggregate closes the row. Heavy work (circuit setup, graph building, signing, proof verification, chain calls) runs in leased jobs whose results come back as inbox events. See `docs/node-protocol.md` §4–§5 and §10. ### Fig-01-2-Component-Interaction @@ -126,10 +131,10 @@ flowchart LR V <--> TV W <--> TW C & O & V & W & R <--> TA - R -.->|"SyncGraph"| TA + V -.->|"GetGraph (direct)"| R ``` -**Description**: Each role subscribes to its corresponding gossipsub topic for message exchange. All roles subscribe to the `/All` topic for broadcast messages. Relayer nodes (Committee members with `ENABLE_RELAYER=true`) respond to graph synchronization requests and distribute graph data across the network. +**Description**: Each role subscribes to its corresponding gossipsub topic for message exchange. All roles subscribe to the `/All` topic for broadcast messages. Graph definitions are not gossiped on request: a node that lacks one asks a relayer, a committee member or the operator directly over the `GetGraph` request-response protocol and authenticates what it receives. Relayer nodes (Committee members with `ENABLE_RELAYER=true`) post graph data to GOAT and serve as the second pull source. --- @@ -194,11 +199,11 @@ classDiagram | Role | Responsibilities | Key Messages | |------|------------------|--------------| -| **Committee** | Multi-sig committee member, responsible for presigning and graph endorsement | `NonceGeneration`, `CommitteePresign`, `EndorseGraph` | -| **Operator** | Bridge operator, creates graphs and executes withdrawal transactions | `CreateGraph`, `KickoffSent`, `Take1Sent`, `Take2Sent` | -| **Verifier** | Dispute verifier, monitors timeouts and submits disproofs | `DisproveReady`, `DisproveSent` | -| **Watchtower** | Chain monitor, validates block headers and submits challenges | `WatchtowerChallengeSent` | -| **Relayer** | Graph data distributor, responds to sync requests | `SyncGraphRequest`, `SyncGraph` | +| **Committee** | Multi-sig committee member, responsible for presigning, graph endorsement and the Peg-In-Confirm signature | `NonceGeneration`, `CommitteePresign`, `EndorseGraph`, `PeginConfirmPartialSig` | +| **Operator** | Bridge operator, creates graphs and executes withdrawal transactions (kickoff, takes are Bitcoin transactions driven by chain scans) | `InitGraph`, `CreateGraph`, `GraphFinalize` | +| **Verifier** | Dispute verifier, builds and solders circuits, monitors timeouts and submits disproofs | `GenCircuits`, `SolderingProofReady`, `VerifierGraphParamsEndorsement` | +| **Watchtower** | Chain monitor, validates block headers and submits challenges | (chain scans only) | +| **Relayer** | Committee member that posts peg-in and graph data to GOAT and serves `GetGraph` pulls | `GetGraph` (direct protocol) | --- @@ -223,10 +228,12 @@ sequenceDiagram User->>GOAT: Initiate BridgeInRequest GOAT-->>EW: Trigger BridgeInRequest event - EW->>Committee: PeginRequest message + EW->>DB: Store instance, pegin_request event Committee->>Committee: Validate fees and availability - Committee->>Operator: ConfirmInstance + Committee->>GOAT: answerPeginRequest + User->>BTC: Broadcast Peg-In-Prepare + BTC-->>Operator: Prepare confirmed (confirm_instance job) Operator->>Operator: Create SimplifiedbitvmGraph Operator->>DB: Store graph_raw_data (JSON) @@ -257,22 +264,27 @@ stateDiagram-v2 UserInited --> NoEnoughCommitteesAnswered: Window timeout UserInited --> UserDiscarded: UTXO spent - CommitteesAnswered --> UserBroadcastPeginPrepare: User broadcasts - UserBroadcastPeginPrepare --> Presigned: All committees signed - UserBroadcastPeginPrepare --> PresignedFailed: Signing failed + CommitteesAnswered --> UserBroadcastPeginPrepare: Peg-In-Prepare confirmed + UserBroadcastPeginPrepare --> Presigned: Enough graphs finalized + UserBroadcastPeginPrepare --> UserDiscarded: Prepare reorged, UTXO spent - Presigned --> RelayerL1Broadcasted: Relayer broadcasts + Presigned --> RelayerL1Broadcasted: Peg-In-Confirm confirmed + UserBroadcastPeginPrepare --> RelayerL1Broadcasted: Peg-In-Confirm confirmed RelayerL1Broadcasted --> RelayerL2Minted: L2 minting success - RelayerL1Broadcasted --> RelayerL2MintedFailed: L2 minting failed - UserBroadcastPeginPrepare --> Timeout: Timeout - Timeout --> UserCanceled: User cancels + UserBroadcastPeginPrepare --> UserCanceled: Peg-In-Cancel confirmed + Presigned --> UserCanceled: Peg-In-Cancel confirmed + UserBroadcastPeginPrepare --> Timeout: Abandon height, Confirm not visible + Presigned --> Timeout: Abandon height, Confirm not visible + Timeout --> UserCanceled: Peg-In-Cancel confirmed RelayerL2Minted --> [*] UserCanceled --> [*] + UserDiscarded --> [*] + NoEnoughCommitteesAnswered --> [*] ``` -**Description**: This state diagram shows the complete lifecycle of a bridge-in instance, from initial request through committee validation, signing, and final minting on L2. +**Description**: This state diagram shows the complete lifecycle of a bridge-in instance, from initial request through committee validation, signing, and final minting on L2. Every deadline is a Bitcoin height: from `Prepare height + connector_z` (T0) no node signs for the instance and the user may cancel; `PEGIN_ABANDON_BUFFER_BLOCKS` later the instance is abandoned (`Timeout`) unless the Peg-In-Confirm is confirmed or visible in the mempool. `Timeout` is a failure terminal that keeps the committee key for manual handling; the other failure states delete it (see `docs/node-protocol.md` §10.2). ### Peg-out (Bridge-Out) @@ -457,7 +469,7 @@ Source: `scan_graph_chain_state`, `node/src/utils.rs:1328-1681` (the sole writer ### Fig-04-3-Challenge-SubStatus -`ChallengeSubStatus` is **not** three parallel single-value status enums as earlier revisions of this document claimed — that shape does not exist in the code. The actual struct (`node/src/scheduled_tasks/graph_maintenance_tasks.rs:52-58`) is: +`ChallengeSubStatus` is **not** three parallel single-value status enums as earlier revisions of this document claimed — that shape does not exist in the code. The actual struct (`node/src/aggregates/graph_scan.rs`) is: ```rust pub struct ChallengeSubStatus { @@ -472,7 +484,7 @@ pub enum VerifierChallengeStatus { None, VerifierAsserted, ProverAnswered, Dispr pub enum DisproveTxType { Disprove, QuickChallenge, ChallengeIncompleteKickoff, PubinDisprove, OperatorChallengeNack, OperatorCommitTimeout } ``` -`watchtower_challenge_status` and `verifier_challenge_status` are **observational bookkeeping only** — they are recorded for the frontend/API (`node/src/rpc_service/bitvm.rs:672-693`, collapsed there into a simpler `SimpleChallengeSubStatus{None, WatchtowerChallenge, Assert}` view) but are never read by anything that decides `GraphStatus`. In particular, `ChallengeSubStatus::is_watchtower_challenge_success` (`graph_maintenance_tasks.rs:98-105`) has no call sites anywhere in the repo. The real `Challenge --> OperatorTake2` gate is purely `tx_on_chain(take2_txid)` after none of the four disprove detectors fired (`node/src/utils.rs:1661-1670`); the operator's actual authorization to broadcast take2 is enforced by Bitcoin-script timelocks (`detect_take2`, `graph_maintenance_tasks.rs:1089-1233`), not by any application-level quorum over watchtower or verifier counts. +`watchtower_challenge_status` and `verifier_challenge_status` are **observational bookkeeping only** — they are recorded for the frontend/API (`node/src/rpc_service/bitvm.rs:672-693`, collapsed there into a simpler `SimpleChallengeSubStatus{None, WatchtowerChallenge, Assert}` view) but are never read by anything that decides `GraphStatus`. The real `Challenge --> OperatorTake2` gate is purely `tx_on_chain(take2_txid)` after none of the four disprove detectors fired (`node/src/utils.rs:1661-1670`); the operator's actual authorization to broadcast take2 is enforced by Bitcoin-script timelocks (the `take2` job re-checks them), not by any application-level quorum over watchtower or verifier counts. ### Known gap: no ordering guarantee between chain-scan and L2-event writers @@ -488,7 +500,7 @@ Both currently go through a raw `UPDATE graph SET status = ?` (`crates/store/src ## Graph Storage & Synchronization -Graph data is stored locally in SQLite and distributed via P2P messaging. +Graph data is stored locally in SQLite. A node that lacks a definition pulls it over the direct `GetGraph` protocol; nothing is re-gossiped. ### Fig-05-1-Graph-Storage-Architecture @@ -506,10 +518,10 @@ flowchart TB SERIALIZE["serialize_graph_raw_data()"] end - subgraph P2P["P2P Synchronization"] - REQ["SyncGraphRequest"] - RESP["SyncGraph"] - RELAY["Relayer Nodes"] + subgraph Pull["Direct Pull"] + REQ["GetGraphRequest\n(graph_id or operator slot)"] + RESP["GetGraphResponse"] + PEERS["Relayer / Committee / Operator"] end CREATE -->|"Atomic Transaction"| GT @@ -519,44 +531,34 @@ flowchart TB PARSE -->|"spawn_blocking"| GRD SERIALIZE -->|"spawn_blocking"| GRD - REQ -->|"Broadcast to All"| RELAY - RELAY -->|"Lookup Local DB"| GRD - RELAY -->|"Respond with Graph"| RESP + REQ -->|"request-response"| PEERS + PEERS -->|"Lookup Local DB"| GRD + PEERS -->|"Answer"| RESP ``` -**Description**: Graph storage uses a two-table approach: `graph` for metadata/transaction IDs and `graph_raw_data` for full serialized graph JSON. Large graph serialization uses `spawn_blocking` to prevent async runtime blocking. P2P synchronization enables nodes to request missing graphs from relayer nodes. +**Description**: Graph storage uses a two-table approach: `graph` for metadata/transaction IDs and `graph_raw_data` for full serialized graph JSON. Large graph serialization uses `spawn_blocking` to prevent async runtime blocking. The `graph` table is unique per `(operator_pubkey, kickoff_index)`: one definition per operator slot. -### Fig-05-2-Graph-Sync-Sequence +### Fig-05-2-Graph-Pull-Sequence ```mermaid sequenceDiagram autonumber participant NodeA as Node A (Missing Graph) - participant P2P as P2P Network - participant Relayer as Relayer Node - participant DB as Relayer DB - - NodeA->>NodeA: Check local DB for graph - Note over NodeA: Graph NOT FOUND - - NodeA->>P2P: SyncGraphRequest(instance_id, graph_id) - P2P->>Relayer: Broadcast to All topic - - Relayer->>Relayer: is_relayer() check - Note over Relayer: ENABLE_RELAYER=true - - Relayer->>DB: Query graph_raw_data - DB-->>Relayer: Return serialized graph - - Relayer->>P2P: SyncGraph(instance_id, graph_id, graph_data) - P2P->>NodeA: Deliver response - - NodeA->>NodeA: Validate graph_id on GOAT chain - NodeA->>NodeA: store_graph() locally - Note over NodeA: Graph synchronized! + participant Peer as Relayer / Committee / Operator + participant DB as Peer DB + + NodeA->>NodeA: Apply parks on the missing definition (waiting_for) + NodeA->>Peer: GetGraphRequest(graph_id | operator slot) over /get-graph/1 + Peer->>Peer: Admission: class budget, per-peer bucket, cooldown, in-flight slots + Peer->>DB: Query graph + graph_raw_data + DB-->>Peer: Serialized graph + Peer-->>NodeA: GetGraphResponse + + NodeA->>NodeA: Authenticate: operator pre-signatures, instance parameters, committee pre-signatures if finalized + NodeA->>NodeA: store_graph() locally, wake the waiting events ``` -**Description**: When a node lacks required graph data, it broadcasts a `SyncGraphRequest`. Relayer nodes (Committee members with `ENABLE_RELAYER=true`) respond with the full graph data via `SyncGraph` message. The requesting node validates and stores the graph locally. +**Description**: The `fetch_graph` job of the Graph aggregate asks relayers and committee members first and the operator last; a definition claimed to be finalized must carry valid committee pre-signatures, and a graph already posted on GOAT is never accepted as an unsigned copy. Serving is rate limited per requester class and per peer (see `docs/node-protocol.md` §2.3). ### Graph Data Tables @@ -597,23 +599,23 @@ classDiagram class GOATMessageContent { <> - PeginRequest + InitGraph + GenCircuits + CutCircuits + SolderingProofReady CreateGraph - ConfirmInstance + VerifierGraphParamsEndorsement NonceGeneration + AggNonceConsensus CommitteePresign EndorseGraph GraphFinalize - KickoffReady - KickoffSent - ChallengeSent - DisproveReady - DisproveSent - Take1Sent - Take2Sent - SyncGraphRequest - SyncGraph - ... + PeginConfirmNonce + PeginConfirmNonceConsensus + PeginConfirmPartialSig + GraphSetupAck + RequestNodeInfo + ResponseNodeInfo } class Actor { @@ -629,50 +631,42 @@ classDiagram GOATMessage --> GOATMessageContent : payload ``` -**Description**: The message system uses role-based addressing. Each message specifies a target role and content type, broadcast via Gossipsub to the corresponding topic. Messages are serialized using `serde_json` with `spawn_blocking` for large payloads. +**Description**: The message system uses role-based addressing. Each message specifies a target topic and a content type, broadcast via Gossipsub. Messages are JSON (`GenCircuits` is bincode behind a `GOATBIN1` prefix). Only the 17 kinds above exist on the wire; chain observations are engine events, never messages. An admitted durable message becomes an inbox event of its aggregate (content id `H(author ‖ kind ‖ business id ‖ H(payload))`), so a re-published copy is a no-op; heavy messages are acknowledged with `GraphSetupAck` once their material is durably stored. ### Message Categories -#### Peg-in Messages +#### Graph setup and signing (Graph aggregate) -| Message | Sender | Receiver | Description | -|---------|--------|----------|-------------| -| `PeginRequest` | EventWatch | Committee | Peg-in request notification | -| `ConfirmInstance` | Committee | Operator | Instance creation confirmation | -| `CreateGraph` | Operator | Committee | Graph creation request (includes full graph) | -| `NonceGeneration` | Committee | Committee/Operator | MuSig2 nonce broadcast | -| `CommitteePresign` | Committee | Committee/Operator | Presignature broadcast | -| `EndorseGraph` | Committee | Operator | Graph endorsement | -| `GraphFinalize` | Operator | All | Graph completion notification | +| Message | Sender | Receiver | ACK | Description | +|---------|--------|----------|-----|-------------| +| `InitGraph` | Operator | Verifier | no | Recruit verifiers for a new graph | +| `GenCircuits` | Verifier | Operator | yes | Candidate's circuit setup package | +| `CutCircuits` | Operator | Verifier | yes | Which circuits the candidate opens or proves | +| `SolderingProofReady` | Verifier | Operator | yes | Soldering proof descriptor (payload out of band) | +| `CreateGraph` | Operator | Verifier, Committee | yes | The operator-presigned graph definition | +| `VerifierGraphParamsEndorsement` | Verifier | Committee | no | Verifier's endorsement of the graph parameters | +| `NonceGeneration` | Committee | Committee, Operator | no | MuSig2 public nonces | +| `AggNonceConsensus` | Committee | Committee | no | Signature over the agreed nonce set | +| `CommitteePresign` | Committee | Committee, Operator | no | Committee partial signatures | +| `EndorseGraph` | Committee | Operator | no | EVM endorsements of the graph | +| `GraphFinalize` | Operator | All | yes | The finalized graph with every endorsement | -#### Peg-out Messages +#### Peg-In-Confirm round (Instance aggregate, committee only) | Message | Sender | Receiver | Description | |---------|--------|----------|-------------| -| `KickoffReady` | System | Operator | Kickoff ready notification | -| `KickoffSent` | Operator | All | Kickoff transaction broadcast | -| `Take1Ready` | System | Operator | Take1 ready (Happy Path) | -| `Take1Sent` | Operator | All | Take1 transaction broadcast | -| `Take2Ready` | System | Operator | Take2 ready (Challenge Path) | -| `Take2Sent` | Operator | All | Take2 transaction broadcast | +| `PeginConfirmNonce` | Committee | Committee | Deterministic nonce for the Peg-In-Confirm | +| `PeginConfirmNonceConsensus` | Committee | Committee | Signature over the agreed nonce set | +| `PeginConfirmPartialSig` | Committee | Committee | Partial signature plus the EVM endorsement the relayer posts | -#### Challenge Messages +#### Control messages (not persisted) | Message | Sender | Receiver | Description | |---------|--------|----------|-------------| -| `WatchtowerChallengeInitSent` | Operator | Watchtower | WT challenge initialization | -| `WatchtowerChallengeSent` | Watchtower | Operator | WT challenge submission | -| `WatchtowerChallengeTimeout` | System | Operator | WT challenge timeout | -| `OperatorAckTimeout` | System | Verifier | Operator ACK timeout | -| `DisproveReady` | System | Verifier | Disprove ready | -| `DisproveSent` | Verifier | All | Disprove transaction broadcast | +| `GraphSetupAck` | Receiver of a heavy message | Sender | Delivery acknowledged (`delivery_id`) | +| `RequestNodeInfo` / `ResponseNodeInfo` | Any | All | Peer discovery and the operator key binding | -#### Synchronization Messages - -| Message | Sender | Receiver | Description | -|---------|--------|----------|-------------| -| `SyncGraphRequest` | Any Node | All | Request missing graph data | -| `SyncGraph` | Relayer | All | Respond with full graph data | +Peg-out has no messages: the Graph aggregate scans both chains every Bitcoin block and turns what it sees (kickoff, challenge, asserts, takes) into jobs for the roles concerned. --- @@ -685,30 +679,23 @@ flowchart TB subgraph EventWatch["Event Watch Task (Continuous)"] EW1["fetch_and_handle_gateway_events"] EW2["fetch_and_handle_bridge_out_events"] - EW1 --> MSG1["PeginRequest"] - EW2 --> MSG2["InitWithdraw detection"] + EW1 --> EV1["BridgeInRequest record, l2_minted, graph_posted, goat_status events"] + EW2 --> EV2["withdraw_init / withdraw_cancel events"] end - subgraph GraphMaint["Graph Maintenance (20s interval)"] - GM1["detect_init_withdraw_call"] - GM2["detect_kickoff"] - GM3["detect_take1_or_challenge"] - GM4["process_graph_challenge"] - - GM1 -->|"KickoffReady"| GM2 - GM2 -->|"Status update"| GM3 - GM3 -->|"Challenge"| GM4 - GM3 -->|"Take1Ready"| T1["Take1 processing"] - GM4 -->|"Take2Ready"| T2["Take2 processing"] - GM4 -->|"DisproveReady"| DP["Disprove processing"] + subgraph Maint["Maintenance (10s interval)"] + M1["pegin_request_monitor"] + M2["instance_answer_window_scan"] + M3["instance_btc_scan"] + M4["graph_chain_scan"] + M5["graph_setup_abandon_scan (operator)"] + M6["swap_escrow_timeout_monitor"] end - subgraph InstMaint["Instance Maintenance (20s interval)"] - IM1["instance_answers_monitor"] - IM2["instance_window_expiration_monitor"] - IM3["instance_btc_tx_monitor"] - IM4["swap_escrow_timeout_monitor"] - IM5["instance_committee_key_cleanup_monitor"] + subgraph Engine["Execution Engine"] + INBOX["engine_inbox"] + APPLY["Instance / Graph apply"] + JOBS["jobs"] end subgraph Other["Other Tasks"] @@ -716,27 +703,29 @@ flowchart TB SPV["SPV Header Updates"] end - EventWatch --> GraphMaint - GraphMaint --> InstMaint + EventWatch --> INBOX + Maint --> INBOX + INBOX --> APPLY --> JOBS ``` -**Description**: The system runs 5 types of background tasks. Event Watch continuously monitors chain events; Graph Maintenance executes graph state machine transitions every 20 seconds; Instance Maintenance manages instance lifecycles; Node Maintenance maintains node health; SPV Maintenance updates the Bitcoin header chain. +**Description**: The event watcher indexes GOAT events; the maintenance tick gathers chain facts (it is the only place that queries the chains for the aggregates) and records them as inbox events; the engine applies them and runs the resulting jobs. No task decides a status by itself. ### Task Responsibilities | Task | Interval | Responsibility | |------|----------|----------------| -| `run_watch_event_task` | Continuous | Monitor GOAT L2 and Bitcoin events | -| `detect_init_withdraw_call` | 20s | Detect withdrawal requests, send KickoffReady | -| `detect_kickoff` | 20s | Monitor Kickoff transaction confirmations | -| `detect_take1_or_challenge` | 20s | Check Happy Path or Challenge | -| `process_graph_challenge` | 20s | Process Challenge sub-phases | -| `instance_answers_monitor` | 20s | Track committee responses | -| `instance_window_expiration_monitor` | 20s | Handle response window timeouts | -| `instance_btc_tx_monitor` | 20s | Track pegin/confirm/cancel BTC transaction confirmations | -| `swap_escrow_timeout_monitor` | 20s | Mark swap escrows past their refund deadline as timed out | -| `instance_committee_key_cleanup_monitor` | 20s | Scan `cache/committee-instance-keys/` and delete expired key envelopes after configurable pegin-confirm timelock | +| `run_watch_event_task` | Continuous | Index GOAT gateway and swap events; record `l2_minted`, `graph_posted`, `goat_status`, `withdraw_init` / `withdraw_cancel` events | +| `pegin_request_monitor` | 10s | Turn pending `BridgeInRequest` records into instance rows and `pegin_request` events (committee nodes answer from the event); late requests with spent inputs are recorded `UserDiscarded` | +| `instance_answer_window_scan` | 10s | Record the committee answers of each `UserInited` instance once its GOAT response window closed | +| `instance_btc_scan` | 10s | One `btc_snapshot` per live instance and Bitcoin height: Prepare / Confirm / Cancel status and input spends | +| `graph_chain_scan` | 10s | One `chain_scan` per non-closed graph and Bitcoin height: chain stage, unusable facts, peg-out signals | +| `graph_setup_abandon_scan` | 10s (operator) | Give up the operator's unfinished setups once the instance passed its abandon height | +| `swap_escrow_timeout_monitor` | 10s | Mark swap escrows past their refund deadline as timed out | +| `node_available_pbtc_update_monitor` | 10s (operator, relayer) | Refresh the node's available pegBTC | | `spv_header_hash_update` | Periodic | Update SPV header hashes | +| `babe_setup_state_cleanup_monitor` | 10s (optional) | Remove orphaned BABE setup state files | + +Admission maintenance (registry refresh, drop summaries, node table purge) runs on its own 20-second timer next to the swarm. --- @@ -744,34 +733,51 @@ flowchart TB ``` node/src/ -├── main.rs # Entry point, starts 4 async tasks +├── main.rs # Entry point, starts the background tasks and the engine ├── lib.rs # Module exports -├── action.rs # Message type definitions (45+ types) -├── handle.rs # Message dispatch and role handlers -├── p2p_msg_handler.rs # P2P message processing interface -├── env.rs # Environment configuration (40+ variables) +├── action.rs # Wire message types, admission entry point, publish +├── handle.rs # Control messages (NodeInfo, GraphSetupAck) +├── p2p_admission.rs # Inbound admission: registry, replay window, rate limits, quota +├── p2p_msg_handler.rs # Swarm handler: admission, control worker, GetGraph service +├── engine_bridge.rs # Engine assembly, gossip transport, ACK forwarder +├── aggregates/ # Aggregates on the execution engine +│ ├── instance.rs # Instance apply: deadlines, F, Answer/Presign/Confirm/Post phases +│ ├── instance_jobs.rs # answerPeginRequest, confirm_instance, PeginConfirm round, post +│ ├── instance_scan.rs # Bitcoin / answer-window facts for instances +│ ├── graph.rs # Graph apply: setup, collect, signing, deliver, post +│ ├── graph_jobs.rs # Setup, signing and finalize jobs +│ ├── graph_setup.rs # Setup helpers +│ ├── graph_fetch.rs # GetGraph pull and authentication +│ ├── graph_pegout.rs # Kickoff / Settlement apply, Skip premise +│ ├── graph_pegout_jobs.rs # Peg-out jobs +│ └── graph_scan.rs # Chain scan of every non-closed graph +├── env.rs # Environment configuration ├── utils.rs # Graph state management & storage utilities ├── error.rs # Error types -├── vk.rs # Verification key management -├── metrics_service.rs # Prometheus metrics -├── rpc_service/ # REST API implementation -│ ├── mod.rs # Service orchestration -│ ├── bitvm.rs # BitVM-specific endpoints -│ ├── routes.rs # HTTP route definitions -│ ├── validation.rs # Input validation -│ └── handler/ # Request handlers -├── middleware/ # P2P network layer -│ ├── swarm.rs # BitvmNetworkManager, libp2p configuration -│ └── behaviour.rs # Kademlia + Gossipsub behavior -└── scheduled_tasks/ # Background tasks - ├── mod.rs # Task exports - ├── event_watch_task.rs # Chain event monitoring (51KB) - ├── graph_maintenance_tasks.rs # Graph state machine (72KB) - ├── instance_maintenance_tasks.rs # Instance lifecycle - ├── node_maintenance_tasks.rs # Node health - └── spv_maintenance_tasks.rs # SPV updates +├── vk.rs # Verification key management +├── metrics_service.rs # Prometheus metrics +├── rpc_service/ # REST API implementation +│ ├── mod.rs # Service orchestration +│ ├── bitvm.rs # BitVM-specific endpoints +│ ├── routes.rs # HTTP route definitions +│ ├── validation.rs # Input validation +│ └── handler/ # Request handlers +├── middleware/ # P2P network layer +│ ├── swarm.rs # BitvmNetworkManager, libp2p configuration +│ ├── behaviour.rs # Kademlia + Gossipsub + GetGraph behaviour +│ ├── get_graph.rs # GetGraph request-response codec +│ ├── publisher.rs # Bounded access to the swarm from other tasks +│ └── connection_gate.rs # Inbound connection limits +└── scheduled_tasks/ # Background tasks + ├── mod.rs # Maintenance tick + ├── event_watch_task.rs # Chain event monitoring + ├── instance_maintenance_tasks.rs # Peg-in request monitor, swap timeouts + ├── node_maintenance_tasks.rs # Node health + └── spv_maintenance_tasks.rs # SPV updates ``` +The execution engine itself lives in `crates/engine`; its tables (`engine_inbox`, `engine_aggregate`, `engine_outbox`, `engine_job`) in `crates/store`. + --- ## Build & Run @@ -794,6 +800,26 @@ BITCOIN_NETWORK=testnet4 cargo build -r cargo build -r -p bitvm-noded ``` +### Mock build for local testnets (`mock-gc`) + +`cargo build -r -p bitvm-noded --features "rpc-debug-endpoints mock-gc" --bins` replaces the +garbled-circuit, soldering and proof machinery with deterministic mocks and additionally builds +`mock-proof-builder` (a stand-in for the proof builder RPC) and `rpc-sign` (prints the +`x-auth-*` headers of a signed request for shell scripts). Everything on chain stays real: +epk commitments, hash locks, the soldered wire deltas and the peg-out transactions are the +genuine ones, so the challenge → assert → disprove / take lifecycle runs unchanged on regtest; +only the proof verdicts are policy-driven. Never deploy a `mock-gc` build. + +Knobs (read at call time, seconds, fractional allowed): `MOCK_GC_BUILDER_INIT_DELAY_SECS`, +`MOCK_GC_SETUP_DELAY_SECS`, `MOCK_GC_SOLDER_DELAY_SECS`, `MOCK_GC_VERIFY_DELAY_SECS`, +`MOCK_GC_CHALLENGE_DELAY_SECS`, `MOCK_GC_EVAL_DELAY_SECS` (the last two are capped at 60 s), +`MOCK_GC_DELAY_SECS` as the fallback; `MOCK_GC_PROOF_VERDICT=valid|always-invalid` decides +the verifier's verdict on an operator assert. The mock proof builder takes +`MOCK_PROOF_DELAY_SECS` / `MOCK_PROOF_OPERATOR_DELAY_SECS` / `MOCK_PROOF_WATCHTOWER_DELAY_SECS` +("not ready" window), `MOCK_PROOF_RESPONSE_LATENCY_MS`, and serves the operator proof fixture +`node/testdata/test-operator-proof.bin`, so `OPERATOR_VK_HASH` / `OPERATOR_ZKM_VERSION` on the +nodes must equal the values the mock builder reports (its defaults), not the testnet ones. + ### Generate Node Keys ```bash @@ -875,11 +901,22 @@ real graph raw data in the database. | `ENABLE_RELAYER` | No | Enable relayer mode for Committee nodes | `false` | | `ENABLE_BABE_SETUP_STATE_CLEANUP` | No | Enable scheduled BABE setup state cleanup for Operator/Verifier nodes | `false` | | `BTC_CHAIN_URL` | No | Bitcoin Esplora API endpoint | Public Esplora | -| `GOAT_PROOF_BUILD_URL` | No | Proof Builder RPC endpoint | - | | `NODE_NAME` | No | Node display name | `ZKM` | | `OPERATOR_NODE_SERVICE_FEE` | No | Operator service fee rate | `0.001` | -| `ENABLE_COMMITTEE_INSTANCE_KEY_DELETE` | No | Enable scheduled deletion of committee instance key envelopes | `false` | +| `ENABLE_COMMITTEE_INSTANCE_KEY_DELETE` | No | Delete the committee instance key envelope when the instance fails (except `Timeout`) or 32 blocks after the Peg-In-Confirm confirmed; off, the intent is only logged | `false` | | `COMMITTEE_INSTANCE_KEY_DELETE_TIMELOCK_BLOCKS` | No | Number of BTC blocks to wait after pegin-confirm confirmation before deleting key envelope | `32` | +| `PEGIN_ABANDON_BUFFER_BLOCKS` | No | Blocks after `Prepare height + connector_z` before an instance whose Peg-In-Confirm is neither confirmed nor in the mempool is abandoned | `18` / `14` / `3` / `6` by network | +| `OPERATOR_GRAPH_START_CUTOFF_BLOCKS` | No | Operators start no new graph for an instance past `Prepare height + cutoff` | `⌊2·connector_z/3⌋` | +| `ENGINE_WORKER_COUNT` / `ENGINE_CPU_POOL_SLOTS` / `ENGINE_PEGOUT_POOL_SLOTS` / `ENGINE_NETWORK_POOL_SLOTS` | No | Execution engine: aggregate workers, concurrent CPU-bound setup jobs, concurrent CPU-bound peg-out jobs (own slots, so a graph build never delays a challenge), concurrent network jobs | `1` / `1` / `1` / `4` | +| `ENGINE_APPLY_TIMEOUT_SECS` / `ENGINE_RECHECK_INTERVAL_SECS` | No | Execution engine: apply budget (also the SQLite busy timeout) and the re-check of waiting events | `120` / `60` | +| `P2P_MAX_JSON_MESSAGE_BYTES` | No | Largest JSON gossip message accepted | `2 MiB` | +| `P2P_INBOX_MAX_QUEUED_BYTES` | No | Payload bytes of admitted network events the engine inbox may hold (the committee reserve and per-peer quotas derive from it; unregistered authors are relayed, never stored) | `2 GiB` | +| `P2P_MAX_INCOMING_CONNECTIONS` / `P2P_INBOUND_REGISTERED_RESERVE` | No | Inbound connection ceiling and the slots reserved for registered peers | `512` / `128` | +| `P2P_RESERVED_PEERS` / `P2P_TRUSTED_OPERATOR_BINDINGS` | No | Peers that always get a connection slot; configured operator key bindings | empty | +| `P2P_LISTEN_IP` | No | Interface the p2p listener binds to; `127.0.0.1` keeps a node on a shared host loopback-only | `0.0.0.0` | +| `BTC_REQUEST_TIMEOUT_SECS` / `GOAT_RPC_TIMEOUT_SECS` | No | Per-request timeout of the Esplora and GOAT RPC clients, seconds (clamped to 1–120) | `15` / `20` | +| `HTTP_REQUEST_TIMEOUT_SECS` | No | Whole-request timeout of the proof-builder HTTP client, seconds (clamped to 1–600); the connect timeout is 10 s | `60` | +| `MAINTENANCE_RUN_TIMEOUT_SECS` | No | Budget of one maintenance tick (all scans in sequence); a tick over budget is logged `maintenance_tick_result outcome=timed_out` and the next tick covers the same chain state | `60` | ### Relayer Configuration @@ -909,28 +946,36 @@ Prometheus metrics are not served by the business RPC listener. Set | Endpoint | Method | Description | |----------|--------|-------------| -| `/node` | GET | Get current node information | -| `/nodes` | GET | List connected nodes | -| `/nodes/overview` | GET | Node statistics overview | -| `/instances` | GET | List all instances | -| `/instance/:id` | GET | Get instance details | -| `/instances/overview` | GET | Instance statistics overview | -| `/v1/graphs` | GET | List all graphs | -| `/v1/graphs/:id` | GET | Get graph details | +| `/v1/health` | GET | Readiness and health of this node (a local database ping, nothing else is called): readiness bits, each backend's availability with its last error, event watcher state and lag, SPV lag, uptime; `200` when ready, `503` otherwise | +| `/v1/nodes`, `/v1/nodes/:id`, `/v1/nodes/overview` | GET | Known nodes, one node, node statistics | +| `/v1/instances`, `/v1/instances/:id`, `/v1/instances/overview`, `/v1/instances/settings` | GET | Peg-in instances, one instance, instance statistics, peg-in settings | +| `/v1/instances/:id/unsigned-pegin-txn` | GET | Unsigned Peg-In-Prepare transaction of an instance | +| `/v1/graphs`, `/v1/graphs/:id` | GET | List all graphs, get graph details | | `/v1/graphs/:id/txn?cursor=0` | GET | Get graph transaction list | | `/v1/graphs/:id/tx?tx_name=cur-pre-kickoff.hex` | GET | Get specific transaction hex | +| `/v1/graphs/:id/neighbor-ids` | GET | Neighbouring graphs of the same instance | | `/v1/graphs/ready-to-kickoff` | GET | Get graphs ready for kickoff | -| `/bridge_in_request` | POST | Initiate Bridge-In request | -| `/v1/swaps` | GET | List swap bridge-out escrows | -| `/v1/swaps/:escrow_hash` | GET | Get swap escrow details | -| `/challenge` | POST | Submit challenge | +| `/v1/graphs/pegout` | POST | Initiate a peg-out (signed request) | +| `/v1/swaps`, `/v1/swaps/:escrow_hash` | GET | Swap bridge-out escrows, one escrow | +| `/v1/proofs/chain_proofs_desc`, `/v1/proofs/operator_proofs_desc` | GET | Proof descriptors, forwarded to `GOAT_PROOF_BUILD_URL` | | `/metrics` | GET | Prometheus metrics (dedicated metrics listener only) | +| `/v1/debug/status`, `/v1/debug/engine` | GET | Node status and the execution engine snapshot (rows by state, waiting and poisoned aggregates); `rpc-debug-endpoints` feature only | +| `/v1/debug/engine/clear-poison`, `/v1/graphs/:id/send-challenge`, `/v1/graphs/:id/send-verifier-challenge` | POST | Release poisoned actions, send a challenge / verifier challenge (signed requests); `rpc-debug-endpoints` feature only | + +### Health and observability + +- **Readiness.** `bitvm_node_ready` (and `ready` in `/v1/health`) is the AND of four bits: start-up done, database reachable (pinged by every `/v1/health` call and every metrics scrape), backends reachable (the height probe of every 6th maintenance tick AND none of the Esplora, GOAT and subgraph availability states below is `unavailable`; the proof builder is informational only, it is called too rarely to drive readiness), event watcher healthy. A node that is up but not ready still serves the RPC and the metrics listener. +- **Backend availability.** The Esplora (`btc`), GOAT RPC (`goat`), subgraph (`graph`) and proof-builder (`proof`) backends each carry one availability state: two consecutive failed calls log a single `event=backend_availability outcome=unavailable` WARN and set `bitvm_node_backend_up{backend}=0` plus `bitvm_node_backend_unavailable_since_seconds{backend}`; once no call has failed for 30 s the next success logs one `outcome=recovered` INFO with `downtime_secs` and `failed_calls`. While a backend is unavailable its individual probe failures are logged at debug. Per-call counters stay in `bitvm_node_btc_backend_requests` / `bitvm_node_goat_backend_requests`. +- **Start-up.** The GOAT reads needed to boot (`eth_chainId`, the gateway's relay contracts) are retried every 5 s for a minute (`event=goat_boot_retry`) before the node gives up; a restart during a short GOAT outage therefore waits instead of exiting. The `event=service_started` line prints the backend URLs, the client timeouts and the maintenance interval the process runs with. +- **Peers.** When the last connection to a bootnode closes the node re-dials it after 5 s and then every 20 s (`event=p2p_bootnode_disconnected` / `p2p_bootnode_redial` / `p2p_bootnode_connected`); `bitvm_node_p2p_connected_peers` and `bitvm_node_p2p_required_topics_healthy` show the result. +- **Maintenance tick.** When the GOAT read that decides whether the gateway history sync is still running fails, the tick is deferred with `maintenance_tick_result outcome=deferred reason=history_sync_check_failed` (WARN) instead of failing; individual scans that fail log `maintenance_subtask_result outcome=failed` and the rest of the tick still runs. +- **Jobs.** An engine job attempt that fails on an error path logs `event=engine_jobs outcome=attempt_error` (WARN) with the error and the attempt number before the backoff (10, 30, 60, 120, 300, then 600 s); an explicit wait (proof not ready, SPV behind, tx not yet confirmed) logs the reason at INFO. The row's `last_error` keeps the latest reason either way. +- **Tooling.** `scripts/ops/` holds read-only helpers for one node: `triage.sh` (one-screen health from `/v1/health`, `/metrics`, the debug routes and the log), `logq.sh` (JSON log queries) and `trace.sh` (timeline of one instance or graph across RPC, database and log). --- ## Related Documentation -- [Architecture Document](../docs/ARCHITECTURE.md) - Full system architecture - [Circuits README](../circuits/README.md) - ZK proof generation - [Deployment Guide](../deployment/README.md) - Deployment instructions - [Incentives](../docs/incentives.md) - Incentives diff --git a/node/src/action.rs b/node/src/action.rs deleted file mode 100644 index 53f86753a..000000000 --- a/node/src/action.rs +++ /dev/null @@ -1,4802 +0,0 @@ -#![allow(clippy::collapsible_match)] -#![allow(clippy::single_match)] -#![allow(clippy::collapsible_else_if)] - -use crate::env::{ - MESSAGE_EXPIRE_TIME, get_p2p_graph_setup_retry_interval_secs, - get_p2p_graph_setup_retry_window_secs, get_p2p_inbox_batch_size, -}; -use crate::handle::{ - HandlerContext, HeavyTaskContext, dispatch as handle_dispatch, heavy_task_from_content, - run_heavy_task, -}; -use crate::metrics_service::MetricsState; -use crate::middleware::AllBehaviours; -use crate::middleware::publisher::MessagePublisher; -use crate::rpc_service::current_time_secs; -use crate::utils::*; -use alloy::primitives::Address as EvmAddress; -use anyhow::{Context, Result, anyhow, bail}; -use bitcoin::{PublicKey, Txid, XOnlyPublicKey}; -use bitvm_lib::actors::Actor; -use bitvm_lib::babe_adapter::{BabeBundleBuilder, CACSetupPackage}; -use bitvm_lib::committee::*; -use bitvm_lib::types::{BitvmGcGraph, SimplifiedBitvmGcGraph}; -use client::goat_chain::DisproveTxType; -use client::http_client::async_client::HttpAsyncClient; -use client::{ - btc_chain::{BTCClient, BtcRpcTimeoutError}, - goat_chain::GOATClient, -}; -use futures::FutureExt; -use libp2p::gossipsub::MessageId; -use libp2p::{PeerId, Swarm, gossipsub}; -use musig2::{PartialSignature, PubNonce}; -use node_macros::MessageBusinessRef; -use secp256k1::{ - Keypair, Message as SecpMessage, SECP256K1, schnorr::Signature as SchnorrSignature, -}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::fmt; -use std::str::FromStr; -use std::sync::{Arc, LazyLock, Mutex}; -use std::time::{Duration, Instant}; -use store::localdb::LocalDB; -use store::{Graph, GraphStatus, InstanceBridgeInStatus, MessageState, P2pInboxMessage}; -use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; -use tokio_util::sync::CancellationToken; -use uuid::Uuid; - -#[derive(Serialize, Deserialize, Clone)] -pub struct GOATMessage { - pub actor: Actor, - pub content: GOATMessageContent, -} - -const GOAT_MESSAGE_BIN_PREFIX: &[u8] = b"GOATBIN1"; -const TRANSIENT_PEGIN_RETRY_DELAY_SECS: usize = 30; -const P2P_INBOX_LEASE_SECS: i64 = 5 * 60; -const P2P_INBOX_LEASE_RENEW_INTERVAL_SECS: u64 = 60; -const P2P_INBOX_ENQUEUE_ATTEMPTS: usize = 3; - -/// Budget for claims that never reported an outcome. Small on purpose: reaching -/// this means the node went down mid-dispatch more than once on the same -/// payload, which is the signature of a message that reproducibly kills it. -const QUEUE_MAX_ABANDONS: i64 = 3; -/// How long a claimed local message stays claimed before another tick may take -/// it over. Heavy work is routed through the durable P2P inbox instead, so local -/// handlers are expected to be short. -const LOCAL_MESSAGE_LEASE_SECS: i64 = 10 * 60; -/// Backoff applied to a local message whose handler returned a non-transient error. -const LOCAL_MESSAGE_RETRY_DELAY_SECS: i64 = 600; -const LOCAL_MESSAGE_BATCH_SIZE: i64 = 50; -/// Delay applied per recorded abandon before a payload may run again, so a -/// supervisor restart after a panic or an unclean exit does not replay it at -/// full speed. -const QUEUE_ABANDON_BACKOFF_SECS: i64 = 60; -/// Maximum retained Quarantined payload bytes. -const P2P_INBOX_QUARANTINE_MAX_BYTES: i64 = 256 * 1024 * 1024; -/// Ceiling on terminal (`Processed`/`Failed`/`Quarantined`) inbox rows kept for -/// diagnostics; the oldest beyond it are deleted. -const P2P_INBOX_TERMINAL_MAX_ROWS: i64 = 100_000; -/// Maximum terminal rows removed per cleanup pass. -const P2P_INBOX_TERMINAL_PURGE_BATCH: i64 = 4_096; -/// Minimum listed-batch shares for Registered and Unregistered senders. -const P2P_INBOX_UNREGISTERED_BATCH_DIVISOR: i64 = 4; -const P2P_INBOX_REGISTERED_BATCH_DIVISOR: i64 = 4; -/// A `node` row of an unregistered peer may be evicted to make room for a new -/// one once it has gone this long without a refresh: three missed heartbeats. -pub const NODE_TABLE_EVICTABLE_AFTER_SECS: i64 = 3 * crate::env::HEARTBEAT_INTERVAL_SECOND as i64; -/// Maximum unregistered node rows. -pub const NODE_TABLE_MAX_UNREGISTERED_ROWS: i64 = 4_096; -/// A `node` row of a peer that is not registered is deleted once it has not been -/// refreshed for this long; live nodes refresh theirs on every heartbeat. -const NODE_TABLE_STALE_SECS: i64 = 7 * 24 * 60 * 60; -const NODE_TABLE_PURGE_INTERVAL: Duration = Duration::from_secs(60 * 60); -/// How long each durable queue may start work in a business-worker tick. -/// Individual handlers have a separate timeout; swarm runs on another thread. -const P2P_INBOX_DRAIN_BUDGET: Duration = Duration::from_secs(4); -/// Initial signing-round retry delay; doubles after successful publishes up to the configured ceiling. -const PROTOCOL_RETRY_BASE_SECS: i64 = 30; -/// Maximum accepted binding timestamp lead. -pub const NODE_INFO_BINDING_MAX_FUTURE_SECS: i64 = 10 * 60; -/// A dispatch future erased behind a box to keep the enclosing task's state -/// machine reasonably small. -type BoxedDispatch<'a> = std::pin::Pin> + 'a>>; - -enum DispatchExecution { - Completed(T), - Shutdown, - Panicked(String), -} - -struct LocalMessageClaim { - message_id: String, - message_version: i64, -} - -/// Identify the current inbox message by kind, business reference and qualifier. -#[derive(Clone, Debug, Eq, PartialEq)] -struct DispatchFingerprint { - kind: MessageKind, - business_ref: BusinessRef, - qualifier: MessageQualifier, -} - -impl DispatchFingerprint { - fn of(content: &GOATMessageContent) -> Self { - Self { - kind: content.kind(), - business_ref: content.business_ref(), - qualifier: content.qualifier(), - } - } -} - -struct InboxDispatchContext { - fingerprint: DispatchFingerprint, - retry: std::cell::RefCell>, -} - -tokio::task_local! { - static ACTIVE_LOCAL_MESSAGE_CLAIM: LocalMessageClaim; - static ACTIVE_INBOX_DISPATCH: InboxDispatchContext; -} - -async fn track_inbox_retry( - fingerprint: DispatchFingerprint, - dispatch: impl std::future::Future>, -) -> Result<()> { - ACTIVE_INBOX_DISPATCH - .scope(InboxDispatchContext { fingerprint, retry: Default::default() }, async { - let result = dispatch.await; - // Apply self-deferral after handler compensation, including when compensation returns an error. - match ACTIVE_INBOX_DISPATCH.with(|active| active.retry.borrow_mut().take()) { - Some(retry) => { - if let Err(error) = &result { - tracing::warn!( - event = "p2p_inbox", - outcome = "deferred_despite_error", - error = %error, - "handler failed after asking to be retried; keeping the retry" - ); - } - Err(retry) - } - None => result, - } - }) - .await -} - -fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String { - if let Some(message) = payload.downcast_ref::<&'static str>() { - (*message).to_owned() - } else if let Some(message) = payload.downcast_ref::() { - message.clone() - } else { - "non-string panic payload".to_owned() - } -} - -/// Catch only at the worker-supervisor boundary. A panic is returned separately -/// so the caller can record an abandon and stop the node; it is never converted -/// into an ordinary handler error or followed by more business work. -async fn supervise_dispatch(future: F, shutdown: &CancellationToken) -> DispatchExecution -where - F: std::future::Future, -{ - match std::panic::AssertUnwindSafe(async { - tokio::select! { - biased; - _ = shutdown.cancelled() => None, - result = future => Some(result), - } - }) - .catch_unwind() - .await - { - Ok(Some(result)) => DispatchExecution::Completed(result), - Ok(None) => DispatchExecution::Shutdown, - Err(payload) => DispatchExecution::Panicked(panic_payload_message(payload.as_ref())), - } -} - -/// Log a per-message bookkeeping failure without aborting the rest of the batch. -/// -/// Propagating here used to abandon every message still claimed in the batch. -/// Those rows stay `Processing` until their lease lapses and are then charged an -/// abandon they never earned — so one transient storage blip could push a whole -/// batch of healthy messages toward quarantine. -fn log_queue_bookkeeping_failure( - queue: &'static str, - message_id: &str, - operation: &str, - error: &anyhow::Error, -) { - tracing::error!( - event = queue, - outcome = "bookkeeping_failed", - message_id, - operation, - error = %error, - "failed to persist a message outcome; leaving it claimed for its lease to lapse" - ); -} - -/// Delivery semantics for externally received P2P messages. -/// -/// Protocol-state messages remain durable. Ephemeral messages carry -/// recoverable peer state or graph synchronization data and can be resent. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum P2PMessageDelivery { - Inbox, - Immediate, -} - -struct ActiveHeavyTask { - message_id: String, - lease_token: String, -} - -static ACTIVE_HEAVY_TASK: LazyLock>> = - LazyLock::new(|| Mutex::new(None)); - -struct HeavyTaskPermit { - message_id: String, - lease_token: String, -} - -/// Ensure a panicking background task cannot leave its detached lease renewer -/// running forever. Once renewal stops, the durable row becomes claimable and -/// the unfinished execution is counted as an abandon. -struct LeaseRenewalGuard(CancellationToken); - -impl Drop for LeaseRenewalGuard { - fn drop(&mut self) { - self.0.cancel(); - } -} - -impl Drop for HeavyTaskPermit { - fn drop(&mut self) { - if let Ok(mut active) = ACTIVE_HEAVY_TASK.lock() - && active.as_ref().is_some_and(|active| { - active.message_id == self.message_id && active.lease_token == self.lease_token - }) - { - *active = None; - } - } -} - -fn active_heavy_task_message_ids() -> Vec { - ACTIVE_HEAVY_TASK - .lock() - .ok() - .and_then(|active| active.as_ref().map(|active| vec![active.message_id.clone()])) - .unwrap_or_default() -} - -fn try_acquire_heavy_task_permit(message_id: &str, lease_token: &str) -> Option { - let mut active = ACTIVE_HEAVY_TASK.lock().ok()?; - if active.is_some() { - return None; - } - *active = Some(ActiveHeavyTask { - message_id: message_id.to_owned(), - lease_token: lease_token.to_owned(), - }); - Some(HeavyTaskPermit { message_id: message_id.to_owned(), lease_token: lease_token.to_owned() }) -} - -/// Stable retry categories shared by P2P inbox consumers and protocol -/// handlers. A handler must opt into retrying; all other failures are treated -/// as terminal and retained as `Failed` inbox records. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum RetryableDispatchReason { - StorageBusy, - ExternalRpcUnavailable, - PayloadNotReady, - DependencyPending, - PublishFailed, - ResourceLocked, -} - -impl RetryableDispatchReason { - pub const fn code(self) -> &'static str { - match self { - Self::StorageBusy => "storage_busy", - Self::ExternalRpcUnavailable => "external_rpc_unavailable", - Self::PayloadNotReady => "payload_not_ready", - Self::DependencyPending => "dependency_pending", - Self::PublishFailed => "publish_failed", - Self::ResourceLocked => "resource_locked", - } - } -} - -#[derive(Debug)] -pub struct RetryableDispatchError { - pub reason: RetryableDispatchReason, - pub retry_after_secs: Option, - detail: String, -} - -impl fmt::Display for RetryableDispatchError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "retryable {}: {}", self.reason.code(), self.detail) - } -} - -impl std::error::Error for RetryableDispatchError {} - -pub fn retryable_dispatch_error( - reason: RetryableDispatchReason, - retry_after_secs: Option, - detail: impl fmt::Display, -) -> anyhow::Error { - anyhow::Error::new(RetryableDispatchError { - reason, - retry_after_secs, - detail: detail.to_string(), - }) -} - -fn is_retryable_http_status(status: u16) -> bool { - status == 429 || (500..=599).contains(&status) -} - -fn is_retryable_reqwest_error(error: &reqwest::Error) -> bool { - error.is_timeout() - || error.is_connect() - || error.status().is_some_and(|status| is_retryable_http_status(status.as_u16())) -} - -/// Only classify transport-level RPC failures. Contract reverts, malformed -/// responses and application errors are intentionally left terminal. -pub(crate) fn is_retryable_external_rpc_error(error: &anyhow::Error) -> bool { - error.chain().any(|cause| { - // Inspect contract transport errors before traversing their transparent sources. - if let Some(alloy::contract::Error::TransportError(error)) = - cause.downcast_ref::() - { - return error.is_transport_error(); - } - if let Some(error) = cause.downcast_ref::() { - return is_retryable_reqwest_error(error); - } - if cause.downcast_ref::().is_some() { - return true; - } - if let Some(error) = cause.downcast_ref::() { - return match error { - // esplora-client uses reqwest 0.11 while this crate uses - // reqwest 0.12, so this check must remain inline instead of - // sharing the 0.12 helper above. - esplora_client::Error::Reqwest(error) => { - error.is_timeout() - || error.is_connect() - || error - .status() - .is_some_and(|status| is_retryable_http_status(status.as_u16())) - } - esplora_client::Error::HttpResponse { status, .. } => { - is_retryable_http_status(*status) - } - _ => false, - }; - } - if let Some(error) = cause.downcast_ref::() { - // GOAT uses an HTTP Alloy provider. A transport error means the - // request did not receive a valid RPC result; RPC ErrorResp is - // deliberately excluded by this predicate. - return error.is_transport_error(); - } - false - }) -} - -fn classify_retryable_dispatch_error(error: anyhow::Error) -> anyhow::Error { - if error.chain().any(|cause| cause.downcast_ref::().is_some()) - || !is_retryable_external_rpc_error(&error) - { - return error; - } - retryable_dispatch_error(RetryableDispatchReason::ExternalRpcUnavailable, None, error) -} - -#[derive(Clone, Copy, Debug)] -pub enum MessageDeferReason { - TransientStorageRetry, - RecoveryRepublish, - PreviousGraphPending, - CommitteeNoncesPending, - CommitteeNonceConsensusPending, - CommitteeEndorsementsPending, - BitcoinTransactionPending, - BitcoinConfirmationPending, - GoatSpvPending, - ProofPending, - ProtocolInputsPending, - TimelockPending, - WithdrawKickoffPending, - ChainStatePending, - ValidationRetry, - GraphSyncPending, - HandlerError, -} - -impl MessageDeferReason { - pub const fn code(self) -> &'static str { - match self { - Self::TransientStorageRetry => "transient_storage_retry", - Self::RecoveryRepublish => "recovery_republish", - Self::PreviousGraphPending => "previous_graph_pending", - Self::CommitteeNoncesPending => "committee_nonces_pending", - Self::CommitteeNonceConsensusPending => "committee_nonce_consensus_pending", - Self::CommitteeEndorsementsPending => "committee_endorsements_pending", - Self::BitcoinTransactionPending => "bitcoin_transaction_pending", - Self::BitcoinConfirmationPending => "bitcoin_confirmation_pending", - Self::GoatSpvPending => "goat_spv_pending", - Self::ProofPending => "proof_pending", - Self::ProtocolInputsPending => "protocol_inputs_pending", - Self::TimelockPending => "timelock_pending", - Self::WithdrawKickoffPending => "withdraw_kickoff_pending", - Self::ChainStatePending => "chain_state_pending", - Self::ValidationRetry => "validation_retry", - Self::GraphSyncPending => "graph_sync_pending", - Self::HandlerError => "handler_error", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum BusinessRef { - Instance { instance_id: Uuid }, - Graph { instance_id: Uuid, graph_id: Uuid }, - Unscoped, -} - -impl BusinessRef { - pub const fn primary_id(self) -> Option { - match self { - Self::Instance { instance_id } => Some(instance_id), - Self::Graph { graph_id, .. } => Some(graph_id), - Self::Unscoped => None, - } - } - - fn key_part(self) -> String { - match self { - Self::Instance { instance_id } => format!("instance:{instance_id}"), - Self::Graph { graph_id, .. } => format!("graph:{graph_id}"), - Self::Unscoped => "unscoped".to_owned(), - } - } -} - -pub trait HasBusinessRef { - fn business_ref(&self) -> BusinessRef; -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum MessageQualifier { - Singleton, - Watchtower(usize), - Verifier(usize), - Disprove { kind: DisproveTxType, index: usize }, -} - -impl MessageQualifier { - fn key_part(&self) -> String { - match self { - Self::Singleton => "singleton".to_owned(), - Self::Watchtower(index) => format!("watchtower:{index}"), - Self::Verifier(index) => format!("verifier:{index}"), - Self::Disprove { kind, index } => format!("disprove:{kind}:{index}"), - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct LocalMessageKey { - actor: Actor, - kind: MessageKind, - business_ref: BusinessRef, - qualifier: MessageQualifier, -} - -impl LocalMessageKey { - pub fn from_content(actor: Actor, content: &GOATMessageContent) -> Result { - let business_ref = content.business_ref(); - if business_ref.primary_id().is_none() { - bail!("cannot persist unscoped {} as a local message", content.event_type()); - } - Ok(Self { actor, kind: content.kind(), business_ref, qualifier: content.qualifier() }) - } - - pub fn business_id(&self) -> Uuid { - match self.business_ref { - BusinessRef::Instance { instance_id } => instance_id, - BusinessRef::Graph { graph_id, .. } => graph_id, - BusinessRef::Unscoped => unreachable!("unscoped messages cannot have a local key"), - } - } - - pub fn message_id(&self) -> String { - format!( - "local:v1:{}:{}:{}:{}", - self.actor, - self.business_ref.key_part(), - self.kind, - self.qualifier.key_part(), - ) - } -} - -#[derive(Serialize, Deserialize, Clone, EnumDiscriminants, MessageBusinessRef)] -#[strum_discriminants(name(MessageKind))] -#[strum_discriminants(derive(Hash, Display, EnumString, EnumIter, IntoStaticStr))] -pub enum GOATMessageContent { - #[business_ref(instance)] - PeginRequest(PeginRequest), - #[business_ref(graph)] - CreateGraph(CreateGraph), - #[business_ref(instance)] - ConfirmInstance(ConfirmInstance), - #[business_ref(graph)] - InitGraph(InitGraph), - #[business_ref(graph)] - GenCircuits(GenCircuits), - #[business_ref(graph)] - CutCircuits(CutCircuits), - #[business_ref(graph)] - SolderingProofReady(SolderingProofReady), - #[business_ref(unscoped)] - GraphSetupAck(GraphSetupAck), - #[business_ref(graph)] - VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement), - #[business_ref(graph)] - NonceGeneration(NonceGeneration), - #[business_ref(graph)] - AggNonceConsensus(AggNonceConsensus), - #[business_ref(graph)] - CommitteePresign(CommitteePresign), - #[business_ref(graph)] - EndorseGraph(EndorseGraph), - #[business_ref(graph)] - GraphFinalize(GraphFinalize), - #[business_ref(instance)] - PeginConfirmNonce(PeginConfirmNonce), - #[business_ref(instance)] - PeginConfirmNonceConsensus(PeginConfirmNonceConsensus), - #[business_ref(instance)] - PeginConfirmPartialSig(PeginConfirmPartialSig), - #[business_ref(instance)] - PostReady(PostReady), - #[business_ref(graph)] - KickoffReady(KickoffReady), - #[business_ref(graph)] - KickoffSent(KickoffSent), - #[business_ref(graph)] - PreKickoffSent(PreKickoffSent), - #[business_ref(graph)] - ChallengeSent(ChallengeSent), - #[business_ref(graph)] - WatchtowerChallengeInitSent(WatchtowerChallengeInitSent), - #[business_ref(graph)] - WatchtowerChallengeSent(WatchtowerChallengeSent), - #[business_ref(graph)] - WatchtowerChallengeTimeout(WatchtowerChallengeTimeout), - #[business_ref(graph)] - NackReady(NackReady), - #[business_ref(graph)] - OperatorCommitPubinReady(OperatorCommitPubinReady), - #[business_ref(graph)] - OperatorCommitPubinTimeout(OperatorCommitPubinTimeout), - #[business_ref(graph)] - AssertReady(AssertReady), - #[business_ref(graph)] - AssertSent(AssertSent), - #[business_ref(graph)] - ChallengeAssertSent(ChallengeAssertSent), - #[business_ref(graph)] - WronglyChallengeTimeout(WronglyChallengeTimeout), - #[business_ref(graph)] - DisproveSent(DisproveSent), - #[business_ref(graph)] - Take1Ready(Take1Ready), - #[business_ref(graph)] - Take1Sent(Take1Sent), - #[business_ref(graph)] - Take2Ready(Take2Ready), - #[business_ref(graph)] - Take2Sent(Take2Sent), - #[business_ref(unscoped)] - RequestNodeInfo(NodeInfo), - #[business_ref(unscoped)] - ResponseNodeInfo(NodeInfo), - #[business_ref(graph)] - SyncGraphRequest(SyncGraphRequest), - #[business_ref(graph)] - SyncGraph(SyncGraph), - #[business_ref(unscoped)] - InstanceDiscarded(InstanceDiscarded), - #[business_ref(unscoped)] - Tick, -} - -impl MessageKind { - const fn is_pegin(self) -> bool { - matches!( - self, - Self::PeginRequest - | Self::ConfirmInstance - | Self::CreateGraph - | Self::InitGraph - | Self::GenCircuits - | Self::CutCircuits - | Self::SolderingProofReady - | Self::VerifierGraphParamsEndorsement - | Self::NonceGeneration - | Self::AggNonceConsensus - | Self::CommitteePresign - | Self::EndorseGraph - | Self::GraphFinalize - | Self::PeginConfirmNonce - | Self::PeginConfirmNonceConsensus - | Self::PeginConfirmPartialSig - | Self::PostReady - ) - } -} - -impl MessageKind { - /// Receiver roles handled by `dispatch` and `heavy_task_from_content`. - pub fn handled_by(self, actor: &Actor) -> bool { - use Actor::{Committee, Operator, Verifier, Watchtower}; - match self { - // Every role has an arm (a role-specific one or the default). - Self::PeginRequest - | Self::ConfirmInstance - | Self::GraphSetupAck - | Self::GraphFinalize - | Self::KickoffSent - | Self::PreKickoffSent - | Self::ChallengeSent - | Self::DisproveSent - | Self::Take1Sent - | Self::Take2Sent - | Self::RequestNodeInfo - | Self::ResponseNodeInfo - | Self::SyncGraph - // Not dispatched through the role table; keep them as they are. - | Self::InstanceDiscarded - | Self::Tick => true, - Self::InitGraph | Self::CutCircuits => matches!(actor, Verifier), - Self::CreateGraph => matches!(actor, Verifier | Committee), - Self::GenCircuits | Self::SolderingProofReady | Self::EndorseGraph => { - matches!(actor, Operator) - } - Self::NonceGeneration | Self::CommitteePresign => matches!(actor, Committee | Operator), - Self::VerifierGraphParamsEndorsement - | Self::AggNonceConsensus - | Self::PeginConfirmNonce - | Self::PeginConfirmNonceConsensus - | Self::PeginConfirmPartialSig - | Self::PostReady - | Self::SyncGraphRequest => matches!(actor, Committee), - Self::KickoffReady - | Self::WatchtowerChallengeSent - | Self::WatchtowerChallengeTimeout - | Self::OperatorCommitPubinReady - | Self::AssertReady - | Self::ChallengeAssertSent - | Self::Take1Ready - | Self::Take2Ready => matches!(actor, Operator), - Self::WatchtowerChallengeInitSent => matches!(actor, Watchtower), - Self::NackReady - | Self::OperatorCommitPubinTimeout - | Self::AssertSent - | Self::WronglyChallengeTimeout => matches!(actor, Verifier), - } - } -} - -impl MessageKind { - /// Sender roles verified against chain registration; other message kinds use `Any`. - pub fn sender_role(self) -> crate::p2p_admission::SenderRole { - use crate::p2p_admission::SenderRole; - match self { - Self::NonceGeneration - | Self::AggNonceConsensus - | Self::CommitteePresign - | Self::EndorseGraph - | Self::PeginConfirmNonce - | Self::PeginConfirmNonceConsensus - | Self::PeginConfirmPartialSig - | Self::SyncGraph => SenderRole::Committee, - Self::GenCircuits | Self::VerifierGraphParamsEndorsement => SenderRole::Verifier, - Self::InitGraph => SenderRole::Operator, - _ => SenderRole::Any, - } - } -} - -impl GOATMessageContent { - pub fn kind(&self) -> MessageKind { - self.into() - } - - /// New messages default to the durable inbox and must explicitly opt into - /// immediate processing when they are safe to drop. - pub const fn p2p_delivery(&self) -> P2PMessageDelivery { - match self { - Self::RequestNodeInfo(_) | Self::ResponseNodeInfo(_) | Self::GraphSetupAck(_) => { - P2PMessageDelivery::Immediate - } - _ => P2PMessageDelivery::Inbox, - } - } - - /// Stable message name for logs/metrics. Keep this independent of `Debug`, whose - /// output can include protocol payloads (and, for proofs, be very large). - pub fn event_type(&self) -> &'static str { - self.kind().into() - } - - pub fn qualifier(&self) -> MessageQualifier { - match self { - Self::WatchtowerChallengeSent(message) => { - MessageQualifier::Watchtower(message.watchtower_index) - } - Self::ChallengeAssertSent(message) => { - MessageQualifier::Verifier(message.verifier_index) - } - Self::WronglyChallengeTimeout(message) => { - MessageQualifier::Verifier(message.verifier_index) - } - Self::DisproveSent(message) => { - MessageQualifier::Disprove { kind: message.disprove_type, index: message.index } - } - _ => MessageQualifier::Singleton, - } - } -} - -fn is_retryable_sqlite_error(error: &anyhow::Error) -> bool { - error.chain().any(|cause| { - let message = cause.to_string().to_ascii_lowercase(); - message.contains("database is locked") - || message.contains("database is busy") - || message.contains("sqlite_busy") - }) -} - -/// Pegin - -#[derive(Serialize, Deserialize, Clone)] -pub struct PeginRequest { - pub instance_id: Uuid, - pub pegin_request_tx_hash: String, // goat tx hash - pub pegin_request_height: i64, - pub pegin_timestamp: i64, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct ConfirmInstance { - pub instance_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct InitGraph { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub operator_pubkey: PublicKey, - pub operator_peer_id: Vec, - pub signature: SchnorrSignature, -} - -const INIT_GRAPH_SIGNATURE_DOMAIN: &[u8] = b"bitvm2-node/init-graph/v1"; - -fn init_graph_signature_message( - instance_id: Uuid, - graph_id: Uuid, - operator_pubkey: &PublicKey, - operator_peer_id: &[u8], -) -> SecpMessage { - let mut hasher = Sha256::new(); - hasher.update(INIT_GRAPH_SIGNATURE_DOMAIN); - hasher.update(instance_id.as_bytes()); - hasher.update(graph_id.as_bytes()); - hasher.update(operator_pubkey.to_bytes()); - hasher.update((operator_peer_id.len() as u32).to_be_bytes()); - hasher.update(operator_peer_id); - SecpMessage::from_digest(hasher.finalize().into()) -} - -pub fn sign_init_graph( - instance_id: Uuid, - graph_id: Uuid, - operator_keypair: &Keypair, - operator_peer_id: Vec, -) -> InitGraph { - let operator_pubkey = operator_keypair.public_key().into(); - let signature = SECP256K1.sign_schnorr( - &init_graph_signature_message(instance_id, graph_id, &operator_pubkey, &operator_peer_id), - operator_keypair, - ); - InitGraph { instance_id, graph_id, operator_pubkey, operator_peer_id, signature } -} - -pub fn verify_init_graph_signature(message: &InitGraph) -> bool { - SECP256K1 - .verify_schnorr( - &message.signature, - &init_graph_signature_message( - message.instance_id, - message.graph_id, - &message.operator_pubkey, - &message.operator_peer_id, - ), - &XOnlyPublicKey::from(message.operator_pubkey), - ) - .is_ok() -} -#[derive(Serialize, Deserialize, Clone)] -pub struct GenCircuits { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub verifier_pubkey: PublicKey, - pub setup_package: CACSetupPackage, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct CutCircuits { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub verifier_pubkey: PublicKey, - pub candidate_index: usize, - pub selected_circuit_indexes: Vec, -} -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct SolderingProofReady { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub candidate_index: usize, - pub payload_hash: [u8; 32], - pub total_len: usize, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct GraphSetupAck { - pub delivery_id: String, -} - -fn graph_setup_payload_hash(payload: &[u8]) -> [u8; 32] { - Sha256::digest(payload).into() -} - -fn graph_setup_delivery_id( - sender_peer_id: &str, - recipient_peer_id: &str, - payload_hash: &[u8; 32], -) -> String { - let mut hasher = Sha256::new(); - hasher.update(b"goat/graph-setup-delivery/v2"); - for field in [sender_peer_id.as_bytes(), recipient_peer_id.as_bytes(), payload_hash.as_slice()] - { - hasher.update((field.len() as u64).to_be_bytes()); - hasher.update(field); - } - format!("graph-setup-v2:{}", hex::encode(hasher.finalize())) -} - -fn graph_setup_ack_required(content: &GOATMessageContent) -> bool { - matches!( - content, - GOATMessageContent::GenCircuits(_) - | GOATMessageContent::CutCircuits(_) - | GOATMessageContent::SolderingProofReady(_) - | GOATMessageContent::CreateGraph(_) - ) -} - -pub fn graph_setup_outbox_id( - content: &GOATMessageContent, - ack_peer_id: Option<&str>, -) -> Option { - match content { - GOATMessageContent::InitGraph(message) => Some(format!("init-graph:{}", message.graph_id)), - GOATMessageContent::GenCircuits(message) => Some(format!( - "gen-circuits:{}:{}:{}", - message.instance_id, message.graph_id, message.verifier_pubkey - )), - GOATMessageContent::CutCircuits(message) => Some(format!( - "cut-circuits:{}:{}:{}", - message.instance_id, message.graph_id, message.verifier_pubkey - )), - GOATMessageContent::SolderingProofReady(message) => Some(format!( - "soldering-proof-ready:{}:{}:{}", - message.graph_id, - message.candidate_index, - hex::encode(message.payload_hash), - )), - GOATMessageContent::CreateGraph(message) => { - let id = format!("create-graph:{}", message.graph_id); - Some(match ack_peer_id.filter(|peer_id| !peer_id.is_empty()) { - Some(peer_id) => format!("{id}:{peer_id}"), - None => id, - }) - } - _ => None, - } -} - -fn graph_setup_delivery_finished(content: &GOATMessageContent, graph: Option<&Graph>) -> bool { - match content { - GOATMessageContent::CreateGraph(_) => graph.is_some_and(|graph| { - GraphStatus::from_str(&graph.status) - .is_ok_and(|status| status != GraphStatus::OperatorPresigned) - }), - _ => graph.is_some(), - } -} - -fn graph_setup_ack( - content: &GOATMessageContent, - sender_peer_id: &PeerId, - payload: &[u8], -) -> Option { - if !graph_setup_ack_required(content) { - return None; - } - // Bind to the authenticated gossip author, never an identity from the payload. - let sender_peer_id = sender_peer_id.to_string(); - let recipient_peer_id = crate::env::get_peer_id(); - let payload_hash = graph_setup_payload_hash(payload); - Some(GraphSetupAck { - delivery_id: graph_setup_delivery_id(&sender_peer_id, &recipient_peer_id, &payload_hash), - }) -} - -pub async fn enqueue_graph_setup_outbox_message( - local_db: &LocalDB, - message: GOATMessage, - ack_peer_id: Option<&str>, -) -> Result { - let sender_peer_id = crate::env::get_peer_id(); - enqueue_graph_setup_outbox_message_from_peer(local_db, message, ack_peer_id, &sender_peer_id) - .await -} - -pub(crate) async fn enqueue_graph_setup_outbox_message_from_peer( - local_db: &LocalDB, - message: GOATMessage, - ack_peer_id: Option<&str>, - sender: &str, -) -> Result { - let serialized = message.serialize_message().await?; - let recipient = if graph_setup_ack_required(&message.content) { - let ack_peer_id = ack_peer_id.context("graph setup ACK requires a recipient peer")?; - Some( - PeerId::from_str(ack_peer_id) - .context("invalid graph setup ACK recipient peer id")? - .to_string(), - ) - } else { - None - }; - let outbox_id = graph_setup_outbox_id(&message.content, recipient.as_deref()) - .ok_or_else(|| anyhow!("not a graph setup message"))?; - let delivery_id = recipient.as_ref().map(|recipient| { - graph_setup_delivery_id(sender, recipient, &graph_setup_payload_hash(&serialized)) - }); - let now = current_time_secs(); - local_db - .acquire() - .await? - .enqueue_p2p_outbox_retry_message( - &outbox_id, - delivery_id.as_deref(), - message.content.event_type(), - &serialized, - now + get_p2p_graph_setup_retry_window_secs(), - get_p2p_graph_setup_retry_interval_secs(), - recipient.as_deref(), - // Only the outbox ever sends these, so the row is due at once. - 0, - ) - .await?; - Ok(outbox_id) -} -#[derive(Serialize, Deserialize, Clone)] -pub struct VerifierGraphParamsEndorsement { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub verifier_pubkey: PublicKey, - pub verifier_index: usize, - pub canonical_graph_params_hash: [u8; 32], - pub signature: SchnorrSignature, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct CreateGraph { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub graph_nonce: u64, - pub graph: SimplifiedBitvmGcGraph, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct NonceGeneration { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub committee_pubkey: PublicKey, - pub pub_nonces: CommitteePubNonces, - pub nonce_sigs: CommitteeNonceSignatures, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct AggNonceConsensus { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub committee_pubkey: PublicKey, - pub consensus_hash: [u8; 32], - pub signature: SchnorrSignature, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct CommitteePresign { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub committee_pubkey: PublicKey, - pub committee_partial_sigs: CommitteePartialSignatures, - pub agg_nonces: CommitteeAggNonces, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct EndorseGraph { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub committee_pubkey: PublicKey, - pub committee_evm_address: EvmAddress, - pub committee_sig_for_graph: Vec, // ECDSA signature signed with committee evm keypair - pub committee_sig_for_params: Vec, // ECDSA signature over canonical_graph_params_hash -} -#[derive(Serialize, Deserialize, Clone)] -pub struct GraphFinalize { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub graph_nonce: u64, - pub graph: SimplifiedBitvmGcGraph, - pub endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, - pub params_endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct PeginConfirmNonce { - pub instance_id: Uuid, - pub committee_pubkey: PublicKey, - pub pub_nonce: PubNonce, - pub nonce_sig: SchnorrSignature, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct PeginConfirmNonceConsensus { - pub instance_id: Uuid, - pub committee_pubkey: PublicKey, - pub consensus_hash: [u8; 32], - pub signature: SchnorrSignature, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct PeginConfirmPartialSig { - pub instance_id: Uuid, - pub committee_pubkey: PublicKey, - pub partial_sig: PartialSignature, - pub endorse_sig: Vec, // ECDSA signature signed with committee evm keypair -} -#[derive(Serialize, Deserialize, Clone)] -pub struct PostReady { - pub instance_id: Uuid, -} - -/// Pegout - -#[derive(Serialize, Deserialize, Clone)] -pub struct KickoffReady { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct KickoffSent { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct PreKickoffSent { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct ChallengeSent { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub challenge_txid: Txid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct WatchtowerChallengeInitSent { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct WatchtowerChallengeSent { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub watchtower_index: usize, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct WatchtowerChallengeTimeout { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct NackReady { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct OperatorCommitPubinReady { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct OperatorCommitPubinTimeout { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct AssertReady { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct AssertSent { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub assert_txid: Txid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct ChallengeAssertSent { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub challenge_assert_txid: Txid, - pub verifier_index: usize, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct WronglyChallengeTimeout { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub challenge_assert_txid: Txid, - pub verifier_index: usize, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct DisproveSent { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub disprove_type: DisproveTxType, - pub index: usize, // nack txns index or assert timeout txns index, ignored for other disprove types - pub challenge_start_txid: Option, - pub challenge_finish_txid: Txid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct Take1Ready { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct Take1Sent { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct Take2Ready { - pub instance_id: Uuid, - pub graph_id: Uuid, -} -#[derive(Serialize, Deserialize, Clone)] -pub struct Take2Sent { - pub instance_id: Uuid, - pub graph_id: Uuid, -} - -/// Others - -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub struct NodeInfo { - pub peer_id: String, - pub actor: String, - pub goat_addr: String, - pub btc_pub_key: String, - pub socket_addr: String, - pub node_name: String, - pub service_fee_rate: f64, - pub available_peg_btc: String, - /// Hex master-key Schnorr signature over peer ID, public key and issuance time. - #[serde(default)] - pub binding_sig: String, - /// Unix seconds when the binding was signed. Orders re-bindings so one key - /// maps to a single peer id at a time. - #[serde(default)] - pub binding_issued_at: i64, -} - -const NODE_INFO_BINDING_DOMAIN: &[u8] = b"bitvm2-node/node-info-binding/v1"; - -fn node_info_binding_message(peer_id: &str, btc_pub_key: &str, issued_at: i64) -> SecpMessage { - let mut hasher = Sha256::new(); - hasher.update(NODE_INFO_BINDING_DOMAIN); - hasher.update((peer_id.len() as u32).to_be_bytes()); - hasher.update(peer_id.as_bytes()); - hasher.update((btc_pub_key.len() as u32).to_be_bytes()); - hasher.update(btc_pub_key.as_bytes()); - hasher.update(issued_at.to_be_bytes()); - SecpMessage::from_digest(hasher.finalize().into()) -} - -/// Sign the binding that proves `master_keypair`'s key authorised `peer_id`. -pub fn sign_node_info_binding( - peer_id: &str, - btc_pub_key: &str, - issued_at: i64, - master_keypair: &Keypair, -) -> String { - let signature = SECP256K1 - .sign_schnorr(&node_info_binding_message(peer_id, btc_pub_key, issued_at), master_keypair); - hex::encode(signature.serialize()) -} - -/// Verify the NodeInfo binding and return its parsed public key. -pub fn verify_node_info_binding(node_info: &NodeInfo) -> Option { - if node_info.binding_sig.is_empty() { - return None; - } - let pubkey = PublicKey::from_str(&node_info.btc_pub_key).ok()?; - let xonly = XOnlyPublicKey::from(pubkey); - let signature_bytes = hex::decode(&node_info.binding_sig).ok()?; - let signature = SchnorrSignature::from_slice(&signature_bytes).ok()?; - SECP256K1 - .verify_schnorr( - &signature, - &node_info_binding_message( - &node_info.peer_id, - &node_info.btc_pub_key, - node_info.binding_issued_at, - ), - &xonly, - ) - .ok() - .map(|()| xonly) -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct SyncGraphRequest { - pub instance_id: Uuid, - pub graph_id: Uuid, -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct SyncGraph { - pub instance_id: Uuid, - pub graph_id: Uuid, - pub graph: SimplifiedBitvmGcGraph, -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct InstanceDiscarded { - // (graph_id, instance_id, OperatorPubkey) - pub graph_infos: Vec<(Uuid, Uuid, String)>, -} - -impl GOATMessage { - pub fn new(actor: Actor, content: GOATMessageContent) -> Self { - Self { actor, content } - } - - pub fn content(&self) -> &GOATMessageContent { - &self.content - } - - pub fn default_message_id() -> MessageId { - MessageId(b"__inner_message_id__".to_vec()) - } - - pub async fn serialize_message(&self) -> Result> { - let cloned = self.clone(); - tokio::task::spawn_blocking(move || { - if matches!(&cloned.content, GOATMessageContent::GenCircuits(_)) { - let mut encoded = bincode::serialize(&cloned) - .context("failed to serialize bincode GOATMessage")?; - let mut message = Vec::with_capacity(GOAT_MESSAGE_BIN_PREFIX.len() + encoded.len()); - message.extend_from_slice(GOAT_MESSAGE_BIN_PREFIX); - message.append(&mut encoded); - Ok(message) - } else { - serde_json::to_vec(&cloned).context("failed to serialize legacy JSON GOATMessage") - } - }) - .await? - } - - /// Whether `message` uses the bincode envelope, which only `GenCircuits` does. - pub fn is_binary_envelope(message: &[u8]) -> bool { - message.starts_with(GOAT_MESSAGE_BIN_PREFIX) - } - - pub async fn deserialize_message(message: &[u8]) -> Result { - let cloned = message.to_vec(); - tokio::task::spawn_blocking(move || { - if let Some(encoded) = cloned.strip_prefix(GOAT_MESSAGE_BIN_PREFIX) { - bincode::deserialize(encoded).context("failed to deserialize bincode GOATMessage") - } else { - serde_json::from_slice(&cloned) - .context("failed to deserialize legacy JSON GOATMessage") - } - }) - .await? - } -} - -/// Report exactly one forwarding verdict for each received gossipsub message. -fn report_gossip_validation( - swarm: &mut Swarm, - id: &MessageId, - propagation_source: &PeerId, - acceptance: gossipsub::MessageAcceptance, -) { - let cached = swarm.behaviour_mut().gossipsub.report_message_validation_result( - id, - propagation_source, - acceptance, - ); - if !cached { - tracing::debug!( - event = "p2p_message", - outcome = "validation_report_missed", - message_id = %hex::encode(&id.0), - "gossipsub had already evicted the message awaiting validation" - ); - } -} - -/// Apply admission before routing; a dropped message is not an error. -#[allow(clippy::too_many_arguments)] -pub async fn handle_inbound_p2p_message( - swarm: &mut Swarm, - local_db: &LocalDB, - _btc_client: &Arc, - goat_client: &Arc, - _http_client: &HttpAsyncClient, - _soldering_builder: &Option>, - actor: Actor, - from_peer_id: PeerId, - propagation_source: PeerId, - sequence_number: Option, - id: MessageId, - message: &[u8], - metrics_state: &MetricsState, - worker: &crate::p2p_msg_handler::WorkerControl, -) -> Result<()> { - use crate::p2p_admission::{self, InboundVerdict}; - use gossipsub::MessageAcceptance; - - let now = Instant::now(); - p2p_admission::refresh_registry_in_background(local_db, goat_client, Some(from_peer_id)); - let verdict = p2p_admission::evaluate_inbound_message( - local_db, - &p2p_admission::AdmissionGates::global(&actor), - &p2p_admission::InboundGossip { - direct_peer: &propagation_source, - source: &from_peer_id, - sequence_number, - data: message, - }, - now, - ) - .await; - let verdict = match verdict { - Ok(verdict) => verdict, - Err(error) => { - report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Ignore); - return Err(error).context("evaluate inbound P2P message admission"); - } - }; - - match verdict { - InboundVerdict::Drop(reason) => { - report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Ignore); - if reason.blames_direct_peer() { - strike_direct_peer(swarm, local_db, goat_client, propagation_source, now); - } - if reason == p2p_admission::DropReason::Undecodable { - metrics_state.record_p2p_receive(false); - } else if reason.after_decode() { - metrics_state.record_p2p_receive(true); - } - p2p_admission::record_drop(reason); - Ok(()) - } - // Forward messages for other roles without persistence or ACK. - InboundVerdict::Forward => { - report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); - metrics_state.record_p2p_receive(true); - refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; - Ok(()) - } - // Persist durable messages before allowing forwarding. - InboundVerdict::Enqueue { message: decoded, class, content_hash } => { - if let Err(error) = enqueue_p2p_message( - local_db, - actor, - from_peer_id, - id.clone(), - message, - &decoded, - class, - content_hash, - ) - .await - { - report_gossip_validation( - swarm, - &id, - &propagation_source, - MessageAcceptance::Ignore, - ); - return Err(error); - } - report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); - metrics_state.record_p2p_receive(true); - refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; - maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, message, now).await; - Ok(()) - } - // Forward duplicates with rate-limited ACKs and logs. - InboundVerdict::Duplicate(decoded) => { - report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); - metrics_state.record_p2p_receive(true); - p2p_admission::record_drop(p2p_admission::DropReason::DuplicatePayload); - refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; - maybe_send_graph_setup_ack(swarm, &decoded, from_peer_id, message, now).await; - Ok(()) - } - InboundVerdict::Immediate(decoded) => { - if !worker.enqueue(from_peer_id, id.clone(), decoded, message.len()) { - report_gossip_validation( - swarm, - &id, - &propagation_source, - MessageAcceptance::Ignore, - ); - p2p_admission::record_drop(p2p_admission::DropReason::ControlQueueFull); - return Ok(()); - } - report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); - metrics_state.record_p2p_receive(true); - refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; - Ok(()) - } - } -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn dispatch_immediate_message( - publisher: &mut dyn MessagePublisher, - local_db: &LocalDB, - btc_client: &Arc, - goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, - actor: Actor, - from_peer_id: PeerId, - id: MessageId, - message: GOATMessage, - metrics_state: &MetricsState, -) -> Result<()> { - let kind = message.content.event_type(); - match tokio::time::timeout( - crate::env::get_p2p_handler_timeout(), - dispatch_decoded_p2p_message( - publisher, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor, - from_peer_id, - id, - message, - metrics_state, - ), - ) - .await - { - Ok(result) => result, - Err(_) => { - tracing::warn!( - event = "p2p_message", - outcome = "handler_timeout", - message_type = kind, - "ephemeral message exceeded business worker budget" - ); - Ok(()) - } - } -} - -/// Count relay strikes and temporarily ban repeat offenders. -/// Registered peers and peers with unresolved registration are exempt. -fn strike_direct_peer( - swarm: &mut Swarm, - local_db: &LocalDB, - goat_client: &Arc, - direct_peer: PeerId, - now: Instant, -) { - let registry = crate::p2p_admission::peer_registry(); - if !registry.is_known_unregistered(&direct_peer, now) { - crate::p2p_admission::refresh_neighbour_in_background(local_db, goat_client, direct_peer); - return; - } - if !crate::p2p_admission::direct_peer_strikes().strike(&direct_peer, now) { - return; - } - // The strike table now lists the peer as banned, and the connection gate - // consults it for every connection: closing this one is all that is left. - let _ = swarm.disconnect_peer_id(direct_peer); - tracing::warn!( - event = "p2p_admission", - outcome = "direct_peer_banned", - peer_id = %direct_peer, - "temporarily ignoring a neighbour that keeps relaying invalid messages" - ); -} - -/// Refresh peer liveness at most once per cooldown. -async fn refresh_peer_timestamp_throttled(local_db: &LocalDB, from_peer_id: PeerId, now: Instant) { - let peer = from_peer_id.to_string(); - if !crate::p2p_admission::peer_timestamp_gate().allow(&peer, now) { - return; - } - if let Err(error) = update_node_timestamp(local_db, &peer).await { - tracing::debug!( - event = "p2p_message", - outcome = "peer_timestamp_update_failed", - from_peer_id = %peer, - error = %error, - "failed to refresh peer liveness timestamp" - ); - } -} - -/// Rate-limit graph-setup ACKs by delivery identity (including both peers). -async fn maybe_send_graph_setup_ack( - swarm: &mut dyn MessagePublisher, - decoded: &GOATMessage, - from_peer_id: PeerId, - payload: &[u8], - now: Instant, -) { - let Some(ack) = graph_setup_ack(&decoded.content, &from_peer_id, payload) else { - return; - }; - if !crate::p2p_admission::dedup_ack_gate().allow(&ack.delivery_id, now) { - return; - } - if let Err(error) = - send_to_peer(swarm, GOATMessage::new(Actor::All, GOATMessageContent::GraphSetupAck(ack))) - .await - { - tracing::debug!( - event = "p2p_graph_setup_ack", - outcome = "publish_failed", - error = %error, - "inbound graph-setup message remains durable; sender will retry" - ); - } -} - -/// Persist a decoded durable P2P message. Protocol work runs from the inbox on -/// a regular tick. -#[allow(clippy::too_many_arguments)] -async fn enqueue_p2p_message( - local_db: &LocalDB, - actor: Actor, - from_peer_id: PeerId, - id: MessageId, - message: &[u8], - decoded: &GOATMessage, - admission_class: store::P2pInboxAdmissionClass, - content_hash: [u8; 32], -) -> Result<()> { - let message_id = hex::encode(&id.0); - let inbox_message = P2pInboxMessage { - message_id: message_id.clone(), - business_id: decoded.content.business_ref().primary_id(), - actor: actor.to_string(), - from_peer: from_peer_id.to_string(), - msg_type: decoded.content.event_type().to_owned(), - content: message.to_vec(), - content_size: message.len() as i64, - admission_class: admission_class.to_string(), - content_hash: Some(content_hash.to_vec()), - ..Default::default() - }; - let mut inserted = None; - for attempt in 1..=P2P_INBOX_ENQUEUE_ATTEMPTS { - let result = async { - let mut storage = local_db.acquire().await?; - storage.insert_p2p_inbox_message(&inbox_message).await - } - .await; - match result { - Ok(value) => { - inserted = Some(value); - break; - } - Err(error) - if is_retryable_sqlite_error(&error) && attempt < P2P_INBOX_ENQUEUE_ATTEMPTS => - { - tracing::warn!( - event = "p2p_inbox", - outcome = "enqueue_retry", - message_id = %message_id, - attempt, - error = %error, - "retrying transient P2P inbox insert" - ); - tokio::time::sleep(Duration::from_millis(50 * attempt as u64)).await; - } - Err(error) => return Err(error).context("persist inbound P2P message"), - } - } - let inserted = inserted.expect("P2P inbox insert loop exits only after success or error"); - // Aggregate unregistered admission logs; log registered messages individually. - if admission_class.is_registered() { - tracing::info!( - event = "p2p_inbox", - outcome = if inserted { "enqueued" } else { "duplicate" }, - message_id = %message_id, - message_type = %inbox_message.msg_type, - from_peer_id = %from_peer_id, - content_size = message.len(), - "received P2P message" - ); - } else { - crate::p2p_admission::record_admitted_unregistered(); - tracing::debug!( - event = "p2p_inbox", - outcome = if inserted { "enqueued" } else { "duplicate" }, - message_id = %message_id, - message_type = %inbox_message.msg_type, - from_peer_id = %from_peer_id, - content_size = message.len(), - "received P2P message" - ); - } - Ok(()) -} - -fn p2p_retry_delay_secs(attempt_count: i64) -> i64 { - match attempt_count { - ..=1 => 10, - 2 => 30, - 3 => 60, - 4 => 120, - _ => 300, - } -} - -fn p2p_retryable_dispatch_error( - error: &anyhow::Error, -) -> Option<(RetryableDispatchReason, Option)> { - if let Some(retryable) = - error.chain().find_map(|cause| cause.downcast_ref::()) - { - return Some((retryable.reason, retryable.retry_after_secs)); - } - if is_retryable_sqlite_error(error) { - return Some((RetryableDispatchReason::StorageBusy, None)); - } - None -} - -fn log_stale_p2p_inbox_lease(message_id: &str, lease_token: &str, operation: &str) { - tracing::warn!( - event = "p2p_inbox", - outcome = "stale_lease", - message_id, - lease_token, - operation, - "ignored P2P inbox state update from a stale lease" - ); -} - -async fn fail_p2p_inbox_without_aborting_batch( - local_db: &LocalDB, - message_id: &str, - lease_token: &str, - error: &str, -) { - let result = async { - let mut storage = local_db.acquire().await?; - storage.fail_p2p_inbox_message(message_id, lease_token, error).await - } - .await; - match result { - Ok(true) => {} - Ok(false) => log_stale_p2p_inbox_lease(message_id, lease_token, "fail"), - Err(error) => log_queue_bookkeeping_failure("p2p_inbox", message_id, "fail", &error), - } -} - -async fn defer_p2p_inbox_without_aborting_batch( - local_db: &LocalDB, - message_id: &str, - lease_token: &str, - next_retry_at: i64, - reason: &str, -) { - let result = async { - let mut storage = local_db.acquire().await?; - storage.defer_p2p_inbox_message(message_id, lease_token, next_retry_at, reason).await - } - .await; - match result { - Ok(true) => {} - Ok(false) => log_stale_p2p_inbox_lease(message_id, lease_token, "defer"), - Err(error) => log_queue_bookkeeping_failure("p2p_inbox", message_id, "defer", &error), - } -} - -async fn abandon_p2p_inbox_after_panic( - local_db: &LocalDB, - message_id: &str, - lease_token: &str, - detail: &str, -) { - let error = format!("handler panicked: {detail}"); - let result = async { - let mut storage = local_db.acquire().await?; - storage - .abandon_p2p_inbox_message( - message_id, - lease_token, - current_time_secs(), - QUEUE_ABANDON_BACKOFF_SECS, - &error, - ) - .await - } - .await; - match result { - Ok(true) => {} - Ok(false) => log_stale_p2p_inbox_lease(message_id, lease_token, "abandon"), - Err(error) => log_queue_bookkeeping_failure("p2p_inbox", message_id, "abandon", &error), - } -} - -async fn abandon_local_message_after_panic( - local_db: &LocalDB, - message_id: &str, - message_version: i64, - detail: &str, -) { - let error = format!("handler panicked: {detail}"); - let result = async { - let mut storage = local_db.acquire().await?; - storage - .abandon_local_message( - message_id, - message_version, - current_time_secs(), - QUEUE_ABANDON_BACKOFF_SECS, - &error, - ) - .await - } - .await; - match result { - Ok(true) => {} - Ok(false) => tracing::warn!( - event = "local_message_queue", - outcome = "stale_claim", - message_id, - message_version, - operation = "abandon", - "ignored local message update from a stale claim" - ), - Err(error) => { - log_queue_bookkeeping_failure("local_message_queue", message_id, "abandon", &error) - } - } -} - -/// Claim one listed inbox row immediately before dispatching it. -/// -/// `None` means the row is no longer claimable or the claim could not be -/// persisted; either way it is skipped this tick and listed again on the next. -async fn claim_p2p_inbox_candidate( - local_db: &LocalDB, - message_id: &str, -) -> Option { - let now = current_time_secs(); - let result = async { - let mut storage = local_db.acquire().await?; - storage.claim_p2p_inbox_message(message_id, now, now + P2P_INBOX_LEASE_SECS).await - } - .await; - match result { - Ok(Some(message)) => Some(message), - Ok(None) => { - tracing::debug!( - event = "p2p_inbox", - outcome = "claim_skipped", - message_id, - "listed inbox message is no longer claimable" - ); - None - } - Err(error) => { - tracing::error!( - event = "p2p_inbox", - outcome = "claim_failed", - message_id, - error = %error, - "failed to claim a listed inbox message; it stays queued for the next tick" - ); - None - } - } -} - -/// Claim one listed local message immediately before dispatching it. See -/// [`claim_p2p_inbox_candidate`]. -async fn claim_local_candidate( - local_db: &LocalDB, - candidate: &store::Message, -) -> Option { - let now = current_time_secs(); - let result = async { - let mut storage = local_db.acquire().await?; - storage - .claim_local_message( - &candidate.message_id, - candidate.message_version, - now, - now + LOCAL_MESSAGE_LEASE_SECS, - ) - .await - } - .await; - match result { - Ok(Some(message)) => Some(message), - Ok(None) => { - tracing::debug!( - event = "local_message_queue", - outcome = "claim_skipped", - queued_message_id = %candidate.message_id, - message_version = candidate.message_version, - "listed local message is no longer claimable" - ); - None - } - Err(error) => { - tracing::error!( - event = "local_message_queue", - outcome = "claim_failed", - queued_message_id = %candidate.message_id, - error = %error, - "failed to claim a listed local message; it stays queued for the next tick" - ); - None - } - } -} - -/// Charge and release every queue claim left behind by a previous process. -/// -/// Run once at startup, before any dispatcher starts. The database is -/// process-local, so a `Processing` row at this point is an attempt that never -/// reported an outcome. Waiting for its lease to lapse instead kept the row -/// locked for minutes while every producer touching it backed off with -/// ResourceLocked; a graceful shutdown never leaves such rows behind. -pub async fn reclaim_stale_queue_claims(local_db: &LocalDB) -> Result<(u64, u64)> { - let now = current_time_secs(); - let mut storage = local_db.start_immediate_transaction().await?; - let local = storage.reclaim_processing_local_messages(now, QUEUE_ABANDON_BACKOFF_SECS).await?; - let inbox = - storage.reclaim_processing_p2p_inbox_messages(now, QUEUE_ABANDON_BACKOFF_SECS).await?; - storage.commit().await?; - Ok((local, inbox)) -} - -/// Restore persisted operator bindings, re-verify signatures and refresh stake asynchronously. -pub async fn load_persisted_peer_bindings( - local_db: &LocalDB, - goat_client: &Arc, -) -> Result { - let bindings = local_db - .acquire() - .await? - .load_p2p_peer_bindings(crate::p2p_admission::OPERATOR_BINDING_LOAD_LIMIT) - .await?; - let registry = crate::p2p_admission::peer_registry(); - let mut loaded = 0; - let mut verified_bindings = std::collections::HashMap::new(); - for node in bindings { - let node_info = NodeInfo { - peer_id: node.peer_id.clone(), - btc_pub_key: node.btc_pub_key.clone(), - binding_sig: node.binding_sig.clone(), - binding_issued_at: node.binding_issued_at, - ..Default::default() - }; - let Some(operator_key) = verify_node_info_binding(&node_info) else { - continue; - }; - if node.binding_issued_at > current_time_secs() + NODE_INFO_BINDING_MAX_FUTURE_SECS { - continue; - } - if let Ok(peer_id) = PeerId::from_str(&node.peer_id) { - let operator_key = crate::p2p_admission::operator_key(&operator_key); - registry.observe_operator_binding(&peer_id, &operator_key, node.binding_issued_at); - verified_bindings.insert(peer_id, (operator_key, node.binding_issued_at)); - loaded += 1; - } - } - // Apply configured bindings only where no signed binding exists; stake still requires confirmation. - for (peer_id, operator_key) in crate::env::get_p2p_trusted_operator_bindings() { - match registry.trust_operator_binding(&peer_id, &operator_key) { - Ok(()) => loaded += 1, - Err(conflict) => tracing::warn!( - event = "p2p_admission", - outcome = "trusted_binding_ignored", - peer_id = %peer_id, - configured_key = %hex::encode(operator_key), - conflict = ?conflict, - env = crate::env::ENV_P2P_TRUSTED_OPERATOR_BINDINGS, - "configured operator binding disagrees with one the operator signed; keeping \ - the signed binding — update or remove the configured entry" - ), - } - } - // Apply configured replay resets, then restore persisted marks before admission. - let reset_peers = crate::env::get_p2p_replay_mark_reset_peers(); - if !reset_peers.is_empty() { - let dropped = local_db.acquire().await?.delete_p2p_replay_marks(&reset_peers).await?; - tracing::warn!( - event = "p2p_admission", - outcome = "replay_marks_reset", - requested = reset_peers.len(), - dropped, - "dropped persisted replay marks on operator request" - ); - } - for (peer_id, highest) in local_db.acquire().await?.load_p2p_replay_marks().await? { - if let Ok(peer_id) = PeerId::from_str(&peer_id) { - crate::p2p_admission::replay_guard().restore_mark(&peer_id, highest as u64); - } - } - // Peers confirmed on chain in an earlier session keep their class and are - // re-validated as known members, outside the discovery budget. - loaded += crate::p2p_admission::seed_registry_from_store( - local_db, - registry, - &verified_bindings, - Instant::now(), - ) - .await?; - // Everything loaded is due; the unconfirmed remainder is picked up by the - // regular tick as lookup slots free up. - crate::p2p_admission::refresh_due_registrations_in_background(local_db, goat_client); - Ok(loaded) -} - -/// A clock disagreement below this is not worth reporting. -const CLOCK_LEAD_REPORT_THRESHOLD: Duration = Duration::from_secs(60); - -/// Warn about registered authors whose message numbering is ahead of the local clock. -fn report_clock_leads(leads: Vec<(PeerId, Duration)>) { - let ahead: Vec<&(PeerId, Duration)> = - leads.iter().filter(|(_, lead)| *lead >= CLOCK_LEAD_REPORT_THRESHOLD).collect(); - let max_lead = ahead.iter().map(|(_, lead)| *lead).max().unwrap_or_default(); - match ahead.as_slice() { - [] => {} - [(peer, lead)] => tracing::warn!( - event = "p2p_admission", - outcome = "author_clock_ahead", - peer_id = %peer, - lead_secs = lead.as_secs(), - "a registered author's clock is ahead of this node's; past the tolerance its \ - messages are refused as coming from the future" - ), - several => tracing::error!( - event = "p2p_admission", - outcome = "local_clock_behind_suspected", - authors = several.len(), - max_lead_secs = max_lead.as_secs(), - "several registered authors' clocks are ahead of this node's: check this node's \ - clock (NTP). Past the tolerance their messages are refused as coming from the future" - ), - } -} - -/// Refresh registrations, summarize drops and purge stale node rows. -pub async fn run_p2p_admission_maintenance(local_db: &LocalDB, goat_client: &Arc) { - let now = Instant::now(); - crate::p2p_admission::refresh_due_registrations_in_background(local_db, goat_client); - crate::p2p_admission::log_drop_summary(); - // Expired bans only need forgetting: the gate compares against the clock. - crate::p2p_admission::direct_peer_strikes().take_expired_bans(now); - report_clock_leads(crate::p2p_admission::replay_guard().take_clock_leads()); - for (peer, rejected) in crate::p2p_admission::replay_guard().take_lockout_suspects() { - tracing::warn!( - event = "p2p_admission", - outcome = "replay_lockout_suspected", - peer_id = %peer, - rejected, - reset_env = crate::env::ENV_P2P_REPLAY_MARK_RESET_PEERS, - "a registered author keeps being refused as a replay; if it is not under \ - attack, its clock stepped back or another instance shares its key" - ); - } - - // Persist replay marks and confirm them only after a successful write. - let owed = crate::p2p_admission::replay_guard().pending_marks(); - let marks: Vec<(String, i64)> = owed - .iter() - .map(|(peer, highest)| (peer.to_string(), (*highest).min(i64::MAX as u64) as i64)) - .collect(); - let purge_due = crate::p2p_admission::node_table_purge_due(NODE_TABLE_PURGE_INTERVAL, now); - if marks.is_empty() && !purge_due { - return; - } - let result = async { - let mut storage = local_db.acquire().await?; - storage.raise_p2p_replay_marks(&marks).await?; - crate::p2p_admission::replay_guard().confirm_persisted(&owed); - if !purge_due { - return Ok(0); - } - if let Some(verifiers) = crate::p2p_admission::peer_registry().verifier_peer_ids() { - storage.prune_p2p_replay_marks(&verifiers).await?; - } - storage - .purge_stale_unregistered_nodes( - current_time_secs() - NODE_TABLE_STALE_SECS, - &crate::env::get_peer_id(), - ) - .await - } - .await; - match result { - Ok(0) => {} - Ok(purged) => tracing::info!( - event = "p2p_admission", - outcome = "stale_nodes_purged", - purged, - "removed node rows of unregistered peers that stopped announcing themselves" - ), - Err(error) => tracing::warn!( - event = "p2p_admission", - outcome = "maintenance_write_failed", - error = %error, - "failed to persist replay marks or purge stale node rows; retried next tick" - ), - } -} - -async fn renew_p2p_inbox_lease_until_cancelled( - local_db: LocalDB, - message_id: String, - lease_token: String, - cancellation: CancellationToken, -) -> bool { - let mut interval = - tokio::time::interval(Duration::from_secs(P2P_INBOX_LEASE_RENEW_INTERVAL_SECS)); - interval.tick().await; - loop { - tokio::select! { - _ = cancellation.cancelled() => return true, - _ = interval.tick() => { - let renewal = match local_db.acquire().await { - Ok(mut storage) => storage - .renew_p2p_inbox_lease( - &message_id, - &lease_token, - current_time_secs() + P2P_INBOX_LEASE_SECS, - ) - .await, - Err(error) => Err(error), - }; - match renewal { - Ok(true) => {} - Ok(false) => { - log_stale_p2p_inbox_lease(&message_id, &lease_token, "renew"); - return false; - } - Err(error) => { - tracing::warn!( - event = "p2p_inbox", - outcome = "lease_renew_failed", - message_id, - error = %error, - "failed to renew P2P inbox lease; will retry before expiry" - ); - } - } - } - } - } -} - -/// Drain inbox batches until caught up or the message-start budget is exhausted. -#[allow(clippy::too_many_arguments)] -async fn handle_p2p_inbox_messages( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - btc_client: &Arc, - goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, - actor: Actor, - metrics_state: &MetricsState, - shutdown: &CancellationToken, -) -> Result { - let started_at = Instant::now(); - let deadline = started_at + P2P_INBOX_DRAIN_BUDGET; - let mut first_batch = true; - // A backlogged worker comes straight back here; the retention sweeps keep - // the cadence of the tick they were written for. - let housekeeping = inbox_housekeeping_due(started_at); - let mut total_claimed = 0; - loop { - let batch = drain_p2p_inbox_batch( - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor.clone(), - metrics_state, - shutdown, - first_batch, - first_batch && housekeeping, - deadline, - ) - .await?; - first_batch = false; - total_claimed += batch.claimed; - // Stop when fewer messages were claimed than requested. - let full_batch = (batch.claimed as i64) >= get_p2p_inbox_batch_size(); - if !full_batch || Instant::now() >= deadline || shutdown.is_cancelled() { - // Continue immediately only when a productive pass stopped on its budget. - return Ok(!shutdown.is_cancelled() - && total_claimed > 0 - && (batch.cut_short || full_batch)); - } - } -} - -/// Whether the inbox retention sweeps should run in this pass: once per regular -/// tick interval, however often a backlog brings the worker back. -fn inbox_housekeeping_due(now: Instant) -> bool { - static LAST_RUN: Mutex> = Mutex::new(None); - let mut last_run = LAST_RUN.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - let interval = Duration::from_secs(crate::env::REGULAR_TASK_INTERVAL_SECOND); - if last_run.is_some_and(|last_run| now.saturating_duration_since(last_run) < interval) { - return false; - } - *last_run = Some(now); - true -} - -struct InboxBatch { - claimed: usize, - /// The budget ran out with listed rows still unserved. - cut_short: bool, -} - -/// List and dispatch one inbox batch. -#[allow(clippy::too_many_arguments)] -async fn drain_p2p_inbox_batch( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - btc_client: &Arc, - goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, - actor: Actor, - metrics_state: &MetricsState, - shutdown: &CancellationToken, - first_batch: bool, - housekeeping: bool, - deadline: Instant, -) -> Result { - let now = current_time_secs(); - let active_heavy_task_ids = active_heavy_task_message_ids(); - let mut storage = local_db.start_immediate_transaction().await?; - let (mut quarantined, mut purged, mut expired, mut trimmed, mut capped) = (0, 0, 0, 0, 0); - if housekeeping { - // Quarantine rows whose dispatch repeatedly failed to report any outcome. - // Returned retryable errors do not consume this budget. - quarantined = storage.quarantine_p2p_inbox_messages(now, QUEUE_MAX_ABANDONS).await?; - // Bound terminal metadata and the temporary payload retained for manual - // inspection of quarantined rows. - purged = storage.purge_terminal_p2p_inbox_messages(now - MESSAGE_EXPIRE_TIME).await?; - // Expire Pending rows by age, using the shorter unregistered retention. - expired = storage - .expire_pending_p2p_inbox_messages( - Some(store::P2pInboxAdmissionClass::Unregistered), - now - crate::p2p_admission::UNREGISTERED_PENDING_TTL_SECS, - ) - .await? - + storage.expire_pending_p2p_inbox_messages(None, now - MESSAGE_EXPIRE_TIME).await?; - // Cap quarantined payload bytes and terminal row count. - trimmed = - storage.trim_quarantined_p2p_inbox_payloads(P2P_INBOX_QUARANTINE_MAX_BYTES).await?; - capped = storage - .purge_p2p_inbox_over_terminal_cap( - P2P_INBOX_TERMINAL_MAX_ROWS, - P2P_INBOX_TERMINAL_PURGE_BATCH, - ) - .await?; - } - // Only list here. Each row is claimed right before its own dispatch so a - // crash mid-dispatch is charged to that row alone. - let batch_size = get_p2p_inbox_batch_size(); - let candidates = storage - .list_claimable_p2p_inbox_messages_by_class( - now, - batch_size, - (batch_size / P2P_INBOX_REGISTERED_BATCH_DIVISOR).max(1), - (batch_size / P2P_INBOX_UNREGISTERED_BATCH_DIVISOR).max(1), - QUEUE_MAX_ABANDONS, - &active_heavy_task_ids, - ) - .await?; - storage.commit().await?; - let mut claimed = 0; - let mut cut_short = false; - - if quarantined > 0 { - tracing::warn!( - event = "p2p_inbox", - outcome = "quarantined", - quarantined, - max_abandons = QUEUE_MAX_ABANDONS, - "quarantined inbox messages that repeatedly abandoned their lease" - ); - } - if purged > 0 { - tracing::info!( - event = "p2p_inbox", - outcome = "purged", - purged, - "removed terminal inbox rows past their retention window" - ); - } - if expired > 0 { - tracing::warn!( - event = "p2p_inbox", - outcome = "expired", - expired, - "retired inbox messages that stayed pending past their retention window" - ); - } - if trimmed > 0 || capped > 0 { - tracing::info!( - event = "p2p_inbox", - outcome = "capacity_trimmed", - trimmed_quarantine_payloads = trimmed, - deleted_terminal_rows = capped, - "trimmed retained inbox data over its capacity limits" - ); - } - - // Serve listed classes using the persistent weighted rotation. - let mut queues: [std::collections::VecDeque; 3] = Default::default(); - for candidate in candidates { - queues[crate::p2p_admission::InboxSchedule::queue_of(&candidate.admission_class)] - .push_back(candidate); - } - loop { - // Check the budget before each claim; always allow the first message to make progress. - if (claimed > 0 || !first_batch) && Instant::now() >= deadline { - cut_short = queues.iter().any(|queue| !queue.is_empty()); - break; - } - let Some(candidate) = crate::p2p_admission::inbox_schedule().next(&mut queues) else { - break; - }; - let Some(message) = claim_p2p_inbox_candidate(local_db, &candidate.message_id).await else { - continue; - }; - claimed += 1; - let from_peer_id = match PeerId::from_str(&message.from_peer) { - Ok(peer_id) => peer_id, - Err(error) => { - fail_p2p_inbox_without_aborting_batch( - local_db, - &message.message_id, - &message.lease_token, - &format!("invalid stored source peer: {error}"), - ) - .await; - continue; - } - }; - let decoded = match GOATMessage::deserialize_message(&message.content).await { - Ok(message) => message, - Err(error) => { - fail_p2p_inbox_without_aborting_batch( - local_db, - &message.message_id, - &message.lease_token, - &error.to_string(), - ) - .await; - continue; - } - }; - // After persistence, sender authorization belongs to the handler, not the admission cache. - let fingerprint = DispatchFingerprint::of(decoded.content()); - let heavy_task = heavy_task_from_content(decoded.content(), &actor); - - if let Some(heavy_task) = heavy_task { - let local_db = local_db.clone(); - let btc_client = Arc::clone(btc_client); - let goat_client = Arc::clone(goat_client); - let soldering_builder = soldering_builder.clone(); - let metrics_state = metrics_state.clone(); - let message_id = message.message_id.clone(); - let lease_token = message.lease_token.clone(); - let attempt_count = message.attempt_count; - let task_type = message.msg_type.clone(); - let task_type_for_task = task_type.clone(); - let task_kind = heavy_task.kind(); - let graph_id = heavy_task.graph_id(); - let Some(permit) = try_acquire_heavy_task_permit(&message_id, &lease_token) else { - let retry_after_secs = 5; - defer_p2p_inbox_without_aborting_batch( - &local_db, - &message_id, - &lease_token, - current_time_secs() + retry_after_secs, - RetryableDispatchReason::ResourceLocked.code(), - ) - .await; - tracing::debug!( - event = "p2p_inbox", - outcome = "deferred", - reason = RetryableDispatchReason::ResourceLocked.code(), - message_id, - retry_after_secs, - task_kind, - "deferred heavy task while the worker is busy" - ); - continue; - }; - let shutdown = shutdown.clone(); - tokio::spawn(async move { - let _permit = permit; - let lease_cancellation = CancellationToken::new(); - let _lease_guard = LeaseRenewalGuard(lease_cancellation.clone()); - let lease_renewal = tokio::spawn(renew_p2p_inbox_lease_until_cancelled( - local_db.clone(), - message_id.clone(), - lease_token.clone(), - lease_cancellation.clone(), - )); - let context = HeavyTaskContext { - local_db: local_db.clone(), - btc_client, - goat_client, - soldering_builder, - metrics_state: metrics_state.clone(), - from_peer_id, - }; - // Set the inbox task-local context explicitly for the spawned heavy task. - let execution = supervise_dispatch( - track_inbox_retry(fingerprint, run_heavy_task(&context, heavy_task)), - &shutdown, - ) - .await; - lease_cancellation.cancel(); - let lease_is_current = match lease_renewal.await { - Ok(lease_is_current) => lease_is_current, - Err(error) => { - tracing::error!(error = %error, message_id, "P2P inbox lease renewal task failed"); - false - } - }; - match execution { - DispatchExecution::Completed(result) => { - if !lease_is_current { - return; - } - // Apply the same RPC retry classification as inline dispatch. - let result = result.map_err(classify_retryable_dispatch_error); - metrics_state.record_message_dispatch( - &task_type_for_task, - if result.is_ok() { "success" } else { "failed" }, - ); - if let Err(error) = finish_p2p_inbox_attempt( - &local_db, - &metrics_state, - &message_id, - &lease_token, - &task_type_for_task, - attempt_count, - result, - ) - .await - { - tracing::error!(error = %error, message_id, "failed to persist heavy task result"); - } - } - DispatchExecution::Shutdown => { - if lease_is_current { - defer_p2p_inbox_without_aborting_batch( - &local_db, - &message_id, - &lease_token, - current_time_secs(), - "graceful_shutdown", - ) - .await; - } - } - DispatchExecution::Panicked(detail) => { - metrics_state.record_message_dispatch(&task_type_for_task, "failed"); - tracing::error!( - event = "heavy_task_panic", - outcome = "node_shutdown", - message_id, - graph_id = %graph_id, - message_type = %task_type_for_task, - task_kind, - detail, - "heavy task panicked; recorded an abandon and stopping the node" - ); - abandon_p2p_inbox_after_panic( - &local_db, - &message_id, - &lease_token, - &detail, - ) - .await; - shutdown.cancel(); - } - } - }); - tracing::info!( - event = "p2p_inbox", - outcome = "heavy_task_started", - message_id = %message.message_id, - graph_id = %graph_id, - message_type = %task_type, - task_kind, - "started background heavy task" - ); - continue; - } - let raw_message_id = match hex::decode(&message.message_id) { - Ok(message_id) => MessageId(message_id), - Err(error) => { - fail_p2p_inbox_without_aborting_batch( - local_db, - &message.message_id, - &message.lease_token, - &format!("invalid stored message id: {error}"), - ) - .await; - continue; - } - }; - - // Keep the deeply nested dispatch future out of the enclosing task's - // inline state machine. - let dispatch: BoxedDispatch<'_> = Box::pin(dispatch_decoded_p2p_message( - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor.clone(), - from_peer_id, - raw_message_id, - decoded, - metrics_state, - )); - let handler_budget = crate::env::get_p2p_handler_timeout(); - let dispatch = track_inbox_retry(fingerprint, dispatch); - let dispatch = tokio::time::timeout(handler_budget, dispatch); - let result = match supervise_dispatch(dispatch, shutdown).await { - DispatchExecution::Completed(Ok(result)) => result, - // Cancel an expired handler at its current await and retry its persisted work. - DispatchExecution::Completed(Err(_elapsed)) => { - tracing::warn!( - event = "p2p_inbox", - outcome = "handler_timeout", - message_id = %message.message_id, - message_type = %message.msg_type, - from_peer_id = %from_peer_id, - budget_secs = handler_budget.as_secs(), - attempt_count = message.attempt_count + 1, - "P2P message handler exceeded its execution budget; deferred for retry" - ); - defer_p2p_inbox_without_aborting_batch( - local_db, - &message.message_id, - &message.lease_token, - current_time_secs() + p2p_retry_delay_secs(message.attempt_count + 1), - "handler_timeout", - ) - .await; - continue; - } - DispatchExecution::Shutdown => { - defer_p2p_inbox_without_aborting_batch( - local_db, - &message.message_id, - &message.lease_token, - current_time_secs(), - "graceful_shutdown", - ) - .await; - return Ok(InboxBatch { claimed, cut_short: false }); - } - DispatchExecution::Panicked(detail) => { - tracing::error!( - event = "p2p_dispatch_panic", - outcome = "node_shutdown", - message_id = %message.message_id, - message_type = %message.msg_type, - detail, - "P2P message handler panicked; recorded an abandon and stopping the node" - ); - abandon_p2p_inbox_after_panic( - local_db, - &message.message_id, - &message.lease_token, - &detail, - ) - .await; - shutdown.cancel(); - bail!("P2P message handler panicked: {detail}"); - } - }; - metrics_state.record_message_dispatch( - &message.msg_type, - if result.is_ok() { "success" } else { "failed" }, - ); - if let Err(error) = finish_p2p_inbox_attempt( - local_db, - metrics_state, - &message.message_id, - &message.lease_token, - &message.msg_type, - message.attempt_count, - result, - ) - .await - { - log_queue_bookkeeping_failure("p2p_inbox", &message.message_id, "finish", &error); - } - } - Ok(InboxBatch { claimed, cut_short }) -} - -async fn finish_p2p_inbox_attempt( - local_db: &LocalDB, - metrics_state: &MetricsState, - message_id: &str, - lease_token: &str, - message_type: &str, - attempt_count: i64, - result: Result<()>, -) -> Result<()> { - let mut storage = local_db.acquire().await?; - match result { - Ok(()) => { - if !storage.complete_p2p_inbox_message(message_id, lease_token).await? { - log_stale_p2p_inbox_lease(message_id, lease_token, "complete"); - } - } - Err(error) => { - let Some((reason, requested_retry_after_secs)) = p2p_retryable_dispatch_error(&error) - else { - if !storage - .fail_p2p_inbox_message(message_id, lease_token, &error.to_string()) - .await? - { - log_stale_p2p_inbox_lease(message_id, lease_token, "fail"); - } - tracing::warn!( - event = "p2p_inbox", - outcome = "failed", - message_id, - message_type, - attempt_count, - error = %error, - "cached P2P message failed permanently" - ); - return Ok(()); - }; - let retry_after_secs = - requested_retry_after_secs.unwrap_or_else(|| p2p_retry_delay_secs(attempt_count)); - if !storage - .retry_p2p_inbox_message( - message_id, - lease_token, - current_time_secs() + retry_after_secs, - &error.to_string(), - ) - .await? - { - log_stale_p2p_inbox_lease(message_id, lease_token, "retry"); - } - metrics_state.record_message_retry(); - tracing::warn!( - event = "p2p_inbox", - outcome = "deferred", - reason = reason.code(), - message_id, - message_type, - attempt_count, - retry_after_secs, - error = %error, - "deferred cached P2P message for retry" - ); - } - } - Ok(()) -} - -/// Publish due outbox rows until caught up, blocked or past the start budget. -async fn handle_p2p_outbox_messages( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, -) -> Result<()> { - let deadline = Instant::now() + P2P_INBOX_DRAIN_BUDGET; - let now = current_time_secs(); - let mut storage = local_db.start_immediate_transaction().await?; - let expired = storage.expire_p2p_outbox_retry_messages(now).await?; - storage.commit().await?; - - if expired > 0 { - tracing::warn!( - event = "p2p_outbox", - outcome = "retry_window_expired", - expired, - "outbound messages reached the end of their retry window: graph-setup messages without the expected ACK, signing-round messages without their round closing" - ); - } - recover_exhausted_setup(local_db).await?; - drain_p2p_outbox(swarm, local_db, deadline).await -} - -/// Recovery is driven by durable delivery state even when no new gossip arrives. -async fn recover_exhausted_setup(local_db: &LocalDB) -> Result<()> { - let now = current_time_secs(); - let Some((id, bytes, created_at, ack_peer_id)) = - local_db.acquire().await?.next_exhausted_setup(now).await? - else { - return Ok(()); - }; - let decoded = GOATMessage::deserialize_message(&bytes).await; - let mut storage = local_db.start_immediate_transaction().await?; - let mut resume = false; - let mut canonical = id.clone(); - if let Ok(message) = decoded - && let Some(key) = graph_setup_outbox_id(&message.content, Some(&ack_peer_id)) - && let BusinessRef::Graph { instance_id, graph_id } = message.content.business_ref() - { - canonical = key; - let instance = storage.find_instance(&instance_id).await?; - let graph = storage.find_graph(&graph_id).await?; - resume = now - created_at < MESSAGE_EXPIRE_TIME - && !graph_setup_delivery_finished(&message.content, graph.as_ref()) - && instance.is_some_and(|row| { - matches!( - InstanceBridgeInStatus::from_str(&row.status), - Ok(InstanceBridgeInStatus::UserBroadcastPeginPrepare - | InstanceBridgeInStatus::Presigned - | InstanceBridgeInStatus::RelayerL1Broadcasted - | InstanceBridgeInStatus::RelayerL2Minted) - ) - }); - } - storage - .recover_setup_delivery( - &id, - &canonical, - now, - get_p2p_graph_setup_retry_window_secs(), - resume, - ) - .await?; - storage.commit().await?; - tracing::info!(event = "setup_delivery_recovery", message_id = %id, resume, - "settled exhausted setup delivery"); - Ok(()) -} - -fn setup_retry_interval_secs(publish_count: i64, base: i64, id: &str) -> i64 { - let base = base.max(1); - let delay = base - .saturating_mul(1_i64 << publish_count.saturating_sub(1).clamp(0, 20)) - .min(base.max(60)); - // Stable jitter keeps retries reproducible across restarts without synchronizing peers. - let jitter = - id.bytes().fold(publish_count as u64, |n, b| n.wrapping_mul(31).wrapping_add(b as u64)) % 7; - delay + jitter as i64 -} - -async fn drain_p2p_outbox( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - deadline: Instant, -) -> Result<()> { - loop { - // Check the start budget before claiming the next row; let in-flight publishes finish. - if Instant::now() >= deadline { - break; - } - let now = current_time_secs(); - let mut storage = local_db.start_immediate_transaction().await?; - let message = - storage.claim_p2p_outbox_messages(now, now + P2P_INBOX_LEASE_SECS, 1).await?.pop(); - storage.commit().await?; - let Some(message) = message else { break }; - match publish_p2p_outbox_row(swarm, local_db, &message).await { - Ok(OutboxRowOutcome::PublishFailed) => break, - Ok(OutboxRowOutcome::Published | OutboxRowOutcome::Closed) => {} - Err(error) => { - tracing::warn!( - event = "p2p_outbox", - outcome = "bookkeeping_failed", - message_id = %message.message_id, - message_type = %message.msg_type, - error = %error, - "could not settle an outbound P2P message; releasing it for the next pass" - ); - if let Ok(mut storage) = local_db.acquire().await { - let _ = storage - .retry_p2p_outbox_message( - &message.message_id, - current_time_secs() + PROTOCOL_RETRY_BASE_SECS, - &error.to_string(), - ) - .await; - } - } - } - } - Ok(()) -} - -enum OutboxRowOutcome { - Published, - PublishFailed, - /// Settled without publishing: corrupt, or nobody needs it any more. - Closed, -} - -async fn publish_p2p_outbox_row( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - message: &store::P2pOutboxMessage, -) -> Result { - let outbound = match GOATMessage::deserialize_message(&message.content).await { - Ok(message) => message, - Err(error) => { - local_db - .acquire() - .await? - .fail_p2p_outbox_message(&message.message_id, &error.to_string()) - .await?; - tracing::error!( - event = "p2p_outbox", - outcome = "failed", - message_id = %message.message_id, - message_type = %message.msg_type, - error = %error, - "discarded corrupt durable outbound P2P message" - ); - return Ok(OutboxRowOutcome::Closed); - } - }; - if let Some(canonical_id) = graph_setup_outbox_id(&outbound.content, Some(&message.ack_peer_id)) - && let BusinessRef::Graph { instance_id, graph_id } = outbound.content.business_ref() - { - let mut storage = local_db.acquire().await?; - let instance = storage.find_instance(&instance_id).await?; - let graph = storage.find_graph(&graph_id).await?; - let failed = instance.as_ref().is_some_and(|row| { - InstanceBridgeInStatus::from_str(&row.status).is_ok_and(|status| { - SigningRound::GraphSigning.delivery_finished(None, Some(&status)) - }) - }); - if failed - || graph_setup_delivery_finished(&outbound.content, graph.as_ref()) - || current_time_secs() - message.created_at >= MESSAGE_EXPIRE_TIME - { - storage.cancel_p2p_outbox_message(&canonical_id).await?; - return Ok(OutboxRowOutcome::Closed); - } - } - let signing_round = if message.message_id.starts_with(PROTOCOL_OUTBOX_PREFIX) { - let round = SigningRound::of(&outbound.content) - .filter(|_| outbound.content.business_ref() != BusinessRef::Unscoped); - if round.is_none() { - local_db - .acquire() - .await? - .fail_p2p_outbox_message( - &message.message_id, - "signing outbox message has no round or business reference", - ) - .await?; - return Ok(OutboxRowOutcome::Closed); - } - round - } else { - None - }; - if signing_round.is_some() && protocol_delivery_finished(local_db, &outbound.content).await? { - local_db.acquire().await?.complete_p2p_outbox_message(&message.message_id).await?; - tracing::info!( - event = "p2p_outbox", - outcome = "round_closed", - message_id = %message.message_id, - message_type = %message.msg_type, - publish_count = message.publish_count, - "stopped re-publishing a signing-round message: its round is over" - ); - return Ok(OutboxRowOutcome::Closed); - } - let result = send_to_peer(swarm, outbound).await; - let mut storage = local_db.acquire().await?; - match result { - Ok(_) => { - if message.retry_until > 0 { - let interval = match signing_round { - Some(round) => protocol_retry_interval_secs( - message.publish_count + 1, - round.retry_ceiling_secs(crate::env::get_p2p_protocol_retry_max_secs()), - ), - None => setup_retry_interval_secs( - message.publish_count + 1, - message.retry_interval_secs, - &message.message_id, - ), - }; - let next_retry_at = current_time_secs() + interval; - storage.schedule_p2p_outbox_retry(&message.message_id, next_retry_at).await?; - tracing::info!( - event = "p2p_outbox", - outcome = "published_retry_window", - message_id = %message.message_id, - message_type = %message.msg_type, - next_retry_at, - retry_until = message.retry_until, - "published durable P2P message; it stays due until acknowledged, closed or expired" - ); - } else { - storage.complete_p2p_outbox_message(&message.message_id).await?; - } - Ok(OutboxRowOutcome::Published) - } - Err(error) => { - // `attempt_count` counts claims, and a signing-round row is claimed - // for days: by it, one failed publish would cost the longest delay. - let retry_after_secs = if signing_round.is_some() { - PROTOCOL_RETRY_BASE_SECS - } else { - p2p_retry_delay_secs(message.attempt_count) - }; - storage - .retry_p2p_outbox_message( - &message.message_id, - current_time_secs() + retry_after_secs, - &error.to_string(), - ) - .await?; - tracing::warn!( - event = "p2p_outbox", - outcome = "deferred", - message_id = %message.message_id, - message_type = %message.msg_type, - attempt_count = message.attempt_count, - retry_after_secs, - error = %error, - "deferred outbound P2P message" - ); - Ok(OutboxRowOutcome::PublishFailed) - } - } -} - -#[allow(clippy::too_many_arguments)] -pub async fn handle_self_p2p_msg( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - btc_client: &Arc, - goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, - actor: Actor, - from_peer_id: PeerId, - id: MessageId, - message: &[u8], - metrics_state: &MetricsState, - shutdown: &CancellationToken, -) -> Result { - if id != GOATMessage::default_message_id() { - tracing::warn!( - event = "local_message_queue", - outcome = "unexpected_message_id", - message_id = ?id, - "ignoring local queue trigger with an unexpected message id" - ); - return Ok(false); - } - let message = GOATMessage::deserialize_message(message).await?; - tracing::info!( - event = "local_message_queue", - outcome = "trigger_received", - role = %message.actor, - message_type = message.content.event_type(), - message_id = ?id, - from_peer_id = %from_peer_id, - "received local queue trigger" - ); - - let (candidates, quarantined) = - list_batch_local_msg(local_db, QUEUE_MAX_ABANDONS, LOCAL_MESSAGE_BATCH_SIZE).await?; - if quarantined > 0 { - tracing::warn!( - event = "local_message_queue", - outcome = "quarantined", - quarantined, - max_abandons = QUEUE_MAX_ABANDONS, - "retired local messages that kept failing to report an outcome" - ); - } - tracing::info!( - event = "local_message_queue", - outcome = "batch_listed", - role = %actor, - batch_size = candidates.len(), - "listed claimable local messages" - ); - let local_deadline = Instant::now() + P2P_INBOX_DRAIN_BUDGET; - let listed_full_batch = candidates.len() as i64 >= LOCAL_MESSAGE_BATCH_SIZE; - let mut local_dispatched = 0_usize; - let mut local_cut_short = false; - for (index, candidate) in candidates.into_iter().enumerate() { - if index > 0 && Instant::now() >= local_deadline { - local_cut_short = true; - break; - } - // Claim right before dispatch so a crash mid-dispatch is charged to - // this row alone, and so a producer that re-armed the row since it was - // listed wins: the stale version is skipped until the next tick. - let Some(message) = claim_local_candidate(local_db, &candidate).await else { - continue; - }; - local_dispatched += 1; - let queue_wait_secs = current_time_secs().saturating_sub(message.created_at); - let started_at = Instant::now(); - let claim = LocalMessageClaim { - message_id: message.message_id.clone(), - message_version: message.message_version, - }; - let dispatch: BoxedDispatch<'_> = Box::pin(ACTIVE_LOCAL_MESSAGE_CLAIM.scope( - claim, - recv_and_dispatch( - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor.clone(), - from_peer_id, - id.clone(), - &message.content, - metrics_state, - ), - )); - let dispatch = tokio::time::timeout(crate::env::get_p2p_handler_timeout(), dispatch); - let result = match supervise_dispatch(dispatch, shutdown).await { - DispatchExecution::Completed(Ok(result)) => result, - DispatchExecution::Completed(Err(_)) => Err(retryable_dispatch_error( - RetryableDispatchReason::ExternalRpcUnavailable, - None, - "local handler execution budget exceeded", - )), - DispatchExecution::Shutdown => return Ok(false), - DispatchExecution::Panicked(detail) => { - tracing::error!( - event = "local_message_dispatch_panic", - outcome = "node_shutdown", - queued_message_id = %message.message_id, - business_id = %message.business_id, - message_type = %message.msg_type, - detail, - "local message handler panicked; recorded an abandon and stopping the node" - ); - abandon_local_message_after_panic( - local_db, - &message.message_id, - message.message_version, - &detail, - ) - .await; - shutdown.cancel(); - bail!("local message handler panicked: {detail}"); - } - }; - match result { - Ok(_) => { - let mut storage_processor = match local_db.acquire().await { - Ok(storage_processor) => storage_processor, - Err(error) => { - log_queue_bookkeeping_failure( - "local_message_queue", - &message.message_id, - "acquire", - &error, - ); - continue; - } - }; - let state_updated = match storage_processor - .complete_local_message(&message.message_id, message.message_version) - .await - { - Ok(state_updated) => state_updated, - Err(error) => { - log_queue_bookkeeping_failure( - "local_message_queue", - &message.message_id, - "complete", - &error, - ); - continue; - } - }; - if state_updated { - tracing::info!( - event = "local_message_queue", - outcome = "processed", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "processed local message" - ); - } else { - // A row that is Pending under our claim version was - // rescheduled by the handler itself. Only now, after the - // handler returned, is that a reported outcome, so only now - // does its consecutive-abandon counter reset. - match storage_processor - .confirm_local_message_self_defer( - &message.message_id, - message.message_version, - ) - .await - { - Ok(true) => { - tracing::debug!( - event = "local_message_queue", - outcome = "self_deferred", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "local message handler rescheduled its own queue entry" - ); - } - Ok(false) => { - let current_state = storage_processor - .find_messages_by_id(&message.message_id) - .await - .ok() - .flatten() - .map(|message| message.state); - tracing::warn!( - event = "local_message_queue", - outcome = "state_update_conflict", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - current_state = ?current_state, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "local message handler completed but its processed state was not persisted" - ); - } - Err(error) => log_queue_bookkeeping_failure( - "local_message_queue", - &message.message_id, - "confirm_self_defer", - &error, - ), - } - } - } - Err(err) => { - let is_transient = is_retryable_sqlite_error(&err); - let requested_retry_delay = - p2p_retryable_dispatch_error(&err).and_then(|(_, delay)| delay); - let lock_time: i64 = requested_retry_delay.unwrap_or_else(|| { - if is_transient - && MessageKind::from_str(&message.msg_type).is_ok_and(MessageKind::is_pegin) - { - TRANSIENT_PEGIN_RETRY_DELAY_SECS as i64 - } else { - LOCAL_MESSAGE_RETRY_DELAY_SECS - } - }); - let mut storage_processor = match local_db.acquire().await { - Ok(storage_processor) => storage_processor, - Err(error) => { - log_queue_bookkeeping_failure( - "local_message_queue", - &message.message_id, - "acquire", - &error, - ); - continue; - } - }; - if let Err(reason_error) = storage_processor - .upsert_message_debug_reason( - &message.message_id, - MessageDeferReason::HandlerError.code(), - &err.to_string(), - ) - .await - { - tracing::warn!( - event = "local_message_queue", - outcome = "debug_reason_store_failed", - queued_message_id = %message.message_id, - error = %reason_error, - "failed to persist local message debug reason" - ); - } - let deferred = storage_processor - .defer_local_message( - &message.message_id, - message.message_version, - current_time_secs() + lock_time, - &err.to_string(), - ) - .await; - match deferred { - Ok(true) => {} - Ok(false) => { - // The handler may have rescheduled its own row before - // returning the error; that is still a reported outcome. - match storage_processor - .confirm_local_message_self_defer( - &message.message_id, - message.message_version, - ) - .await - { - Ok(true) => tracing::debug!( - event = "local_message_queue", - outcome = "self_deferred", - queued_message_id = %message.message_id, - error = %err, - "local message handler rescheduled its own queue entry before failing" - ), - Ok(false) => tracing::warn!( - event = "local_message_queue", - outcome = "stale_claim", - queued_message_id = %message.message_id, - message_version = message.message_version, - operation = "defer", - "ignored local message update from a stale claim" - ), - Err(error) => log_queue_bookkeeping_failure( - "local_message_queue", - &message.message_id, - "confirm_self_defer", - &error, - ), - } - continue; - } - Err(error) => { - log_queue_bookkeeping_failure( - "local_message_queue", - &message.message_id, - "defer", - &error, - ); - continue; - } - } - metrics_state.record_message_retry(); - tracing::warn!( - event = "local_message_queue", - outcome = "deferred", - role = %actor, - business_id = %message.business_id, - queued_message_id = %message.message_id, - message_type = %message.msg_type, - retry_after_secs = lock_time, - attempt_count = message.attempt_count + 1, - queue_wait_secs, - elapsed_ms = started_at.elapsed().as_millis() as u64, - error = %err, - "failed to process local message; deferred for retry" - ); - } - } - } - // Handle each queue's failure independently. - if let Err(error) = handle_p2p_outbox_messages(swarm, local_db).await { - tracing::error!(error = %error, "failed to drain the durable P2P outbox"); - } - // A queue that failed reports no backlog: asking for another pass at once - // would spin on whatever made it fail. - let local_backlog = local_dispatched > 0 && (local_cut_short || listed_full_batch); - let inbox_backlog = match handle_p2p_inbox_messages( - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor, - metrics_state, - shutdown, - ) - .await - { - Ok(backlog) => backlog, - Err(error) => { - tracing::error!(error = %error, "failed to drain the durable P2P inbox"); - if shutdown.is_cancelled() { - return Err(error); - } - false - } - }; - Ok(local_backlog || inbox_backlog) -} - -/// Filter the message and dispatch message to different handlers, like rpc handler, or other peers -/// * database: inner_rpc: Write or Read. -/// * peers: send -/// TODO: we should create a trait for all the actions of different roles to simplify this function. -#[allow(clippy::too_many_arguments)] -pub async fn recv_and_dispatch( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - btc_client: &Arc, - goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, - actor: Actor, - from_peer_id: PeerId, - id: MessageId, - message: &[u8], - metrics_state: &MetricsState, -) -> Result<()> { - let message = GOATMessage::deserialize_message(message).await?; - dispatch_decoded_p2p_message( - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - actor, - from_peer_id, - id, - message, - metrics_state, - ) - .await -} - -#[allow(clippy::too_many_arguments)] -async fn dispatch_decoded_p2p_message( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - btc_client: &Arc, - goat_client: &Arc, - http_client: &HttpAsyncClient, - soldering_builder: &Option>, - actor: Actor, - from_peer_id: PeerId, - id: MessageId, - message: GOATMessage, - metrics_state: &MetricsState, -) -> Result<()> { - // Determine whether the message comes from this node itself to optionally skip validations. - // Not `get_local_node_info()`: that signs a fresh binding on every call, and - // this runs for every dispatched message. - let is_self_peer = crate::env::get_peer_id() == from_peer_id.to_string(); - let message_type = message.content.event_type(); - let role = actor.to_string(); - let from_peer_id_string = from_peer_id.to_string(); - let started_at = Instant::now(); - let mut handler_ctx = HandlerContext { - swarm, - local_db, - btc_client, - goat_client, - http_client, - soldering_builder, - metrics_state, - actor, - from_peer_id, - id, - is_self_peer, - }; - let result = handle_dispatch(&mut handler_ctx, message.content()) - .await - .map_err(classify_retryable_dispatch_error); - metrics_state - .record_message_dispatch(message_type, if result.is_ok() { "success" } else { "failed" }); - match &result { - Ok(()) => tracing::info!( - event = "message_dispatch_result", - outcome = "handled", - role, - message_type, - from_peer_id = %from_peer_id_string, - is_self_peer, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "message dispatch completed" - ), - Err(err) => tracing::warn!( - event = "message_dispatch_result", - outcome = "failed", - role, - message_type, - from_peer_id = %from_peer_id_string, - is_self_peer, - elapsed_ms = started_at.elapsed().as_millis() as u64, - error = %err, - "message dispatch failed" - ), - } - result -} - -pub(crate) async fn try_finalize_graph( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - goat_client: &GOATClient, - instance_id: Uuid, - graph_id: Uuid, - graph: Option<&SimplifiedBitvmGcGraph>, - broadcast_graph_finalize: bool, -) -> Result> { - let endorsements = - get_committee_endorsements_for_graph(local_db, instance_id, graph_id).await?; - let params_endorsements = - get_committee_params_endorsements_for_graph(local_db, instance_id, graph_id).await?; - let pub_nonoces = get_committee_pub_nonces_for_graph(local_db, instance_id, graph_id).await?; - let partial_sigs = - get_committee_partial_sigs_for_graph(local_db, instance_id, graph_id).await?; - let committee_pubkeys = goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - if endorsements.len() == committee_pubkeys.len() - && params_endorsements.len() == committee_pubkeys.len() - && pub_nonoces.len() == committee_pubkeys.len() - && partial_sigs.len() == committee_pubkeys.len() - { - let mut graph = match graph { - Some(g) => BitvmGcGraph::from_simplified(g)?, - None => { - let g = get_graph(local_db, instance_id, graph_id) - .await? - .ok_or_else(|| anyhow!("Graph not found for {instance_id}:{graph_id}"))?; - BitvmGcGraph::from_simplified(&g)? - } - }; - if graph.parameters.instance_parameters.instance_id != instance_id - || graph.parameters.graph_id != graph_id - { - bail!( - "refuse to finalize graph {instance_id}:{graph_id} with mismatched graph parameters {}:{}", - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id - ); - } - // Reuse stored committee signatures when retrying graph finalization. - if !graph.committee_pre_signed() { - let pub_nonces = order_committee_values( - &committee_pubkeys, - pub_nonoces, - "graph committee pub nonces", - )?; - let agg_nonces = nonces_aggregation(&pub_nonces)?; - let partial_sigs = order_committee_values( - &committee_pubkeys, - partial_sigs, - "graph committee partial sigs", - )?; - let committee_sig_for_graph = - signature_aggregation(&partial_sigs, &agg_nonces, &graph)?; - push_committee_pre_signatures(&mut graph, &committee_sig_for_graph)?; - } - let simplified_graph = graph.to_simplified()?; - let store_outcome = store_finalized_graph_if_needed(local_db, &simplified_graph).await?; - mark_graph_as_endorsed(local_db, instance_id, graph_id).await?; - try_transition_instance_to_presigned(local_db, instance_id).await?; - // Throttle GraphFinalize recovery and stamp the gate only on success. - let gate = crate::p2p_admission::protocol_republish_gate(); - let gate_key = format!("graph-finalize:{graph_id}"); - let now = Instant::now(); - if broadcast_graph_finalize && !gate.is_cooling(&gate_key, now) { - let message_content = GOATMessageContent::GraphFinalize(GraphFinalize { - instance_id, - graph_id, - graph_nonce: graph.parameters.graph_nonce, - endorse_sigs: endorsements, - params_endorse_sigs: params_endorsements, - graph: simplified_graph, - }); - send_protocol_message( - swarm, - local_db, - GOATMessage::new(Actor::All, message_content), - StoredValue::Fresh, - ) - .await?; - gate.allow(&gate_key, now); - } - return Ok(Some((graph, store_outcome))); - } - Ok(None) -} - -const PROTOCOL_OUTBOX_PREFIX: &str = "protocol:"; - -/// The rounds whose messages are kept in the outbox and re-published until the -/// round is over, whatever traffic arrives or does not. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum SigningRound { - /// NonceGeneration, AggNonceConsensus, CommitteePresign and EndorseGraph: - /// what the operator needs to assemble GraphFinalize. - GraphSigning, - /// The operator's GraphFinalize, the only source of the finalized graph - /// until a relayer has posted it. - GraphFinalize, - /// The committee's PeginConfirm nonce, consensus and partial-signature rounds. - PeginConfirm, -} - -impl SigningRound { - pub(crate) fn of(content: &GOATMessageContent) -> Option { - match content { - GOATMessageContent::NonceGeneration(_) - | GOATMessageContent::AggNonceConsensus(_) - | GOATMessageContent::CommitteePresign(_) - | GOATMessageContent::EndorseGraph(_) => Some(Self::GraphSigning), - GOATMessageContent::GraphFinalize(_) => Some(Self::GraphFinalize), - GOATMessageContent::PeginConfirmNonce(_) - | GOATMessageContent::PeginConfirmNonceConsensus(_) - | GOATMessageContent::PeginConfirmPartialSig(_) => Some(Self::PeginConfirm), - _ => None, - } - } - - /// Retry ceiling; GraphFinalize uses six times the ordinary signing-round ceiling. - fn retry_ceiling_secs(self, configured_max_secs: i64) -> i64 { - match self { - Self::GraphFinalize => configured_max_secs.saturating_mul(6), - Self::GraphSigning | Self::PeginConfirm => configured_max_secs, - } - } - - /// Check delivery completion from local state; unknown statuses remain open. - /// Graph rounds close by graph status, not another operator's instance progress. - fn delivery_finished( - self, - graph: Option, - instance: Option<&InstanceBridgeInStatus>, - ) -> bool { - use InstanceBridgeInStatus as Instance; - let instance_failed = matches!( - instance, - Some( - Instance::PresignedFailed - | Instance::RelayerL2MintedFailed - | Instance::Timeout - | Instance::UserCanceled - | Instance::NoEnoughCommitteesAnswered - | Instance::UserDiscarded - ) - ); - if instance_failed { - return true; - } - match self { - Self::PeginConfirm => { - matches!(instance, Some(Instance::RelayerL1Broadcasted | Instance::RelayerL2Minted)) - } - // A finalized or posted graph proves every member's values reached - // the operator. An obsoleted one will never be posted or used. - Self::GraphSigning => graph.is_some_and(|status| match status { - GraphStatus::OperatorPresigned => false, - GraphStatus::CommitteePresigned - | GraphStatus::OperatorDataPushed - | GraphStatus::PreKickoff - | GraphStatus::OperatorKickOff - | GraphStatus::Challenge - | GraphStatus::Disprove - | GraphStatus::Obsoleted - | GraphStatus::Skipped - | GraphStatus::OperatorTake1 - | GraphStatus::OperatorTake2 => true, - }), - // Keep GraphFinalize delivery open until the graph is posted or retired. - Self::GraphFinalize => graph.is_some_and(|status| match status { - GraphStatus::OperatorPresigned | GraphStatus::CommitteePresigned => false, - GraphStatus::OperatorDataPushed - | GraphStatus::PreKickoff - | GraphStatus::OperatorKickOff - | GraphStatus::Challenge - | GraphStatus::Disprove - | GraphStatus::Obsoleted - | GraphStatus::Skipped - | GraphStatus::OperatorTake1 - | GraphStatus::OperatorTake2 => true, - }), - } - } -} - -/// [`SigningRound::delivery_finished`] over the local graph and instance rows. -/// Reads the rows only: loading the graph itself re-verifies its signatures. -pub(crate) async fn protocol_delivery_finished( - local_db: &LocalDB, - content: &GOATMessageContent, -) -> Result { - let Some(round) = SigningRound::of(content) else { - return Ok(false); - }; - let (instance_id, graph_id) = match content.business_ref() { - BusinessRef::Graph { instance_id, graph_id } => (instance_id, Some(graph_id)), - BusinessRef::Instance { instance_id } => (instance_id, None), - BusinessRef::Unscoped => return Ok(false), - }; - let mut storage = local_db.acquire().await?; - let instance = storage - .find_instance(&instance_id) - .await? - .and_then(|instance| InstanceBridgeInStatus::from_str(&instance.status).ok()); - let graph = match graph_id { - Some(graph_id) => storage - .find_graph(&graph_id) - .await? - .filter(|graph| graph.instance_id == instance_id) - .and_then(|graph| GraphStatus::from_str(&graph.status).ok()), - None => None, - }; - Ok(round.delivery_finished(graph, instance.as_ref())) -} - -/// Wait before the next re-publish of a signing-round row that has gone out of -/// the outbox `publish_count` times. -fn protocol_retry_interval_secs(publish_count: i64, max_secs: i64) -> i64 { - let doublings = publish_count.clamp(0, 20) as u32; - PROTOCOL_RETRY_BASE_SECS - .saturating_mul(1_i64 << doublings) - .min(max_secs.max(PROTOCOL_RETRY_BASE_SECS)) -} - -fn protocol_outbox_id(content: &GOATMessageContent) -> String { - format!( - "{PROTOCOL_OUTBOX_PREFIX}{}:{}", - content.event_type(), - content.business_ref().key_part() - ) -} - -/// Store a signing-round message for re-publication. Returns whether this call -/// created the row: an existing one, live or closed, is left as it is. -async fn store_protocol_message(local_db: &LocalDB, message: &GOATMessage) -> Result { - let bytes = message.serialize_message().await?; - let now = current_time_secs(); - local_db - .acquire() - .await? - .enqueue_p2p_outbox_retry_message( - &protocol_outbox_id(&message.content), - None, - message.content.event_type(), - &bytes, - now + MESSAGE_EXPIRE_TIME, - PROTOCOL_RETRY_BASE_SECS, - None, - // The caller publishes right after this; a row due at once would - // have the outbox send a second copy seconds later. - now + PROTOCOL_RETRY_BASE_SECS, - ) - .await -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum StoredValue { - Fresh, - Found, -} - -/// Persist signing-round bytes; recovery leaves live outbox rows to the worker. -pub async fn send_protocol_message( - publisher: &mut dyn MessagePublisher, - local_db: &LocalDB, - message: GOATMessage, - origin: StoredValue, -) -> Result<()> { - let outbox_id = - SigningRound::of(&message.content).map(|_| protocol_outbox_id(&message.content)); - if let Some(id) = &outbox_id { - let inserted = store_protocol_message(local_db, &message).await?; - if origin == StoredValue::Found && !inserted { - let state = local_db.acquire().await?.p2p_outbox_entry_state(id).await?; - let live = state.is_some_and(|(state, _)| state == "Pending" || state == "Processing"); - if live || protocol_delivery_finished(local_db, &message.content).await? { - return Ok(()); - } - } - } - let result = send_to_peer(publisher, message).await; - if let (Err(_), Some(outbox_id)) = (&result, outbox_id) { - // Nothing went out, so there is no copy for the outbox to duplicate: - // let it publish on its next pass instead of a full interval from now. - if let Ok(mut storage) = local_db.acquire().await { - let _ = storage.expedite_p2p_outbox_message(&outbox_id).await; - } - } - result.map(|_| ()) -} - -pub async fn send_to_peer( - swarm: &mut dyn MessagePublisher, - message: GOATMessage, -) -> Result { - let target_actor = message.actor.to_string(); - let message_type = message.content.event_type(); - let topic = crate::middleware::get_topic_name(&target_actor); - let gossipsub_topic = gossipsub::IdentTopic::new(topic); - let serialized = match message.serialize_message().await { - Ok(serialized) => serialized, - Err(error) => { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_p2p_publish(false); - } - return Err(error); - } - }; - // Receivers drop what exceeds their envelope limit without telling anyone, - // so an outgrown message has to be loud on the sending side. - let limits = crate::p2p_admission::inbound_limits(); - let receiver_limit = if GOATMessage::is_binary_envelope(&serialized) { - limits.max_binary_bytes - } else { - limits.max_json_bytes - }; - if serialized.len() > receiver_limit { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_p2p_oversized_message(); - } - tracing::error!( - event = "p2p_message_publish", - outcome = "oversized", - target_actor, - message_type, - content_size = serialized.len(), - receiver_limit, - "outbound protocol message exceeds the inbound size limit peers enforce" - ); - } - match swarm.publish_message(gossipsub_topic, serialized).await { - Ok(message_id) => { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_p2p_publish(true); - } - tracing::info!( - event = "p2p_message_publish", - outcome = "published", - target_actor, - message_type, - message_id = ?message_id, - "published protocol message" - ); - Ok(message_id) - } - Err(err) => { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_p2p_publish(false); - } - tracing::warn!( - event = "p2p_message_publish", - outcome = "failed", - target_actor, - message_type, - error = %err, - "failed to publish protocol message" - ); - Err(retryable_dispatch_error( - RetryableDispatchReason::PublishFailed, - Some(10), - format!("publish {message_type} to {target_actor}: {err}"), - )) - } - } -} - -/// Defer the message currently being dispatched, or enqueue a new local task. -pub async fn defer_or_enqueue_message( - local_db: &LocalDB, - message: &GOATMessage, - delay_secs: usize, - reason: MessageDeferReason, - reason_detail: &str, -) -> Result<()> { - if defer_current_inbox(message, delay_secs, reason_detail) { - return Ok(()); - } - - let actor = message.actor.clone(); - let content: GOATMessageContent = message.content().clone(); - let key = LocalMessageKey::from_content(actor.clone(), &content)?; - let business_id = key.business_id(); - let message_type = message.content.event_type(); - let target_message_id = key.message_id(); - let active_claim = ACTIVE_LOCAL_MESSAGE_CLAIM - .try_with(|claim| (claim.message_id.clone(), claim.message_version)) - .ok(); - let mut storage_processor = local_db.start_immediate_transaction().await?; - - let queued_message_id = if let Some((message_id, message_version)) = - active_claim.filter(|(message_id, _)| message_id == &target_message_id) - { - let deferred = storage_processor - .self_defer_local_message( - &message_id, - message_version, - current_time_secs() + delay_secs as i64, - reason_detail, - ) - .await?; - if !deferred { - return Err(retryable_dispatch_error( - RetryableDispatchReason::ResourceLocked, - Some(delay_secs.max(1) as i64), - format!( - "local message {business_id}:{message_type} is no longer owned by this claim" - ), - )); - } - message_id - } else { - let upserted = upsert_message( - &mut storage_processor, - true, - SELF_SENDER.to_string(), - actor, - content, - 0, - delay_secs as i64, - ) - .await?; - if !upserted { - let current = storage_processor.find_messages_by_id(&target_message_id).await?; - if current - .as_ref() - .is_some_and(|message| message.state == MessageState::Processing.to_string()) - { - return Err(retryable_dispatch_error( - RetryableDispatchReason::ResourceLocked, - Some(delay_secs.max(1) as i64), - format!( - "local message {business_id}:{message_type} is owned by another active claim" - ), - )); - } - } - target_message_id - }; - let persist_result = storage_processor - .upsert_message_debug_reason(&queued_message_id, reason.code(), reason_detail) - .await; - if let Err(error) = persist_result { - tracing::warn!( - event = "local_message_queue", - outcome = "debug_reason_store_failed", - error = %error, - "failed to persist local message defer reason" - ); - } - storage_processor.commit().await?; - if delay_secs > 0 - && let Some(metrics_state) = crate::metrics_service::node_metrics_state() - { - metrics_state.record_message_retry(); - } - Ok(()) -} - -fn defer_current_inbox(message: &GOATMessage, delay_secs: usize, reason_detail: &str) -> bool { - ACTIVE_INBOX_DISPATCH - .try_with(|active| { - if DispatchFingerprint::of(message.content()) != active.fingerprint { - return false; - } - *active.retry.borrow_mut() = Some(retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(delay_secs.max(1) as i64), - reason_detail.to_owned(), - )); - true - }) - .unwrap_or(false) -} - -/// Helper: try to get graph. If missing, send SyncGraphRequest and defer current handling. -pub(crate) async fn get_graph_or_defer( - swarm: &mut dyn MessagePublisher, - local_db: &LocalDB, - goat_client: &GOATClient, - instance_id: Uuid, - graph_id: Uuid, - message: &GOATMessage, -) -> Result> { - match get_graph(local_db, instance_id, graph_id).await? { - Some(g) => Ok(Some(g)), - None => { - // Ask for sync and push to local queue with a short retry delay - let sync_request_outcome = if let Err(error) = - try_send_sync_graph_request(swarm, goat_client, instance_id, graph_id).await - { - tracing::warn!( - event = "graph_resolution", - outcome = "sync_request_failed", - instance_id = %instance_id, - graph_id = %graph_id, - message_type = message.content.event_type(), - error_class = "p2p", - error = %error, - "failed to request graph synchronization" - ); - "failed" - } else { - "submitted" - }; - let delay_secs: usize = 60; // 1 min default retry - if let Err(error) = defer_or_enqueue_message( - local_db, - message, - delay_secs, - MessageDeferReason::GraphSyncPending, - "graph is missing locally; requested SyncGraph from a relayer", - ) - .await - { - tracing::error!( - event = "graph_resolution", - outcome = "defer_failed", - instance_id = %instance_id, - graph_id = %graph_id, - message_type = message.content.event_type(), - retry_after_secs = delay_secs, - error_class = "database", - error = %error, - "failed to enqueue message while graph is missing" - ); - return Err(error).context("failed to defer message while graph is missing"); - } - tracing::info!( - event = "graph_resolution", - outcome = "deferred_missing_graph", - instance_id = %instance_id, - graph_id = %graph_id, - message_type = message.content.event_type(), - sync_request_outcome, - retry_after_secs = delay_secs, - "graph missing locally; requested sync and deferred message" - ); - Ok(None) - } - } -} - -pub async fn try_send_sync_graph_request( - swarm: &mut dyn MessagePublisher, - goat_client: &GOATClient, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - validate_graph_id_on_goat(goat_client, instance_id, graph_id).await?; - // Record the graph request before publishing it. - if !crate::p2p_admission::requested_graphs().insert(&graph_id.to_string(), Instant::now()) { - bail!( - "too many graph sync requests outstanding; {instance_id}:{graph_id} is retried later" - ); - } - let message_content = - GOATMessageContent::SyncGraphRequest(SyncGraphRequest { instance_id, graph_id }); - let message = GOATMessage::new(Actor::All, message_content); - send_to_peer(swarm, message).await?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - struct UnavailablePublisher; - impl MessagePublisher for UnavailablePublisher { - fn publish_message( - &mut self, - _: gossipsub::IdentTopic, - _: Vec, - ) -> futures::future::BoxFuture<'_, Result> { - Box::pin(async { bail!("network unavailable") }) - } - } - - #[tokio::test] - async fn signing_round_survives_publish_failure_without_reopening_completed_delivery() { - let db = store::create_local_db("sqlite::memory:").await; - let key = Keypair::from_secret_key( - SECP256K1, - &secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), - ); - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::AggNonceConsensus(AggNonceConsensus { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - committee_pubkey: key.public_key().into(), - consensus_hash: [3; 32], - signature: SECP256K1.sign_schnorr(&SecpMessage::from_digest([3; 32]), &key), - }), - ); - assert!( - send_protocol_message( - &mut UnavailablePublisher, - &db, - message.clone(), - StoredValue::Fresh - ) - .await - .is_err() - ); - let now = current_time_secs(); - let mut storage = db.acquire().await.unwrap(); - let rows = storage.claim_p2p_outbox_messages(now, now + 300, 10).await.unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].content, message.serialize_message().await.unwrap()); - assert_eq!(rows[0].retry_interval_secs, 30); - storage.complete_p2p_outbox_message(&rows[0].message_id).await.unwrap(); - drop(storage); - let _ = send_protocol_message(&mut UnavailablePublisher, &db, message, StoredValue::Fresh) - .await; - assert!( - db.acquire() - .await - .unwrap() - .claim_p2p_outbox_messages(now + 301, now + 601, 10) - .await - .unwrap() - .is_empty() - ); - } - - #[derive(Default)] - struct RecordingPublisher(Vec>); - impl MessagePublisher for RecordingPublisher { - fn publish_message( - &mut self, - _: gossipsub::IdentTopic, - data: Vec, - ) -> futures::future::BoxFuture<'_, Result> { - self.0.push(data); - Box::pin(async { Ok(MessageId::from("published")) }) - } - } - - async fn queued_outbox_fixture() -> LocalDB { - let db = store::create_local_db("sqlite::memory:").await; - let content = - committee_vote(Uuid::new_v4(), Uuid::new_v4()).serialize_message().await.unwrap(); - for index in 0..3 { - db.acquire() - .await - .unwrap() - .enqueue_p2p_outbox_message( - &format!("budget-{index}"), - "AggNonceConsensus", - &content, - ) - .await - .unwrap(); - } - db - } - - async fn assert_unclaimed_outbox_rows(db: &LocalDB, expected: usize) { - let now = current_time_secs(); - let rows = db - .acquire() - .await - .unwrap() - .claim_p2p_outbox_messages(now, now + 300, 16) - .await - .unwrap(); - assert_eq!(rows.len(), expected, "untouched rows must remain immediately claimable"); - assert!(rows.iter().all(|row| row.attempt_count == 1)); - } - - #[tokio::test] - async fn outbox_expired_budget_claims_nothing() { - let db = queued_outbox_fixture().await; - let mut publisher = RecordingPublisher::default(); - drain_p2p_outbox(&mut publisher, &db, Instant::now()).await.unwrap(); - assert!(publisher.0.is_empty()); - assert_unclaimed_outbox_rows(&db, 3).await; - } - - #[tokio::test] - async fn outbox_publish_failure_does_not_claim_the_rest() { - let db = queued_outbox_fixture().await; - drain_p2p_outbox(&mut UnavailablePublisher, &db, Instant::now() + Duration::from_secs(60)) - .await - .unwrap(); - assert_unclaimed_outbox_rows(&db, 2).await; - } - - #[tokio::test] - async fn outbox_checks_budget_between_publishes() { - struct DeadlinePublisher(Instant); - impl MessagePublisher for DeadlinePublisher { - fn publish_message( - &mut self, - _: gossipsub::IdentTopic, - _: Vec, - ) -> futures::future::BoxFuture<'_, Result> { - Box::pin(async move { - tokio::time::sleep_until(tokio::time::Instant::from_std(self.0)).await; - Ok(MessageId::from("published")) - }) - } - } - let db = queued_outbox_fixture().await; - let deadline = Instant::now() + Duration::from_secs(2); - drain_p2p_outbox(&mut DeadlinePublisher(deadline), &db, deadline).await.unwrap(); - assert_unclaimed_outbox_rows(&db, 2).await; - } - - fn committee_vote(instance_id: Uuid, graph_id: Uuid) -> GOATMessage { - let key = Keypair::from_secret_key( - SECP256K1, - &secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), - ); - GOATMessage::new( - Actor::Committee, - GOATMessageContent::AggNonceConsensus(AggNonceConsensus { - instance_id, - graph_id, - committee_pubkey: key.public_key().into(), - consensus_hash: [3; 32], - signature: SECP256K1.sign_schnorr(&SecpMessage::from_digest([3; 32]), &key), - }), - ) - } - - async fn store_graph_row(db: &LocalDB, instance_id: Uuid, graph_id: Uuid, status: GraphStatus) { - let mut storage = db.acquire().await.unwrap(); - storage - .upsert_graph_definition(&store::Graph { - graph_id, - instance_id, - status: GraphStatus::OperatorPresigned.to_string(), - definition_hash: "definition".to_owned(), - ..Default::default() - }) - .await - .unwrap(); - if status != GraphStatus::OperatorPresigned { - storage - .transition_graph_status( - instance_id, - graph_id, - status, - store::GraphStatusSource::Definition, - None, - ) - .await - .unwrap(); - } - } - - #[test] - fn signing_round_backoff_doubles_up_to_its_ceiling() { - let waits: Vec = (0..8).map(|n| protocol_retry_interval_secs(n, 600)).collect(); - assert_eq!(waits, [30, 60, 120, 240, 480, 600, 600, 600]); - // A short-timelock network asks for a ceiling below the base step. - assert_eq!(protocol_retry_interval_secs(5, 10), 30); - assert_eq!(protocol_retry_interval_secs(i64::MAX, 600), 600); - assert_eq!(protocol_retry_interval_secs(-1, 600), 30); - // The whole graph backs off further than the small round messages. - assert_eq!(SigningRound::GraphSigning.retry_ceiling_secs(600), 600); - assert_eq!(SigningRound::GraphFinalize.retry_ceiling_secs(600), 3_600); - assert_eq!(protocol_retry_interval_secs(9, 3_600), 3_600); - } - - #[test] - fn a_graph_round_is_closed_by_its_graph_and_not_by_the_instance() { - use GraphStatus as G; - use InstanceBridgeInStatus as I; - // The instance moves on with the first graph; a second operator's graph - // is still mid-round then, and must keep its retries. - for instance in [I::Presigned, I::RelayerL1Broadcasted, I::RelayerL2Minted] { - for round in [SigningRound::GraphSigning, SigningRound::GraphFinalize] { - assert!(!round.delivery_finished(Some(G::OperatorPresigned), Some(&instance))); - assert!(!round.delivery_finished(None, Some(&instance))); - } - } - // This node holding the finalized graph closes its own round values, but - // says nothing about whether a relayer has the GraphFinalize. - assert!(SigningRound::GraphSigning.delivery_finished(Some(G::CommitteePresigned), None)); - assert!(!SigningRound::GraphFinalize.delivery_finished(Some(G::CommitteePresigned), None)); - for posted in [G::OperatorDataPushed, G::PreKickoff, G::OperatorTake1, G::Obsoleted] { - assert!(SigningRound::GraphSigning.delivery_finished(Some(posted), None)); - assert!(SigningRound::GraphFinalize.delivery_finished(Some(posted), None)); - } - // PeginConfirm is the instance's round. - assert!(!SigningRound::PeginConfirm.delivery_finished(None, Some(&I::Presigned))); - assert!(!SigningRound::PeginConfirm.delivery_finished(None, None)); - assert!(SigningRound::PeginConfirm.delivery_finished(None, Some(&I::RelayerL1Broadcasted))); - // A failed instance closes everything. - for failed in [I::PresignedFailed, I::Timeout, I::UserCanceled, I::UserDiscarded] { - assert!( - SigningRound::GraphSigning - .delivery_finished(Some(G::OperatorPresigned), Some(&failed)) - ); - assert!(SigningRound::GraphFinalize.delivery_finished(None, Some(&failed))); - assert!(SigningRound::PeginConfirm.delivery_finished(None, Some(&failed))); - } - } - - #[tokio::test] - async fn a_published_signing_message_is_not_sent_again_until_its_first_interval() { - let db = store::create_local_db("sqlite::memory:").await; - let message = committee_vote(Uuid::new_v4(), Uuid::new_v4()); - let mut publisher = RecordingPublisher::default(); - send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Fresh) - .await - .unwrap(); - assert_eq!(publisher.0.len(), 1); - - // The handler has just published: the outbox must not follow with a copy. - handle_p2p_outbox_messages(&mut publisher, &db).await.unwrap(); - assert_eq!(publisher.0.len(), 1); - - // A later visit that finds the value stored leaves the live row alone... - send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Found) - .await - .unwrap(); - assert_eq!(publisher.0.len(), 1); - - // ...and the outbox re-publishes it on schedule, backing off. - let now = current_time_secs(); - let mut storage = db.acquire().await.unwrap(); - let due = storage - .claim_p2p_outbox_messages(now + PROTOCOL_RETRY_BASE_SECS, now + 600, 10) - .await - .unwrap(); - assert_eq!(due.len(), 1); - assert_eq!(due[0].publish_count, 0); - storage.schedule_p2p_outbox_retry(&due[0].message_id, now + 1_000).await.unwrap(); - let due = storage.claim_p2p_outbox_messages(now + 1_000, now + 2_000, 10).await.unwrap(); - assert_eq!(due[0].publish_count, 1, "only publishes that went through are counted"); - } - - #[tokio::test] - async fn a_value_stored_without_an_outbox_row_is_published_by_the_retry() { - // The first run stored its value and was cut off before the outbox row: - // nothing was ever published, and the retry is the only one who can. - let db = store::create_local_db("sqlite::memory:").await; - let message = committee_vote(Uuid::new_v4(), Uuid::new_v4()); - let mut publisher = RecordingPublisher::default(); - send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Found) - .await - .unwrap(); - assert_eq!(publisher.0.len(), 1); - let id = protocol_outbox_id(&message.content); - let state = db.acquire().await.unwrap().p2p_outbox_entry_state(&id).await.unwrap(); - assert_eq!(state.unwrap().0, "Pending"); - - // A failed publish leaves the row due at once rather than an interval away. - let other = committee_vote(Uuid::new_v4(), Uuid::new_v4()); - assert!( - send_protocol_message(&mut UnavailablePublisher, &db, other, StoredValue::Found) - .await - .is_err() - ); - let now = current_time_secs(); - let due = db - .acquire() - .await - .unwrap() - .claim_p2p_outbox_messages(now, now + 300, 10) - .await - .unwrap(); - assert_eq!(due.len(), 1); - } - - #[tokio::test] - async fn a_closed_outbox_row_is_published_past_only_while_the_round_is_open() { - let db = store::create_local_db("sqlite::memory:").await; - let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); - store_graph_row(&db, instance_id, graph_id, GraphStatus::OperatorPresigned).await; - let message = committee_vote(instance_id, graph_id); - assert!( - send_protocol_message( - &mut UnavailablePublisher, - &db, - message.clone(), - StoredValue::Fresh - ) - .await - .is_err() - ); - let now = current_time_secs(); - let mut storage = db.acquire().await.unwrap(); - let rows = storage.claim_p2p_outbox_messages(now, now + 300, 10).await.unwrap(); - storage.fail_p2p_outbox_message(&rows[0].message_id, "corrupt").await.unwrap(); - drop(storage); - - // The row is gone for good, the round is not over: silence would strand it. - let mut publisher = RecordingPublisher::default(); - send_protocol_message(&mut publisher, &db, message.clone(), StoredValue::Found) - .await - .unwrap(); - assert_eq!(publisher.0.len(), 1); - - // Once the graph is finalized here, nobody needs the vote any more. - store_graph_row(&db, instance_id, graph_id, GraphStatus::CommitteePresigned).await; - send_protocol_message(&mut publisher, &db, message, StoredValue::Found).await.unwrap(); - assert_eq!(publisher.0.len(), 1); - } - - #[tokio::test] - async fn the_outbox_stops_a_round_by_its_graph_and_survives_a_bad_row() { - let db = store::create_local_db("sqlite::memory:").await; - let (instance_id, open_graph, finalized_graph) = - (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); - store_graph_row(&db, instance_id, open_graph, GraphStatus::OperatorPresigned).await; - store_graph_row(&db, instance_id, finalized_graph, GraphStatus::CommitteePresigned).await; - // A graph row of another instance must not close this one's round. - let foreign_graph = Uuid::new_v4(); - store_graph_row(&db, Uuid::new_v4(), foreign_graph, GraphStatus::CommitteePresigned).await; - for graph_id in [open_graph, finalized_graph, foreign_graph] { - let _ = send_protocol_message( - &mut UnavailablePublisher, - &db, - committee_vote(instance_id, graph_id), - StoredValue::Fresh, - ) - .await; - } - // A corrupt row sits first in the batch; the rows behind it are still served. - let now = current_time_secs(); - db.acquire() - .await - .unwrap() - .enqueue_p2p_outbox_retry_message( - "protocol:AggNonceConsensus:graph:corrupt", - None, - "AggNonceConsensus", - b"not a message", - now + 600, - 30, - None, - 0, - ) - .await - .unwrap(); - - let mut publisher = RecordingPublisher::default(); - handle_p2p_outbox_messages(&mut publisher, &db).await.unwrap(); - assert_eq!(publisher.0.len(), 2, "the open round and the foreign-row round publish"); - - let mut storage = db.acquire().await.unwrap(); - let state = - |graph_id: Uuid| protocol_outbox_id(&committee_vote(instance_id, graph_id).content); - let open = storage.p2p_outbox_entry_state(&state(open_graph)).await.unwrap().unwrap(); - assert_eq!(open.0, "Pending"); - let closed = - storage.p2p_outbox_entry_state(&state(finalized_graph)).await.unwrap().unwrap(); - assert_eq!(closed, ("Processed".to_owned(), 0)); - let foreign = storage.p2p_outbox_entry_state(&state(foreign_graph)).await.unwrap().unwrap(); - assert_eq!(foreign.0, "Pending"); - } - - #[test] - fn inbox_retention_sweeps_keep_the_tick_cadence_under_backlog() { - let start = Instant::now(); - // Whatever an earlier test left behind, two calls inside one interval - // cannot both be due. - let first = inbox_housekeeping_due(start); - let second = inbox_housekeeping_due(start + Duration::from_secs(1)); - assert!(!(first && second)); - assert!(!second); - let interval = Duration::from_secs(crate::env::REGULAR_TASK_INTERVAL_SECOND); - assert!(inbox_housekeeping_due(start + interval + Duration::from_secs(2))); - } - - #[tokio::test] - async fn inbox_dependency_retry_never_collapses_into_local_queue() { - let local_db = store::create_local_db("sqlite::memory:").await; - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::SyncGraphRequest(SyncGraphRequest { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - }), - ); - let mut compensated = false; - let result = track_inbox_retry(DispatchFingerprint::of(message.content()), async { - defer_or_enqueue_message( - &local_db, - &message, - 30, - MessageDeferReason::PreviousGraphPending, - "wait for graph", - ) - .await?; - // Some handlers rebroadcast a prerequisite only after asking - // for deferral. An early Err would skip that recovery action. - compensated = true; - Ok(()) - }) - .await; - assert!(compensated); - assert!(result.unwrap_err().downcast_ref::().is_some()); - let key = LocalMessageKey::from_content(Actor::Committee, message.content()).unwrap(); - assert!( - local_db - .acquire() - .await - .unwrap() - .find_messages_by_id(&key.message_id()) - .await - .unwrap() - .is_none() - ); - } - - /// Verify self-deferral survives later handler errors; other events still use the local queue. - #[tokio::test] - async fn deferral_survives_a_later_error_and_other_events_stay_local() { - let local_db = store::create_local_db("sqlite::memory:").await; - let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); - let dispatched = GOATMessage::new( - Actor::Committee, - GOATMessageContent::SyncGraphRequest(SyncGraphRequest { instance_id, graph_id }), - ); - // The handler builds its own copy, under its own role: same message. - let own_copy = GOATMessage::new(Actor::Operator, dispatched.content().clone()); - let other_event = GOATMessage::new( - Actor::Committee, - GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), - ); - - let result = track_inbox_retry(DispatchFingerprint::of(dispatched.content()), async { - defer_or_enqueue_message( - &local_db, - &other_event, - 30, - MessageDeferReason::ChainStatePending, - "a different event", - ) - .await?; - defer_or_enqueue_message( - &local_db, - &own_copy, - 45, - MessageDeferReason::PreviousGraphPending, - "wait for graph", - ) - .await?; - bail!("compensation failed after the deferral was recorded") - }) - .await; - let retry = result.unwrap_err(); - let retry = retry - .downcast_ref::() - .expect("the recorded deferral wins over the later, non-retryable error"); - assert_eq!(retry.retry_after_secs, Some(45)); - - let mut storage = local_db.acquire().await.unwrap(); - let own_key = LocalMessageKey::from_content(Actor::Operator, own_copy.content()).unwrap(); - assert!(storage.find_messages_by_id(&own_key.message_id()).await.unwrap().is_none()); - let other_key = - LocalMessageKey::from_content(Actor::Committee, other_event.content()).unwrap(); - assert!( - storage.find_messages_by_id(&other_key.message_id()).await.unwrap().is_some(), - "an event of another kind is not the dispatched message" - ); - } - - /// Verify handler timeout, completion and shutdown outcomes. - #[tokio::test] - async fn handler_budget_cuts_off_an_overrunning_dispatch() { - let shutdown = CancellationToken::new(); - let budget = Duration::from_millis(100); - - let overrunning = async { - tokio::time::sleep(Duration::from_secs(3600)).await; - Ok::<(), anyhow::Error>(()) - }; - let outcome = - supervise_dispatch(tokio::time::timeout(budget, overrunning), &shutdown).await; - assert!(matches!(outcome, DispatchExecution::Completed(Err(_))), "cut off at the budget"); - - let prompt = async { - tokio::time::sleep(Duration::from_millis(1)).await; - Ok::<(), anyhow::Error>(()) - }; - let outcome = supervise_dispatch(tokio::time::timeout(budget, prompt), &shutdown).await; - assert!(matches!(outcome, DispatchExecution::Completed(Ok(Ok(()))))); - - shutdown.cancel(); - let outcome = supervise_dispatch( - tokio::time::timeout(budget, std::future::pending::>()), - &shutdown, - ) - .await; - assert!(matches!(outcome, DispatchExecution::Shutdown)); - } - - #[test] - fn soldering_proof_ready_is_descriptor_only() { - let ready = SolderingProofReady { - instance_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), - graph_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), - candidate_index: 3, - payload_hash: [0xabu8; 32], - total_len: 1024, - }; - - let value = serde_json::to_value(ready).unwrap(); - let object = value.as_object().unwrap(); - - assert!(object.contains_key("instance_id")); - assert!(object.contains_key("graph_id")); - assert!(object.contains_key("candidate_index")); - assert!(object.contains_key("payload_hash")); - assert!(object.contains_key("total_len")); - assert!(!object.contains_key("payload_path")); - assert!(!object.contains_key("payload")); - assert!(!object.contains_key("setup_package")); - assert!(!object.contains_key("verifier_pubkey")); - } - - #[tokio::test] - async fn genuine_transient_errors_are_still_retryable() { - let error = anyhow!("database is locked"); - assert!( - p2p_retryable_dispatch_error(&error).is_some(), - "a real SQLite-busy error must remain retryable" - ); - } - - #[test] - fn contract_revert_is_not_a_retryable_transport_failure() { - let response = - serde_json::from_str(r#"{"code":3,"message":"execution reverted"}"#).unwrap(); - let error = anyhow::Error::new(alloy::contract::Error::TransportError( - alloy::transports::TransportError::ErrorResp(response), - )) - .context("validate SyncGraph instance parameters"); - assert!(!is_retryable_external_rpc_error(&error)); - assert!( - classify_retryable_dispatch_error(error) - .downcast_ref::() - .is_none() - ); - } - - #[tokio::test] - async fn dispatch_supervisor_distinguishes_shutdown_and_panic() { - let shutdown = CancellationToken::new(); - shutdown.cancel(); - assert!(matches!( - supervise_dispatch(std::future::pending::<()>(), &shutdown).await, - DispatchExecution::Shutdown - )); - - let running = CancellationToken::new(); - let result = supervise_dispatch( - async { - panic!("poison message"); - }, - &running, - ) - .await; - assert!(matches!( - result, - DispatchExecution::Panicked(detail) if detail == "poison message" - )); - } - - #[tokio::test] - async fn non_owner_requeue_reports_processing_conflict() { - let local_db = store::create_local_db("sqlite::memory:").await; - let instance_id = Uuid::new_v4(); - let message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - defer_or_enqueue_message( - &local_db, - &message, - 0, - MessageDeferReason::HandlerError, - "initial", - ) - .await - .unwrap(); - - let claimed = { - let mut storage = local_db.acquire().await.unwrap(); - storage - .claim_local_messages( - current_time_secs() + 1, - current_time_secs() + 300, - 0, - 1, - QUEUE_MAX_ABANDONS, - ) - .await - .unwrap() - }; - assert_eq!(claimed.len(), 1); - - let error = defer_or_enqueue_message( - &local_db, - &message, - 30, - MessageDeferReason::HandlerError, - "retry", - ) - .await - .unwrap_err(); - let retryable = error - .chain() - .find_map(|cause| cause.downcast_ref::()) - .expect("Processing conflict must be retryable"); - assert_eq!(retryable.reason, RetryableDispatchReason::ResourceLocked); - assert_eq!(retryable.retry_after_secs, Some(30)); - - let mut storage = local_db.acquire().await.unwrap(); - let stored = storage.find_messages_by_id(&claimed[0].message_id).await.unwrap().unwrap(); - assert_eq!(stored.state, MessageState::Processing.to_string()); - assert_eq!(stored.message_version, claimed[0].message_version); - } - - #[tokio::test] - async fn owner_requeue_uses_content_derived_message_id() { - let local_db = store::create_local_db("sqlite::memory:").await; - let instance_id = Uuid::new_v4(); - let message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - let message_id = LocalMessageKey::from_content(message.actor.clone(), &message.content) - .unwrap() - .message_id(); - { - let mut storage = local_db.acquire().await.unwrap(); - assert!( - upsert_message( - &mut storage, - false, - SELF_SENDER.to_owned(), - message.actor.clone(), - message.content.clone(), - 0, - 0, - ) - .await - .unwrap() - ); - } - let claimed = { - let mut storage = local_db.acquire().await.unwrap(); - storage - .claim_local_messages( - current_time_secs() + 1, - current_time_secs() + 300, - 0, - 1, - QUEUE_MAX_ABANDONS, - ) - .await - .unwrap() - .pop() - .unwrap() - }; - assert_eq!(claimed.message_id, message_id); - - // Two earlier attempts died mid-dispatch; the row carries their abandons - // into this claim, and the handler's own reschedule must not erase them. - { - let mut storage = local_db.acquire().await.unwrap(); - for _ in 0..2 { - assert!( - storage - .abandon_local_message( - &message_id, - claimed.message_version, - current_time_secs(), - 0, - "died mid-dispatch", - ) - .await - .unwrap() - ); - } - } - let claimed = { - let mut storage = local_db.acquire().await.unwrap(); - storage - .claim_local_messages( - current_time_secs() + 301, - current_time_secs() + 600, - 0, - 1, - QUEUE_MAX_ABANDONS, - ) - .await - .unwrap() - .pop() - .unwrap() - }; - assert_eq!(claimed.abandon_count, 2); - - ACTIVE_LOCAL_MESSAGE_CLAIM - .scope( - LocalMessageClaim { - message_id: claimed.message_id.clone(), - message_version: claimed.message_version, - }, - defer_or_enqueue_message( - &local_db, - &message, - 30, - MessageDeferReason::HandlerError, - "retry subtyped message", - ), - ) - .await - .unwrap(); - - let mut storage = local_db.acquire().await.unwrap(); - let stored = storage.find_messages_by_id(&message_id).await.unwrap().unwrap(); - assert_eq!(stored.state, MessageState::Pending.to_string()); - assert_eq!(stored.abandon_count, 2, "a self-defer must keep the abandon count"); - - // The dispatcher confirms the self-defer once the handler has returned. - assert!( - storage - .confirm_local_message_self_defer(&message_id, claimed.message_version) - .await - .unwrap() - ); - assert_eq!( - storage.find_messages_by_id(&message_id).await.unwrap().unwrap().abandon_count, - 0 - ); - } - - #[tokio::test] - async fn local_claim_is_not_deferred_by_another_actor_message() { - let local_db = store::create_local_db("sqlite::memory:").await; - let instance_id = Uuid::new_v4(); - let claimed_message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - let other_actor_message = - GOATMessage::new(Actor::Committee, claimed_message.content.clone()); - let claimed_id = - LocalMessageKey::from_content(claimed_message.actor.clone(), claimed_message.content()) - .unwrap() - .message_id(); - let other_id = LocalMessageKey::from_content( - other_actor_message.actor.clone(), - other_actor_message.content(), - ) - .unwrap() - .message_id(); - { - let mut storage = local_db.acquire().await.unwrap(); - upsert_message( - &mut storage, - false, - SELF_SENDER.to_owned(), - claimed_message.actor.clone(), - claimed_message.content.clone(), - 0, - 0, - ) - .await - .unwrap(); - } - let claimed = local_db - .acquire() - .await - .unwrap() - .claim_local_messages( - current_time_secs() + 1, - current_time_secs() + 300, - 0, - 1, - QUEUE_MAX_ABANDONS, - ) - .await - .unwrap() - .pop() - .unwrap(); - - ACTIVE_LOCAL_MESSAGE_CLAIM - .scope( - LocalMessageClaim { - message_id: claimed.message_id.clone(), - message_version: claimed.message_version, - }, - defer_or_enqueue_message( - &local_db, - &other_actor_message, - 30, - MessageDeferReason::HandlerError, - "other actor", - ), - ) - .await - .unwrap(); - - let mut storage = local_db.acquire().await.unwrap(); - assert_eq!( - storage.find_messages_by_id(&claimed_id).await.unwrap().unwrap().state, - MessageState::Processing.to_string() - ); - assert_eq!( - storage.find_messages_by_id(&other_id).await.unwrap().unwrap().state, - MessageState::Pending.to_string() - ); - } -} diff --git a/node/src/aggregates/common.rs b/node/src/aggregates/common.rs new file mode 100644 index 000000000..6fd908230 --- /dev/null +++ b/node/src/aggregates/common.rs @@ -0,0 +1,259 @@ +//! What the Instance and Graph aggregates share: the messages between the +//! network, the applies and the jobs, the membership probe and scan marks. + +use super::{graph, instance}; +use crate::messages::{BusinessRef, GOATMessage, GOATMessageContent, HasBusinessRef}; +use crate::utils::committee_instance_keys_envelope_path; +use bitvm_lib::actors::Actor; +use engine::{ApplyContext, EventSource, NewEvent}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashMap}; +use std::str::FromStr; +use std::sync::{Mutex, MutexGuard}; +use store::InstanceBridgeInStatus; +use uuid::Uuid; + +/// How long a message may wait for a definition that never arrives before +/// it is dropped; the instance deadline normally ends the wait first. +pub const PARKED_MESSAGE_TTL_SECS: i64 = 2 * 24 * 60 * 60; + +// ----------------------------------------------------------------- messages + +/// A network message handed to the aggregate: the raw wire bytes plus the +/// identity the admission layer already decoded. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct InboundMessage { + pub author: String, + pub instance_id: Uuid, + pub graph_nonce: Option, + pub operator_pubkey: Option, + pub message: Vec, +} + +impl InboundMessage { + pub fn decode(bytes: &[u8]) -> anyhow::Result { + Ok(bincode::deserialize(bytes)?) + } + + pub fn encode(&self) -> anyhow::Result> { + Ok(bincode::serialize(self)?) + } +} + +/// A message the apply writes to the outbox. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlannedMessage { + pub outbox_id: String, + pub phase_ref: String, + pub kind: String, + /// Target role topic. + pub actor: String, + pub payload: Vec, + /// Heavy message: recipient peer id -> delivery id. + pub ack_set: Option>, +} + +/// A follow-up job the apply enqueues once the plan is committed. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NextJob { + pub action: String, + pub params: Vec, + pub delay_secs: Option, +} + +/// Delivery id of a heavy message: `H("goat/graph-setup-delivery/v2", sender, +/// recipient, SHA-256(payload))`, the same on both ends. +pub fn delivery_id(sender_peer: &str, recipient_peer: &str, payload: &[u8]) -> String { + let payload_hash: [u8; 32] = Sha256::digest(payload).into(); + let mut hasher = Sha256::new(); + hasher.update(b"goat/graph-setup-delivery/v2"); + for field in [sender_peer.as_bytes(), recipient_peer.as_bytes(), payload_hash.as_slice()] { + hasher.update((field.len() as u64).to_be_bytes()); + hasher.update(field); + } + format!("graph-setup-v2:{}", hex::encode(hasher.finalize())) +} + +/// Content id of a network message: `H(author ‖ kind ‖ business_ref ‖ H(payload))`. +pub fn content_event_id(author: &str, kind: &str, business: &str, payload: &[u8]) -> String { + let payload_hash: [u8; 32] = Sha256::digest(payload).into(); + let mut hasher = Sha256::new(); + for field in [author.as_bytes(), kind.as_bytes(), business.as_bytes(), payload_hash.as_slice()] + { + hasher.update((field.len() as u64).to_be_bytes()); + hasher.update(field); + } + format!("msg:{kind}:{}", hex::encode(hasher.finalize())) +} + +/// The inbox event for an admitted network message: a graph-phase message +/// for the Graph aggregate, a PeginConfirm round message for the Instance +/// aggregate, or `None` when this role takes the kind through neither. +pub fn inbound_event( + actor: &Actor, + author: &str, + local_peer: &str, + decoded: &GOATMessage, + raw: &[u8], +) -> anyhow::Result> { + let kind = decoded.content.event_type(); + let business = decoded.content.business_ref(); + let (BusinessRef::Graph { instance_id, .. } | BusinessRef::Instance { instance_id }) = business + else { + return Ok(None); + }; + let (aggregate, business_id, ack) = match business { + BusinessRef::Graph { graph_id, .. } if graph::engine_handles(kind, actor) => ( + graph::graph_ref(graph_id), + format!("{instance_id}:{graph_id}"), + graph::ack_required(kind), + ), + BusinessRef::Instance { .. } if instance::engine_handles(kind, actor) => { + (instance::instance_ref(instance_id), instance_id.to_string(), false) + } + _ => return Ok(None), + }; + let (graph_nonce, operator_pubkey) = match &decoded.content { + GOATMessageContent::CreateGraph(message) => { + (Some(message.graph_nonce), Some(message.graph.parameters.operator_pubkey.to_string())) + } + GOATMessageContent::GraphFinalize(message) => { + (Some(message.graph_nonce), Some(message.graph.parameters.operator_pubkey.to_string())) + } + GOATMessageContent::InitGraph(message) => (None, Some(message.operator_pubkey.to_string())), + _ => (None, None), + }; + let inbound = InboundMessage { + author: author.to_string(), + instance_id, + graph_nonce, + operator_pubkey, + message: raw.to_vec(), + }; + let mut event = NewEvent::new( + content_event_id(author, kind, &business_id, raw), + aggregate, + kind, + EventSource::Network, + inbound.encode()?, + ) + .with_author(author); + if ack { + event = event.with_ack(author, delivery_id(author, local_peer, raw)); + } + Ok(Some(event)) +} + +// --------------------------------------------------------------- membership + +/// Whether this committee node holds the key of an instance, which is what +/// makes the instance's graph messages its business. A cheap local check for +/// the apply; the jobs verify membership against Goat. +pub trait MembershipProbe: Send + Sync { + fn holds_instance_key(&self, instance_id: Uuid) -> bool; + + /// Whether the author of a message is a peer the registry holds as + /// registered on chain. A parked message from anyone else waits for the + /// definition to arrive by itself: no GetGraph pull is spent on it. + fn is_registered_peer(&self, _peer: &str) -> bool { + true + } +} + +/// The instance key envelope on disk, and the peer registry. +pub struct EnvelopeProbe; + +impl MembershipProbe for EnvelopeProbe { + fn holds_instance_key(&self, instance_id: Uuid) -> bool { + committee_instance_keys_envelope_path(instance_id).exists() + } + + fn is_registered_peer(&self, peer: &str) -> bool { + libp2p::PeerId::from_str(peer).is_ok_and(|peer| { + crate::p2p_admission::peer_registry() + .sender_class(&peer, std::time::Instant::now()) + .is_registered() + }) + } +} + +/// The instance is in a failure state; an unknown instance is not. +pub(super) async fn instance_failed( + ctx: &mut ApplyContext<'_>, + instance_id: Uuid, +) -> anyhow::Result { + Ok(ctx + .storage + .find_instance(&instance_id) + .await? + .and_then(|instance| InstanceBridgeInStatus::from_str(&instance.status).ok()) + .is_some_and(|status| status.is_failed())) +} + +// -------------------------------------------------------------------- scans + +/// `(height, status)` of the last scan recorded per id: a scan at the same +/// tip for the same status would repeat the same facts. +pub(super) struct ScanMarks { + marks: Mutex>, +} + +impl ScanMarks { + /// Marks kept before the table is started over. + const LIMIT: usize = 50_000; + + pub(super) fn new() -> Self { + Self { marks: Mutex::new(HashMap::new()) } + } + + fn lock(&self) -> MutexGuard<'_, HashMap> { + self.marks.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub(super) fn seen(&self, id: Uuid, height: i64, status: S) -> bool { + self.lock().get(&id).is_some_and(|mark| *mark == (height, status)) + } + + pub(super) fn mark(&self, id: Uuid, height: i64, status: S) { + let mut marks = self.lock(); + if marks.len() >= Self::LIMIT { + marks.clear(); + } + marks.insert(id, (height, status)); + } + + pub(super) fn forget(&self, id: Uuid) { + self.lock().remove(&id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn delivery_and_content_ids_are_stable_and_bound() { + let a = delivery_id("peer-a", "peer-b", b"payload"); + assert_eq!(a, delivery_id("peer-a", "peer-b", b"payload")); + assert_ne!(a, delivery_id("peer-b", "peer-a", b"payload")); + assert_ne!(a, delivery_id("peer-a", "peer-b", b"other")); + assert!(a.starts_with("graph-setup-v2:")); + let c = content_event_id("peer-a", "CreateGraph", "i:g", b"payload"); + assert_eq!(c, content_event_id("peer-a", "CreateGraph", "i:g", b"payload")); + assert_ne!(c, content_event_id("peer-b", "CreateGraph", "i:g", b"payload")); + assert_ne!(c, content_event_id("peer-a", "GraphFinalize", "i:g", b"payload")); + } + + #[test] + fn scan_marks_remember_one_tip_per_id() { + let marks = ScanMarks::new(); + let id = Uuid::nil(); + assert!(!marks.seen(id, 10, 1)); + marks.mark(id, 10, 1); + assert!(marks.seen(id, 10, 1)); + assert!(!marks.seen(id, 11, 1) && !marks.seen(id, 10, 2)); + marks.forget(id); + assert!(!marks.seen(id, 10, 1)); + } +} diff --git a/node/src/aggregates/graph.rs b/node/src/aggregates/graph.rs new file mode 100644 index 000000000..c97e69ea6 --- /dev/null +++ b/node/src/aggregates/graph.rs @@ -0,0 +1,1258 @@ +//! Graph event handling and coordination. +//! +//! Apply routes messages to validation jobs, waits for missing definitions and +//! commits job plans as outgoing messages, follow-up jobs and state updates. +//! Graph status also receives updates from stored definitions and chain events. + +use crate::aggregates::common::{ + EnvelopeProbe, InboundMessage, MembershipProbe, NextJob, PARKED_MESSAGE_TTL_SECS, + PlannedMessage, instance_failed, +}; +use crate::aggregates::graph_fetch::GraphLookup; +use crate::aggregates::graph_pegout::{ + EVENT_CHAIN_SCAN, EVENT_GOAT_STATUS, EVENT_WITHDRAW_CANCEL, EVENT_WITHDRAW_INIT, PEGOUT_JOBS, + PHASE_KICKOFF, PHASE_SETTLEMENT, pegout_job_pool, +}; +use crate::aggregates::instance::graph_endorsed_event; +use crate::aggregates::jobs::Plan; +use crate::engine_bridge::record_event_in; +use crate::metrics_service::node_metrics_state; +use crate::middleware::get_topic_name; +use anyhow::anyhow; +use async_trait::async_trait; +use bitvm_lib::actors::Actor; +use engine::{ + AggregateHandler, AggregateRef, ApplyContext, ApplyError, ApplyOutcome, EngineInboxEvent, + EngineOutboxCloseTarget, EventSource, JOB_RESULT_KIND, JobPool, JobResultEvent, + JobResultOutcome, JobSpec, NewEvent, OutboxMessage, WaitingFor, +}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use std::sync::Arc; +use store::localdb::GraphQuery; +use store::{EngineJobState, GraphStatus, GraphStatusSource}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +pub const GRAPH_KIND: &str = "graph"; + +pub const EVENT_OPERATOR_START: &str = "operator_start"; +pub const EVENT_GRAPH_POSTED: &str = "graph_posted"; +pub const EVENT_GRAPH_PULL: &str = "graph_pull"; +/// A definition this node pulled was stored (sent to the pulled graph). +pub const EVENT_GRAPH_PULLED: &str = "graph_pulled"; +/// The operator's setup was still unfinished at the instance's abandon height. +pub const EVENT_SETUP_ABANDONED: &str = "setup_abandoned"; +/// The operator's setup collected its proofs: build the definition once no +/// other build of the operator is in flight. +pub const EVENT_BUILD_READY: &str = "build_ready"; + +pub const PHASE_SETUP: &str = "Setup"; +pub const PHASE_COLLECT: &str = "Collect"; +pub const PHASE_SIGNING: &str = "Signing"; +pub const PHASE_SIGNED: &str = "Signed"; +pub const PHASE_DELIVER: &str = "Deliver"; +pub const PHASE_POST: &str = "Post"; +pub const PHASE_AWAIT_WITHDRAW: &str = "AwaitWithdraw"; +pub const PHASE_CLOSED: &str = "Closed"; +pub const PHASE_OBSOLETED: &str = "Obsoleted"; +/// Outbox phase of the verifier's endorsement (closes on CommitteePresigned). +pub const PHASE_ENDORSE: &str = "Endorse"; +/// Job phase of the setup and signing jobs (everything before the relayer's +/// post); peg-out jobs run in the Kickoff and Settlement phases. +pub const JOB_PHASE: &str = "graph"; + +pub const JOB_OPERATOR_INIT: &str = "operator_init_graph"; +pub const JOB_VERIFIER_SETUP: &str = "verifier_setup"; +pub const JOB_INGEST_GEN_CIRCUITS: &str = "ingest_gen_circuits"; +pub const JOB_FREEZE_CANDIDATES: &str = "freeze_candidates"; +pub const JOB_VERIFIER_SOLDER: &str = "verifier_solder"; +pub const JOB_INGEST_SOLDERING_PROOF: &str = "ingest_soldering_proof"; +pub const JOB_SEAL_PROOFS: &str = "seal_proofs"; +pub const JOB_BUILD_GRAPH: &str = "build_graph"; +pub const JOB_VERIFIER_VALIDATE_GRAPH: &str = "verifier_validate_graph"; +pub const JOB_COMMITTEE_INGEST_GRAPH: &str = "committee_ingest_graph"; +pub const JOB_INGEST_VERIFIER_ENDORSEMENT: &str = "ingest_verifier_endorsement"; +pub const JOB_COMMITTEE_NONCES: &str = "committee_nonces"; +pub const JOB_INGEST_NONCES: &str = "ingest_nonces"; +pub const JOB_COMMITTEE_VOTE: &str = "committee_vote"; +pub const JOB_INGEST_VOTE: &str = "ingest_vote"; +pub const JOB_COMMITTEE_PRESIGN: &str = "committee_presign"; +pub const JOB_INGEST_PRESIGN: &str = "ingest_presign"; +pub const JOB_COMMITTEE_ENDORSE: &str = "committee_endorse"; +pub const JOB_INGEST_ENDORSEMENT: &str = "ingest_endorsement"; +pub const JOB_FINALIZE_GRAPH: &str = "finalize_graph"; +pub const JOB_INGEST_FINALIZED: &str = "ingest_finalized"; +pub const JOB_POST_GRAPH_DATA: &str = "post_graph_data"; +pub const JOB_FETCH_GRAPH: &str = "fetch_graph"; + +pub const ALL_JOBS: &[&str] = &[ + JOB_OPERATOR_INIT, + JOB_VERIFIER_SETUP, + JOB_INGEST_GEN_CIRCUITS, + JOB_FREEZE_CANDIDATES, + JOB_VERIFIER_SOLDER, + JOB_INGEST_SOLDERING_PROOF, + JOB_SEAL_PROOFS, + JOB_BUILD_GRAPH, + JOB_VERIFIER_VALIDATE_GRAPH, + JOB_COMMITTEE_INGEST_GRAPH, + JOB_INGEST_VERIFIER_ENDORSEMENT, + JOB_COMMITTEE_NONCES, + JOB_INGEST_NONCES, + JOB_COMMITTEE_VOTE, + JOB_INGEST_VOTE, + JOB_COMMITTEE_PRESIGN, + JOB_INGEST_PRESIGN, + JOB_COMMITTEE_ENDORSE, + JOB_INGEST_ENDORSEMENT, + JOB_FINALIZE_GRAPH, + JOB_INGEST_FINALIZED, + JOB_POST_GRAPH_DATA, + JOB_FETCH_GRAPH, +]; + +/// CPU-bound jobs share the engine's `J` slots; the rest talk to the chain. +pub fn job_pool(action: &str) -> JobPool { + if PEGOUT_JOBS.contains(&action) { + return pegout_job_pool(action); + } + match action { + JOB_VERIFIER_SETUP + | JOB_VERIFIER_SOLDER + | JOB_INGEST_SOLDERING_PROOF + | JOB_BUILD_GRAPH + | JOB_VERIFIER_VALIDATE_GRAPH + | JOB_COMMITTEE_INGEST_GRAPH + | JOB_COMMITTEE_PRESIGN + | JOB_INGEST_PRESIGN + | JOB_FINALIZE_GRAPH + | JOB_INGEST_FINALIZED => JobPool::Cpu, + _ => JobPool::Network, + } +} + +pub fn graph_ref(graph_id: Uuid) -> AggregateRef { + AggregateRef::new(GRAPH_KIND, graph_id.to_string()) +} + +/// Wake reference of an operator slot; waiters are released when a +/// definition for the slot is stored by any path. +pub fn slot_ref(operator_pubkey: &str, graph_nonce: u64) -> String { + format!("slot:{operator_pubkey}:{graph_nonce}") +} + +/// Outbox row id of a message kind of this graph (per recipient for the +/// operator's `CutCircuits`). +pub fn outbox_id(graph_id: Uuid, kind: &str, qualifier: Option<&str>) -> String { + match qualifier { + Some(qualifier) => format!("graph:{graph_id}:{kind}:{qualifier}"), + None => format!("graph:{graph_id}:{kind}"), + } +} + +// ----------------------------------------------------------------- payloads + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OperatorStart { + pub instance_id: Uuid, + /// The local operator key: the slot chain the new graph extends. + pub operator_pubkey: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphPull { + pub lookup: GraphLookup, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphPulled { + pub instance_id: Uuid, + pub finalized: bool, +} + +/// The Bitcoin height a scan saw at or past the instance's abandon height. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SetupAbandoned { + pub instance_id: Uuid, + pub abandon_height: i64, + pub height: i64, +} + +/// Bookkeeping a job learned about the graph. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphStateUpdate { + pub instance_id: Option, + pub operator_pubkey: Option, + pub graph_nonce: Option, + pub finalized: Option, +} + +/// What a job decided; the apply commits it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphPlan { + pub messages: Vec, + pub close_rows: Vec, + pub next: Vec, + pub state: GraphStateUpdate, + pub delete_pending_session: bool, + pub notify_instance_endorsed: bool, + pub wake_refs: Vec, + pub note: Option, +} + +impl Plan for GraphPlan { + fn noted(note: impl Into) -> Self { + Self { note: Some(note.into()), ..Default::default() } + } +} + +impl GraphPlan { + pub fn encode(&self) -> anyhow::Result> { + Ok(bincode::serialize(self)?) + } + + pub fn decode(bytes: &[u8]) -> anyhow::Result { + Ok(bincode::deserialize(bytes)?) + } + + pub fn send(&mut self, message: PlannedMessage) -> &mut Self { + self.messages.push(message); + self + } + + pub fn then(&mut self, action: &str, params: Vec, delay_secs: Option) -> &mut Self { + self.next.push(NextJob { action: action.to_string(), params, delay_secs }); + self + } +} + +/// Opaque aggregate state (JSON in the engine row). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphState { + pub instance_id: Option, + pub operator_pubkey: Option, + pub graph_nonce: Option, + #[serde(default)] + pub finalized: bool, + /// The operator's live `InitWithdraw` (its Goat tx hash): Kickoff phase. + #[serde(default)] + pub withdraw: Option, + /// Withdraws cancelled on Goat; a late re-apply of their init is ignored. + #[serde(default)] + pub retired_withdraws: Vec, + /// The graph's Pre-Kickoff can never confirm (reported once). + #[serde(default)] + pub dead: bool, + /// The operator gave up its unfinished setup at the abandon height. + #[serde(default)] + pub abandoned: bool, +} + +impl GraphState { + fn load(raw: Option<&str>) -> Self { + raw.and_then(|raw| serde_json::from_str(raw).ok()).unwrap_or_default() + } + + fn merge(&mut self, update: GraphStateUpdate) { + if update.instance_id.is_some() { + self.instance_id = update.instance_id; + } + if update.operator_pubkey.is_some() { + self.operator_pubkey = update.operator_pubkey; + } + if update.graph_nonce.is_some() { + self.graph_nonce = update.graph_nonce; + } + if let Some(finalized) = update.finalized { + self.finalized = finalized; + } + } +} + +// ------------------------------------------------------------------- routing + +/// Message kinds the aggregate takes from the network, and which job ingests +/// them for which role. +pub fn route_action(kind: &str, actor: &Actor) -> Option<&'static str> { + use Actor::{Committee, Operator, Verifier}; + Some(match (kind, actor) { + ("InitGraph", Verifier) => JOB_VERIFIER_SETUP, + ("GenCircuits", Operator) => JOB_INGEST_GEN_CIRCUITS, + ("CutCircuits", Verifier) => JOB_VERIFIER_SOLDER, + ("SolderingProofReady", Operator) => JOB_INGEST_SOLDERING_PROOF, + ("CreateGraph", Verifier) => JOB_VERIFIER_VALIDATE_GRAPH, + ("CreateGraph", Committee) => JOB_COMMITTEE_INGEST_GRAPH, + ("VerifierGraphParamsEndorsement", Committee) => JOB_INGEST_VERIFIER_ENDORSEMENT, + ("NonceGeneration", Committee | Operator) => JOB_INGEST_NONCES, + ("AggNonceConsensus", Committee) => JOB_INGEST_VOTE, + ("CommitteePresign", Committee | Operator) => JOB_INGEST_PRESIGN, + ("EndorseGraph", Operator) => JOB_INGEST_ENDORSEMENT, + ("GraphFinalize", _) => JOB_INGEST_FINALIZED, + _ => return None, + }) +} + +/// Whether this role takes `kind` through the engine at all. +pub fn engine_handles(kind: &str, actor: &Actor) -> bool { + route_action(kind, actor).is_some() +} + +/// Heavy messages are ACKed once durably recorded. +pub fn ack_required(kind: &str) -> bool { + matches!( + kind, + "GenCircuits" | "CutCircuits" | "SolderingProofReady" | "CreateGraph" | "GraphFinalize" + ) +} + +pub fn operator_start_event( + instance_id: Uuid, + graph_id: Uuid, + operator_pubkey: &str, +) -> anyhow::Result { + Ok(NewEvent::new( + format!("graph:{graph_id}:operator-start"), + graph_ref(graph_id), + EVENT_OPERATOR_START, + EventSource::Local, + bincode::serialize(&OperatorStart { + instance_id, + operator_pubkey: operator_pubkey.to_string(), + })?, + )) +} + +/// Create the event that requests serialized graph building. +pub fn build_ready_event(graph_id: Uuid) -> NewEvent { + NewEvent::new( + format!("graph:{graph_id}:build-ready"), + graph_ref(graph_id), + EVENT_BUILD_READY, + EventSource::Local, + Vec::new(), + ) +} + +/// Keyed by height, not only by graph: a graph whose build lands after its +/// session was released is still given up on a later block. +pub fn setup_abandoned_event( + graph_id: Uuid, + abandoned: &SetupAbandoned, +) -> anyhow::Result { + Ok(NewEvent::new( + format!("graph:{graph_id}:abandon:{}", abandoned.height), + graph_ref(graph_id), + EVENT_SETUP_ABANDONED, + EventSource::Height, + bincode::serialize(abandoned)?, + )) +} + +/// A pull stored this graph's definition; the graph's own apply decides what +/// follows (the relayer's post for a finalized one). +pub fn graph_pulled_event( + graph_id: Uuid, + instance_id: Uuid, + finalized: bool, +) -> anyhow::Result { + let flavour = if finalized { "finalized" } else { "presigned" }; + Ok(NewEvent::new( + format!("graph:{graph_id}:pulled:{flavour}"), + graph_ref(graph_id), + EVENT_GRAPH_PULLED, + EventSource::Local, + bincode::serialize(&GraphPulled { instance_id, finalized })?, + )) +} + +pub fn graph_posted_event(graph_id: Uuid) -> anyhow::Result { + Ok(NewEvent::new( + format!("graph:{graph_id}:posted"), + graph_ref(graph_id), + EVENT_GRAPH_POSTED, + EventSource::Chain, + Vec::new(), + )) +} + +/// Ask the aggregate to pull a definition this node lacks. +pub fn graph_pull_event(graph_id: Uuid, lookup: GraphLookup) -> anyhow::Result { + let key = match &lookup { + GraphLookup::Id { finalized: false, .. } => "id".to_string(), + GraphLookup::Id { finalized: true, .. } => "finalized".to_string(), + GraphLookup::Slot { operator_pubkey, graph_nonce } => { + format!("slot:{operator_pubkey}:{graph_nonce}") + } + }; + Ok(NewEvent::new( + format!("graph:{graph_id}:pull:{key}"), + graph_ref(graph_id), + EVENT_GRAPH_PULL, + EventSource::Local, + bincode::serialize(&GraphPull { lookup })?, + )) +} + +// -------------------------------------------------------------------- phases + +/// Choose the local phase from graph status, node role and live withdrawal state. +pub fn phase_of( + status: Option, + actor: &Actor, + relayer: bool, + withdrawing: bool, +) -> &'static str { + use GraphStatus::*; + match status { + None => PHASE_SETUP, + Some(OperatorPresigned) => match actor { + Actor::Operator => PHASE_COLLECT, + Actor::Committee => PHASE_SIGNING, + _ => PHASE_SETUP, + }, + Some(CommitteePresigned) => match actor { + Actor::Operator => PHASE_DELIVER, + _ if relayer => PHASE_POST, + _ => PHASE_SIGNED, + }, + Some(OperatorDataPushed) if withdrawing => PHASE_KICKOFF, + Some(OperatorDataPushed) => PHASE_AWAIT_WITHDRAW, + Some(PreKickoff) => PHASE_KICKOFF, + Some(OperatorKickOff | Challenge) => PHASE_SETTLEMENT, + Some(Obsoleted) => PHASE_OBSOLETED, + Some(Skipped | Disprove | OperatorTake1 | OperatorTake2) => PHASE_CLOSED, + } +} + +/// Whether committee round messages and deliveries are no longer needed. +pub fn signing_round_over(status: Option) -> bool { + status.is_some_and(|status| !matches!(status, GraphStatus::OperatorPresigned)) +} + +/// Past the signing phases (posted, obsoleted or skipped): every delivery +/// of the graph is closed. +pub fn past_signing(status: Option) -> bool { + status.is_some_and(|status| { + !matches!(status, GraphStatus::OperatorPresigned | GraphStatus::CommitteePresigned) + }) +} + +/// Whether setup and deliveries should stop. Peg-out work may still be needed +/// for chain monitoring or Goat settlement. +pub fn retired(status: Option) -> bool { + status.is_some_and(|status| status.is_obsoleted() || status.is_closed()) +} + +// ------------------------------------------------------------------- handler + +pub struct GraphAggregate { + actor: Actor, + relayer: bool, + local_peer: String, + membership: Arc, + /// The local operator key as stored in `graph.operator_pubkey`. + local_operator: Option, +} + +impl GraphAggregate { + pub fn new(actor: Actor, relayer: bool, local_peer: String) -> Self { + Self { + actor, + relayer, + local_peer, + membership: Arc::new(EnvelopeProbe), + local_operator: None, + } + } + + pub fn with_membership(mut self, membership: Arc) -> Self { + self.membership = membership; + self + } + + /// The operator key whose graphs this node withdraws and defends. + pub fn with_local_operator(mut self, operator_pubkey: Option) -> Self { + self.local_operator = operator_pubkey; + self + } + + pub(super) fn actor(&self) -> Actor { + self.actor.clone() + } + + pub(super) fn relayer(&self) -> bool { + self.relayer + } + + /// The graph belongs to this node's operator. + pub(super) fn owns(&self, row: &store::Graph) -> bool { + self.actor == Actor::Operator + && self.local_operator.as_deref() == Some(row.operator_pubkey.as_str()) + } + + /// Wait for a missing definition, scheduling a fetch only for registered authors. + /// Reject the message if its instance failed or its waiting deadline elapsed. + async fn park_for_definition( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + inbound: &InboundMessage, + created_at: i64, + lookup: GraphLookup, + waiting: WaitingFor, + ) -> anyhow::Result { + if instance_failed(ctx, inbound.instance_id).await? { + debug!(event = "graph_apply", graph_id = %graph_id, detail = %waiting.detail, "instance failed; message ignored"); + return Ok(ApplyOutcome::Applied); + } + let waited = ctx.now - created_at; + if waited > PARKED_MESSAGE_TTL_SECS { + return Ok(ApplyOutcome::Rejected(format!( + "waited {waited}s for the {}; giving up", + waiting.detail + ))); + } + if self.membership.is_registered_peer(&inbound.author) { + let params = bincode::serialize(&lookup) + .map_err(|error| anyhow!("encode graph lookup: {error}"))?; + self.enqueue_setup_job(ctx, JOB_FETCH_GRAPH, ¶ms); + } else { + debug!(event = "graph_apply", graph_id = %graph_id, author = %inbound.author, detail = %waiting.detail, "author is not a registered peer; waiting without a pull"); + } + Ok(ApplyOutcome::Waiting(waiting)) + } + + fn enqueue_setup_job(&self, ctx: &mut ApplyContext<'_>, action: &str, payload: &[u8]) { + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + JOB_PHASE, + action, + job_pool(action), + payload.to_vec(), + )); + } + + #[allow(clippy::too_many_arguments)] + async fn apply_message( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + state: &mut GraphState, + status: Option, + kind: &str, + payload: &[u8], + created_at: i64, + ) -> anyhow::Result { + let inbound = match InboundMessage::decode(payload) { + Ok(inbound) => inbound, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!("malformed {kind} event: {error}"))); + } + }; + if inbound.author == self.local_peer { + return Ok(ApplyOutcome::Applied); + } + let Some(action) = route_action(kind, &self.actor) else { + return Ok(ApplyOutcome::Rejected(format!("{kind} is not handled by {}", self.actor))); + }; + // A committee node outside the instance's committee neither signs + // nor waits for the graph; the finalized definition is still taken. + if self.actor == Actor::Committee + && kind != "GraphFinalize" + && !self.membership.holds_instance_key(inbound.instance_id) + { + debug!(event = "graph_apply", graph_id = %graph_id, kind, instance_id = %inbound.instance_id, "not in this instance's committee; message ignored"); + return Ok(ApplyOutcome::Applied); + } + if retired(status) { + debug!(event = "graph_apply", graph_id = %graph_id, kind, "graph retired; message ignored"); + return Ok(ApplyOutcome::Applied); + } + if state.instance_id.is_none() { + state.instance_id = Some(inbound.instance_id); + } + if state.operator_pubkey.is_none() { + state.operator_pubkey = inbound.operator_pubkey.clone(); + } + if state.graph_nonce.is_none() { + state.graph_nonce = inbound.graph_nonce; + } + match kind { + "InitGraph" => { + if status.is_some() { + return Ok(ApplyOutcome::Applied); + } + } + "GenCircuits" | "SolderingProofReady" => { + let pending = ctx.storage.find_pending_graph_init_by_graph_id(&graph_id).await?; + if pending.is_none_or(|pending| pending.instance_id != inbound.instance_id) { + debug!(event = "graph_apply", graph_id = %graph_id, kind, "no pending setup session; message ignored"); + return Ok(ApplyOutcome::Applied); + } + } + "CutCircuits" => {} + "CreateGraph" => { + if self.actor == Actor::Committee + && let (Some(operator_pubkey), Some(graph_nonce)) = + (inbound.operator_pubkey.as_deref(), inbound.graph_nonce) + { + // One definition per slot. + if let Some(existing) = ctx + .storage + .find_graph_by_operator_slot(operator_pubkey, graph_nonce as i64) + .await? + && existing.graph_id != graph_id + { + return Ok(ApplyOutcome::Rejected(format!( + "slot {operator_pubkey}:{graph_nonce} already holds graph {}", + existing.graph_id + ))); + } + // Continuity needs the previous slot's definition; pull it + // and wait instead of refusing or retrying blindly. + if graph_nonce > 0 + && ctx + .storage + .find_graph_by_operator_slot(operator_pubkey, graph_nonce as i64 - 1) + .await? + .is_none() + { + let lookup = GraphLookup::Slot { + operator_pubkey: operator_pubkey.to_string(), + graph_nonce: graph_nonce - 1, + }; + let waiting = WaitingFor::on_ref( + slot_ref(operator_pubkey, graph_nonce - 1), + format!("previous graph definition for slot {graph_nonce}"), + ); + return self + .park_for_definition( + ctx, graph_id, &inbound, created_at, lookup, waiting, + ) + .await; + } + } + } + "VerifierGraphParamsEndorsement" + | "NonceGeneration" + | "AggNonceConsensus" + | "CommitteePresign" + | "EndorseGraph" => { + if status.is_none() { + if self.actor == Actor::Operator { + debug!(event = "graph_apply", graph_id = %graph_id, kind, "not a local graph; message ignored"); + return Ok(ApplyOutcome::Applied); + } + // The definition may never reach this node by gossip + // (it was offline, or missing from the operator's ack + // set): pull it while waiting. + let lookup = GraphLookup::Id { + instance_id: inbound.instance_id, + graph_id, + finalized: false, + }; + let waiting = WaitingFor::on(&ctx.aggregate, "graph definition not stored yet"); + return self + .park_for_definition(ctx, graph_id, &inbound, created_at, lookup, waiting) + .await; + } + if signing_round_over(status) { + debug!(event = "graph_apply", graph_id = %graph_id, kind, "signing round is over; message ignored"); + return Ok(ApplyOutcome::Applied); + } + } + "GraphFinalize" => { + if state.finalized { + return Ok(ApplyOutcome::Applied); + } + } + _ => {} + } + self.enqueue_setup_job(ctx, action, payload); + Ok(ApplyOutcome::Applied) + } + + async fn apply_job_result( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + state: &mut GraphState, + status: Option, + result: JobResultEvent, + ) -> anyhow::Result { + match &result.outcome { + JobResultOutcome::Done(bytes) => { + let mut plan = match GraphPlan::decode(bytes) { + Ok(plan) => plan, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!( + "malformed plan from {}: {error}", + result.action + ))); + } + }; + if let Some(note) = &plan.note { + info!(event = "graph_job", graph_id = %graph_id, action = %result.action, stale = result.stale, note = %note, "job note"); + } + // Drop late setup/post actions for retired graphs, but preserve dependency wakes. + // Peg-out results may still be needed for settlement. + if retired(status) && !PEGOUT_JOBS.contains(&result.action.as_str()) { + info!(event = "graph_job", graph_id = %graph_id, action = %result.action, status = ?status, "graph retired; plan dropped"); + plan = GraphPlan { wake_refs: plan.wake_refs, ..Default::default() }; + } + self.apply_plan(ctx, graph_id, state, plan).await?; + } + JobResultOutcome::Failed(error) => { + warn!(event = "graph_job", graph_id = %graph_id, action = %result.action, error = %error, "job failed"); + if result.action == JOB_POST_GRAPH_DATA + && let Some(metrics) = node_metrics_state() + { + metrics.record_instance_alert("post_graph_data_rejected"); + } + // The builds waiting behind a failed build go on. + if result.action == JOB_BUILD_GRAPH + && let Some(operator_pubkey) = state.operator_pubkey.as_deref() + { + let slot = next_slot(ctx, operator_pubkey).await?; + ctx.effects.wake(slot_ref(operator_pubkey, slot)); + } + } + JobResultOutcome::Cancelled => { + debug!(event = "graph_job", graph_id = %graph_id, action = %result.action, "job cancelled"); + } + JobResultOutcome::Poisoned(error) => { + warn!(event = "graph_job", graph_id = %graph_id, action = %result.action, error = %error, "job panicked"); + } + } + ctx.storage.clear_engine_job_result(&result.job_id, ctx.now).await?; + Ok(ApplyOutcome::Applied) + } + + async fn apply_plan( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + state: &mut GraphState, + plan: GraphPlan, + ) -> anyhow::Result<()> { + state.merge(plan.state); + for message in plan.messages { + ctx.effects.send(OutboxMessage { + outbox_id: message.outbox_id, + phase_ref: message.phase_ref, + kind: message.kind, + topic: get_topic_name(&message.actor), + payload: message.payload, + ack_set: message.ack_set, + }); + } + for row in plan.close_rows { + ctx.effects.close_outbox(EngineOutboxCloseTarget::Row(row), "superseded"); + } + for NextJob { action, params, delay_secs } in plan.next { + if action == JOB_BUILD_GRAPH { + // Builds of one operator run one at a time. + record_event_in(ctx.storage, build_ready_event(graph_id)).await?; + continue; + } + let phase = if action == JOB_POST_GRAPH_DATA { + PHASE_POST + } else if PEGOUT_JOBS.contains(&action.as_str()) { + PHASE_SETTLEMENT + } else { + JOB_PHASE + }; + let mut job = JobSpec::new(&ctx.aggregate, phase, &action, job_pool(&action), params); + if let Some(delay) = delay_secs { + job = job.with_run_after(ctx.now + delay.max(0)); + } + ctx.effects.enqueue_job(job); + } + if plan.delete_pending_session + && let (Some(instance_id), Some(operator_pubkey)) = + (state.instance_id, state.operator_pubkey.as_deref()) + { + ctx.storage.delete_pending_graph_init(&instance_id, operator_pubkey).await?; + } + if plan.notify_instance_endorsed + && let Some(instance_id) = state.instance_id + { + record_event_in(ctx.storage, graph_endorsed_event(instance_id, graph_id)?).await?; + } + for wake_ref in plan.wake_refs { + ctx.effects.wake(wake_ref); + } + Ok(()) + } + + /// Start building after proof collection unless this operator has another build + /// in flight. Wait for that build's slot or session release to avoid slot reuse. + async fn apply_build_ready( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + row: Option<&store::Graph>, + ) -> anyhow::Result { + if row.is_some() { + return Ok(ApplyOutcome::Applied); + } + let Some(session) = ctx.storage.find_pending_graph_init_by_graph_id(&graph_id).await? + else { + debug!(event = "graph_apply", graph_id = %graph_id, "no pending setup session; build dropped"); + return Ok(ApplyOutcome::Applied); + }; + if instance_failed(ctx, session.instance_id).await? { + return Ok(ApplyOutcome::Applied); + } + if let Some((builder, slot)) = + building_sibling(ctx, &session.operator_pubkey, graph_id).await? + { + return Ok(ApplyOutcome::Waiting(WaitingFor::on_ref( + slot_ref(&session.operator_pubkey, slot), + format!("graph {builder} is building slot {slot}"), + ))); + } + self.enqueue_setup_job(ctx, JOB_BUILD_GRAPH, &[]); + Ok(ApplyOutcome::Applied) + } + + /// Retire unfinished setup at the abandon height and release its pending session, + /// allowing later setups to proceed. Built graphs without committee signatures + /// remain available for Skip handling. + async fn abandon_setup( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + state: &mut GraphState, + row: Option<&store::Graph>, + abandoned: &SetupAbandoned, + ) -> anyhow::Result { + if self.actor != Actor::Operator || instance_failed(ctx, abandoned.instance_id).await? { + return Ok(ApplyOutcome::Applied); + } + match row { + Some(row) => { + if !self.owns(row) || row.status != GraphStatus::OperatorPresigned.to_string() { + return Ok(ApplyOutcome::Applied); + } + ctx.storage + .transition_graph_status( + row.instance_id, + graph_id, + GraphStatus::Obsoleted, + GraphStatusSource::ChainReconcile, + None, + ) + .await?; + // A build whose result has not applied yet still holds the + // session; the retired graph drops that result, so the session + // is released here. + if let Some(session) = + ctx.storage.find_pending_graph_init_by_graph_id(&graph_id).await? + { + release_setup_session(ctx, &session, "setup abandoned").await?; + } + } + None => { + let Some(session) = + ctx.storage.find_pending_graph_init_by_graph_id(&graph_id).await? + else { + return Ok(ApplyOutcome::Applied); + }; + release_setup_session(ctx, &session, "setup abandoned").await?; + } + } + state.abandoned = true; + warn!( + event = "graph_abandoned", + graph_id = %graph_id, + instance_id = %abandoned.instance_id, + built = row.is_some(), + height = abandoned.height, + abandon_height = abandoned.abandon_height, + "setup unfinished at the abandon height; given up" + ); + Ok(ApplyOutcome::Applied) + } + + /// Mirror graph status and close deliveries for completed or retired phases. + async fn finish( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + state: &GraphState, + ) -> anyhow::Result<()> { + // The apply may have moved the row (a chain scan): mirror what is stored. + let row = ctx.storage.find_graph(&graph_id).await?; + let status = row.as_ref().and_then(|row| GraphStatus::from_str(&row.status).ok()); + let previous = ctx.status().to_string(); + let current = status.map(|status| status.to_string()).unwrap_or_default(); + if current != previous { + // Waiters on the slot re-check: the operator's next build and kickoff. + if let Some(row) = &row { + ctx.effects.wake(slot_ref(&row.operator_pubkey, row.kickoff_index as u64)); + } + if signing_round_over(status) { + for phase in [PHASE_COLLECT, PHASE_SIGNING, PHASE_ENDORSE, PHASE_SETUP] { + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Phase { + aggregate_kind: GRAPH_KIND.to_string(), + aggregate_id: graph_id.to_string(), + phase_ref: phase.to_string(), + }, + format!("graph {current}"), + ); + } + } + if past_signing(status) { + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Aggregate { + aggregate_kind: GRAPH_KIND.to_string(), + aggregate_id: graph_id.to_string(), + }, + format!("graph {current}"), + ); + ctx.effects.cancel_phase_jobs(PHASE_POST); + } + if retired(status) { + ctx.effects.cancel_phase_jobs(JOB_PHASE); + } + // Taken, disproved or skipped: the slot can no longer be kicked off. Obsoleted + // alone forbids nothing. + if status.is_some_and(|status| status.is_closed()) { + ctx.effects.cancel_phase_jobs(PHASE_KICKOFF); + } + info!(event = "graph_transition", graph_id = %graph_id, from = %previous, to = %current, "graph status mirrored"); + } + let withdrawing = state.withdraw.is_some(); + ctx.effects + .set_status(current) + .set_phase(phase_of(status, &self.actor, self.relayer, withdrawing).to_string()) + .set_state(Some(serde_json::to_string(state)?)); + Ok(()) + } +} + +#[async_trait] +impl AggregateHandler for GraphAggregate { + async fn apply( + &self, + ctx: &mut ApplyContext<'_>, + event: &EngineInboxEvent, + ) -> Result { + let graph_id = Uuid::from_str(&ctx.aggregate.id).map_err(|error| { + ApplyError::invariant(anyhow!("graph aggregate id {}: {error}", ctx.aggregate.id)) + })?; + let mut state = GraphState::load(ctx.state()); + let row = ctx.storage.find_graph(&graph_id).await?; + let status = row.as_ref().and_then(|row| GraphStatus::from_str(&row.status).ok()); + if let Some(row) = &row + && state.instance_id.is_none() + { + state.instance_id = Some(row.instance_id); + state.operator_pubkey = Some(row.operator_pubkey.clone()); + state.graph_nonce = Some(row.kickoff_index as u64); + } + let payload = event.payload.as_deref().unwrap_or_default(); + let outcome = match event.kind.as_str() { + JOB_RESULT_KIND => { + let result = JobResultEvent::decode(payload).map_err(ApplyError::invariant)?; + self.apply_job_result(ctx, graph_id, &mut state, status, result).await? + } + EVENT_OPERATOR_START => { + let start: OperatorStart = match bincode::deserialize(payload) { + Ok(start) => start, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!( + "malformed operator_start: {error}" + ))); + } + }; + if self.actor != Actor::Operator { + return Ok(ApplyOutcome::Rejected("only operators start graphs".into())); + } + if row.is_some() { + // Built already; its deliveries live in the outbox. + ApplyOutcome::Applied + } else if state.abandoned { + info!(event = "graph_apply", graph_id = %graph_id, instance_id = %start.instance_id, "instance passed its abandon height before the graph started; start dropped"); + ApplyOutcome::Applied + } else if instance_failed(ctx, start.instance_id).await? { + ctx.storage + .delete_pending_graph_init(&start.instance_id, &start.operator_pubkey) + .await?; + info!(event = "graph_apply", graph_id = %graph_id, instance_id = %start.instance_id, "instance failed before its graph started; session dropped"); + ApplyOutcome::Applied + } else { + // Setups of one operator overlap; only their builds are serialized + // (`build_ready`), so the previous slot need not be finalized, or even + // built, to start. + state.instance_id = Some(start.instance_id); + state.operator_pubkey = Some(start.operator_pubkey.clone()); + self.enqueue_setup_job(ctx, JOB_OPERATOR_INIT, payload); + ApplyOutcome::Applied + } + } + EVENT_GRAPH_PULLED => { + let pulled: GraphPulled = match bincode::deserialize(payload) { + Ok(pulled) => pulled, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!( + "malformed graph_pulled: {error}" + ))); + } + }; + if state.instance_id.is_none() { + state.instance_id = Some(pulled.instance_id); + } + // Pulled definitions lack EVM endorsements. Keep accepting GraphFinalize for + // those endorsements, and queue publishing for a finalized, unposted copy. + if self.relayer && pulled.finalized && !past_signing(status) { + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + PHASE_POST, + JOB_POST_GRAPH_DATA, + job_pool(JOB_POST_GRAPH_DATA), + Vec::new(), + )); + } + ApplyOutcome::Applied + } + EVENT_GRAPH_POSTED | EVENT_GOAT_STATUS => { + // The watcher already wrote the status (OperatorDataPushed or a + // settlement); mirroring below closes deliveries and wakes the + // next slot. + ApplyOutcome::Applied + } + EVENT_BUILD_READY => self.apply_build_ready(ctx, graph_id, row.as_ref()).await?, + EVENT_SETUP_ABANDONED => { + let abandoned: SetupAbandoned = match bincode::deserialize(payload) { + Ok(abandoned) => abandoned, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!( + "malformed setup_abandoned: {error}" + ))); + } + }; + self.abandon_setup(ctx, graph_id, &mut state, row.as_ref(), &abandoned).await? + } + EVENT_WITHDRAW_INIT => { + self.apply_withdraw_init(ctx, &mut state, row.as_ref(), payload).await? + } + EVENT_WITHDRAW_CANCEL => self.apply_withdraw_cancel(ctx, &mut state), + EVENT_CHAIN_SCAN => { + self.apply_chain_scan(ctx, graph_id, &mut state, row.as_ref(), payload).await? + } + EVENT_GRAPH_PULL => { + let pull: GraphPull = match bincode::deserialize(payload) { + Ok(pull) => pull, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!( + "malformed graph_pull: {error}" + ))); + } + }; + if row.is_none() { + let params = bincode::serialize(&pull.lookup) + .map_err(|error| anyhow::anyhow!("encode graph lookup: {error}"))?; + self.enqueue_setup_job(ctx, JOB_FETCH_GRAPH, ¶ms); + } + ApplyOutcome::Applied + } + kind => { + self.apply_message( + ctx, + graph_id, + &mut state, + status, + kind, + payload, + event.created_at, + ) + .await? + } + }; + if matches!(outcome, ApplyOutcome::Applied | ApplyOutcome::Waiting(_)) { + self.finish(ctx, graph_id, &state).await?; + } + Ok(outcome) + } +} + +/// The operator's graph holding the highest slot, if any. +async fn latest_slot( + ctx: &mut ApplyContext<'_>, + operator_pubkey: &str, +) -> anyhow::Result> { + Ok(ctx + .storage + .get_operator_graphs( + GraphQuery::default() + .with_operator_pubkey(operator_pubkey.to_string()) + .with_order("kickoff_index DESC".to_string()) + .with_limit(1), + ) + .await? + .pop()) +} + +/// The slot the operator's next graph takes: one past the highest stored. +pub(crate) async fn next_slot( + ctx: &mut ApplyContext<'_>, + operator_pubkey: &str, +) -> anyhow::Result { + Ok(latest_slot(ctx, operator_pubkey) + .await? + .map(|graph| graph.kickoff_index as u64 + 1) + .unwrap_or(0)) +} + +/// Close a setup session that never stored its definition: its deliveries +/// and jobs go, the session goes, and the slot it was taking is released to +/// the starts waiting on it. Returns that slot. +pub(crate) async fn release_setup_session( + ctx: &mut ApplyContext<'_>, + session: &store::PendingGraphInit, + reason: &str, +) -> anyhow::Result { + let graph_ref = graph_ref(session.graph_id); + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Aggregate { + aggregate_kind: graph_ref.kind.clone(), + aggregate_id: graph_ref.id.clone(), + }, + reason, + ); + ctx.effects.cancel_jobs_for_aggregate(&graph_ref); + ctx.storage.delete_pending_graph_init(&session.instance_id, &session.operator_pubkey).await?; + let next_slot = next_slot(ctx, &session.operator_pubkey).await?; + ctx.effects.wake(slot_ref(&session.operator_pubkey, next_slot)); + Ok(next_slot) +} + +/// Another session of the operator whose `build_graph` is pending or running +/// and whose definition is not stored yet, with the slot it will take. A +/// session whose instance failed holds nothing: the instance's failure +/// removes it and cancels its build. +async fn building_sibling( + ctx: &mut ApplyContext<'_>, + operator_pubkey: &str, + graph_id: Uuid, +) -> anyhow::Result> { + let sessions = ctx.storage.list_pending_graph_inits_by_operator(operator_pubkey).await?; + for session in sessions { + if session.graph_id == graph_id + || ctx.storage.find_graph(&session.graph_id).await?.is_some() + || instance_failed(ctx, session.instance_id).await? + { + continue; + } + let building = ctx + .storage + .list_engine_jobs_for_aggregate(GRAPH_KIND, &session.graph_id.to_string()) + .await? + .into_iter() + .any(|job| { + job.action == JOB_BUILD_GRAPH + && (job.state == EngineJobState::Pending.to_string() + || job.state == EngineJobState::Running.to_string()) + }); + if building { + return Ok(Some((session.graph_id, next_slot(ctx, operator_pubkey).await?))); + } + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + use GraphStatus::*; + use std::collections::BTreeMap; + + #[test] + fn routes_follow_the_role_table() { + assert_eq!(route_action("InitGraph", &Actor::Verifier), Some(JOB_VERIFIER_SETUP)); + assert_eq!(route_action("InitGraph", &Actor::Committee), None); + assert_eq!(route_action("GenCircuits", &Actor::Operator), Some(JOB_INGEST_GEN_CIRCUITS)); + assert_eq!( + route_action("CreateGraph", &Actor::Verifier), + Some(JOB_VERIFIER_VALIDATE_GRAPH) + ); + assert_eq!( + route_action("CreateGraph", &Actor::Committee), + Some(JOB_COMMITTEE_INGEST_GRAPH) + ); + assert_eq!(route_action("CreateGraph", &Actor::Operator), None); + assert_eq!(route_action("NonceGeneration", &Actor::Operator), Some(JOB_INGEST_NONCES)); + assert_eq!(route_action("AggNonceConsensus", &Actor::Operator), None); + assert_eq!(route_action("EndorseGraph", &Actor::Committee), None); + assert_eq!(route_action("GraphFinalize", &Actor::Watchtower), Some(JOB_INGEST_FINALIZED)); + assert_eq!(route_action("PeginConfirmNonce", &Actor::Committee), None); + for kind in + ["GenCircuits", "CutCircuits", "SolderingProofReady", "CreateGraph", "GraphFinalize"] + { + assert!(ack_required(kind), "{kind}"); + } + assert!(!ack_required("NonceGeneration") && !ack_required("InitGraph")); + } + + #[test] + fn phases_follow_the_phase_table() { + let phase = |status, actor: &Actor, relayer| phase_of(status, actor, relayer, false); + assert_eq!(phase(None, &Actor::Operator, false), PHASE_SETUP); + assert_eq!(phase(Some(OperatorPresigned), &Actor::Operator, false), PHASE_COLLECT); + assert_eq!(phase(Some(OperatorPresigned), &Actor::Committee, false), PHASE_SIGNING); + assert_eq!(phase(Some(OperatorPresigned), &Actor::Verifier, false), PHASE_SETUP); + assert_eq!(phase(Some(CommitteePresigned), &Actor::Operator, false), PHASE_DELIVER); + assert_eq!(phase(Some(CommitteePresigned), &Actor::Committee, true), PHASE_POST); + assert_eq!(phase(Some(CommitteePresigned), &Actor::Committee, false), PHASE_SIGNED); + assert_eq!(phase(Some(OperatorDataPushed), &Actor::Operator, false), PHASE_AWAIT_WITHDRAW); + assert_eq!( + phase_of(Some(OperatorDataPushed), &Actor::Operator, false, true), + PHASE_KICKOFF + ); + assert_eq!(phase(Some(PreKickoff), &Actor::Verifier, false), PHASE_KICKOFF); + assert_eq!(phase(Some(OperatorKickOff), &Actor::Operator, false), PHASE_SETTLEMENT); + assert_eq!(phase(Some(Challenge), &Actor::Committee, true), PHASE_SETTLEMENT); + assert_eq!(phase(Some(Obsoleted), &Actor::Operator, false), PHASE_OBSOLETED); + assert_eq!(phase(Some(Skipped), &Actor::Operator, false), PHASE_CLOSED); + assert!(!signing_round_over(Some(OperatorPresigned)) && !signing_round_over(None)); + assert!( + signing_round_over(Some(CommitteePresigned)) && signing_round_over(Some(Obsoleted)) + ); + assert!(!past_signing(Some(CommitteePresigned)) && past_signing(Some(OperatorDataPushed))); + assert!( + retired(Some(Obsoleted)) && retired(Some(OperatorTake1)) && !retired(Some(PreKickoff)) + ); + } + + #[test] + fn plans_and_state_round_trip() { + let mut plan = GraphPlan::noted("x"); + plan.send(PlannedMessage { + outbox_id: "o".into(), + phase_ref: PHASE_SETUP.into(), + kind: "InitGraph".into(), + actor: "Verifier".into(), + payload: vec![1, 2, 3], + ack_set: Some(BTreeMap::from([("p".to_string(), "d".to_string())])), + }) + .then(JOB_FREEZE_CANDIDATES, vec![7], Some(15)); + plan.state.graph_nonce = Some(2); + let decoded = GraphPlan::decode(&plan.encode().unwrap()).unwrap(); + assert_eq!(decoded, plan); + let mut state = GraphState::default(); + state.merge(decoded.state); + assert_eq!(state.graph_nonce, Some(2)); + let json = serde_json::to_string(&state).unwrap(); + assert_eq!(GraphState::load(Some(&json)), state); + assert_eq!(GraphState::load(Some("garbage")), GraphState::default()); + assert_eq!( + outbox_id(Uuid::nil(), "CutCircuits", Some("k")), + format!("graph:{}:CutCircuits:k", Uuid::nil()) + ); + assert_eq!(slot_ref("op", 3), "slot:op:3"); + } +} diff --git a/node/src/aggregates/graph_fetch.rs b/node/src/aggregates/graph_fetch.rs new file mode 100644 index 000000000..19fcf6f26 --- /dev/null +++ b/node/src/aggregates/graph_fetch.rs @@ -0,0 +1,282 @@ +//! Fetch and serve graph definitions over `GetGraph`. +//! +//! Fetched definitions must match the requested identity and chain parameters. +//! Operator pre-signatures are verified, as are committee pre-signatures when +//! present. A graph posted on Goat requires committee pre-signatures. + +use crate::middleware::get_graph::{GetGraphRequest, GetGraphResponse}; +use crate::middleware::publisher::NetworkPublisher; +use crate::utils::{ + load_validated_graph_definition, parse_graph_raw_data, store_finalized_graph_if_needed, + store_operator_presigned_graph, validate_graph_instance_parameters, +}; +use anyhow::{Context, bail}; +use bitvm_lib::operator::verify_graph_operator_pre_signatures; +use bitvm_lib::types::{BitvmGcGraph, SimplifiedBitvmGcGraph}; +use client::btc_chain::BTCClient; +use client::goat_chain::GOATClient; +use libp2p::PeerId; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use store::GraphStatus; +use store::localdb::LocalDB; +use tracing::{info, warn}; +use uuid::Uuid; + +/// Which definition a fetch job wants. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum GraphLookup { + /// By id; `finalized` asks for a copy with committee pre-signatures, so a + /// stored operator-presigned copy does not satisfy it. + Id { + instance_id: Uuid, + graph_id: Uuid, + finalized: bool, + }, + Slot { + operator_pubkey: String, + graph_nonce: u64, + }, +} + +impl GraphLookup { + pub fn request(&self) -> GetGraphRequest { + match self { + GraphLookup::Id { instance_id, graph_id, finalized } => { + GetGraphRequest::by_id(*instance_id, *graph_id, *finalized) + } + GraphLookup::Slot { operator_pubkey, graph_nonce } => { + GetGraphRequest::by_slot(operator_pubkey.clone(), *graph_nonce) + } + } + } +} + +/// Answer a request from the local store. +pub async fn serve_graph_request( + local_db: &LocalDB, + request: &GetGraphRequest, +) -> anyhow::Result { + let mut storage = local_db.acquire().await?; + let row = match (request.graph_id, &request.operator_pubkey, request.graph_nonce) { + (Some(graph_id), _, _) => storage.find_graph(&graph_id).await?, + (None, Some(operator_pubkey), Some(graph_nonce)) => { + storage.find_graph_by_operator_slot(operator_pubkey, graph_nonce as i64).await? + } + _ => return Ok(GetGraphResponse::not_found()), + }; + let Some(row) = row else { + return Ok(GetGraphResponse::not_found()); + }; + if let Some(instance_id) = request.instance_id + && row.instance_id != instance_id + { + return Ok(GetGraphResponse::not_found()); + } + // Only the operator's copy is stored while the graph is OperatorPresigned. + if request.finalized && row.status == GraphStatus::OperatorPresigned.to_string() { + return Ok(GetGraphResponse::not_found()); + } + let Some(raw) = storage.find_graph_raw_data(&row.graph_id).await? else { + return Ok(GetGraphResponse::not_found()); + }; + Ok(GetGraphResponse::found(row.instance_id, row.graph_id, raw.raw_data)) +} + +/// What a fetch stored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FetchedGraph { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub operator_pubkey: String, + pub graph_nonce: u64, + pub finalized: bool, +} + +/// Candidate peers: the registered committee members first, then the +/// graph's operator when its key is known (slot lookups), never this node. +pub fn fetch_candidates(operator_pubkey: Option<&str>) -> Vec { + let now = std::time::Instant::now(); + let registry = crate::p2p_admission::peer_registry(); + let mut peers = registry.committee_peer_ids(now); + if let Some(operator_pubkey) = operator_pubkey + && let Ok(pubkey) = bitcoin::PublicKey::from_str(operator_pubkey) + { + let key = crate::p2p_admission::operator_key(&bitcoin::XOnlyPublicKey::from(pubkey)); + if let Some(peer) = registry.operator_peer_for(&key) + && !peers.contains(&peer) + { + peers.push(peer); + } + } + let local = crate::env::get_peer_id(); + peers.retain(|peer| peer.to_string() != local); + peers +} + +/// Ask the candidates one by one until a copy validates and is stored. +pub async fn fetch_and_store_graph( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + publisher: &NetworkPublisher, + lookup: &GraphLookup, +) -> anyhow::Result> { + // Already present (another path stored it meanwhile). + if let Some(found) = local_copy(local_db, lookup).await? { + return Ok(Some(found)); + } + let operator_pubkey = match lookup { + GraphLookup::Slot { operator_pubkey, .. } => Some(operator_pubkey.as_str()), + GraphLookup::Id { .. } => None, + }; + let candidates = fetch_candidates(operator_pubkey); + if candidates.is_empty() { + bail!("no committee or operator peer known to ask for {lookup:?}"); + } + let mut last_error = None; + for peer in candidates { + let response = match publisher.request_graph(peer, lookup.request()).await { + Ok(response) => response, + Err(error) => { + warn!(event = "get_graph", outcome = "request_failed", %peer, error = %error, "GetGraph request failed"); + last_error = Some(error); + continue; + } + }; + let (Some(instance_id), Some(graph_id), Some(raw)) = + (response.instance_id, response.graph_id, response.graph) + else { + info!(event = "get_graph", outcome = "not_found", %peer, ?lookup, "peer has no copy"); + continue; + }; + match validate_and_store( + local_db, + btc_client, + goat_client, + lookup, + instance_id, + graph_id, + raw, + ) + .await + { + Ok(found) => { + info!(event = "get_graph", outcome = "stored", %peer, graph_id = %found.graph_id, finalized = found.finalized, "stored a pulled graph definition"); + return Ok(Some(found)); + } + Err(error) => { + warn!(event = "get_graph", outcome = "rejected", %peer, graph_id = %graph_id, error = %error, "rejected a pulled graph definition"); + last_error = Some(error); + } + } + } + match last_error { + Some(error) => Err(error.context("no candidate served an acceptable definition")), + None => Ok(None), + } +} + +async fn local_copy( + local_db: &LocalDB, + lookup: &GraphLookup, +) -> anyhow::Result> { + let mut storage = local_db.acquire().await?; + let row = match lookup { + GraphLookup::Id { graph_id, .. } => storage.find_graph(graph_id).await?, + GraphLookup::Slot { operator_pubkey, graph_nonce } => { + storage.find_graph_by_operator_slot(operator_pubkey, *graph_nonce as i64).await? + } + }; + let Some(row) = row else { return Ok(None) }; + let Some(graph) = + load_validated_graph_definition(&mut storage, row.instance_id, row.graph_id).await? + else { + return Ok(None); + }; + let finalized = graph.committee_pre_signed(); + if matches!(lookup, GraphLookup::Id { finalized: true, .. }) && !finalized { + // The stored copy is the operator's version; the finalized one is fetched. + return Ok(None); + } + Ok(Some(FetchedGraph { + instance_id: row.instance_id, + graph_id: row.graph_id, + operator_pubkey: row.operator_pubkey, + graph_nonce: graph.parameters.graph_nonce, + finalized, + })) +} + +#[allow(clippy::too_many_arguments)] +async fn validate_and_store( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + lookup: &GraphLookup, + instance_id: Uuid, + graph_id: Uuid, + raw: String, +) -> anyhow::Result { + let graph: SimplifiedBitvmGcGraph = parse_graph_raw_data(raw, graph_id).await?; + if graph.parameters.graph_id != graph_id + || graph.parameters.instance_parameters.instance_id != instance_id + { + bail!("response identifiers do not match the definition"); + } + match lookup { + GraphLookup::Id { instance_id: wanted_instance, graph_id: wanted_graph, finalized } => { + if *wanted_instance != instance_id || *wanted_graph != graph_id { + bail!("response names another graph than requested"); + } + if *finalized && !graph.committee_pre_signed() { + bail!("response lacks the committee pre-signatures the lookup asks for"); + } + } + GraphLookup::Slot { operator_pubkey, graph_nonce } => { + if graph.parameters.operator_pubkey.to_string() != *operator_pubkey + || graph.parameters.graph_nonce != *graph_nonce + { + bail!("response names another slot than requested"); + } + } + } + if !graph.operator_pre_signed() { + bail!("definition lacks operator pre-signatures"); + } + let full_graph = BitvmGcGraph::from_simplified(&graph).context("rebuild pulled graph")?; + verify_graph_operator_pre_signatures(&full_graph).context("operator pre-signatures")?; + // Bind the instance parameters to both chains before trusting anything else. + validate_graph_instance_parameters( + btc_client, + goat_client, + &graph.parameters.instance_parameters, + ) + .await + .context("instance parameters")?; + let posted = goat_client.gateway_get_graph_data(&graph_id).await?.operator_pubkey != [0u8; 32]; + let finalized = if graph.committee_pre_signed() { + bitvm_lib::committee::verify_graph_committee_pre_signatures(&full_graph) + .context("committee pre-signatures")?; + true + } else { + if posted { + bail!( + "graph {graph_id} is posted on Goat; a copy without committee pre-signatures is not acceptable" + ); + } + false + }; + if finalized { + store_finalized_graph_if_needed(local_db, &graph).await?; + } else { + store_operator_presigned_graph(local_db, &graph).await?; + } + Ok(FetchedGraph { + instance_id, + graph_id, + operator_pubkey: graph.parameters.operator_pubkey.to_string(), + graph_nonce: graph.parameters.graph_nonce, + finalized, + }) +} diff --git a/node/src/aggregates/graph_jobs.rs b/node/src/aggregates/graph_jobs.rs new file mode 100644 index 000000000..19afcf06f --- /dev/null +++ b/node/src/aggregates/graph_jobs.rs @@ -0,0 +1,2323 @@ +//! Graph setup, validation, signing and publishing jobs. +//! +//! Jobs validate messages, query chain state and persist protocol material. +//! They return a [`GraphPlan`] for outgoing messages and follow-up work; +//! temporary failures are retried. + +use super::common::{PlannedMessage, delivery_id}; +use super::graph::{ + ALL_JOBS, GRAPH_KIND, GraphPlan, GraphStateUpdate, JOB_BUILD_GRAPH, JOB_COMMITTEE_ENDORSE, + JOB_COMMITTEE_INGEST_GRAPH, JOB_COMMITTEE_NONCES, JOB_COMMITTEE_PRESIGN, JOB_COMMITTEE_VOTE, + JOB_FETCH_GRAPH, JOB_FINALIZE_GRAPH, JOB_FREEZE_CANDIDATES, JOB_INGEST_ENDORSEMENT, + JOB_INGEST_FINALIZED, JOB_INGEST_GEN_CIRCUITS, JOB_INGEST_NONCES, JOB_INGEST_PRESIGN, + JOB_INGEST_SOLDERING_PROOF, JOB_INGEST_VERIFIER_ENDORSEMENT, JOB_INGEST_VOTE, + JOB_OPERATOR_INIT, JOB_POST_GRAPH_DATA, JOB_SEAL_PROOFS, JOB_VERIFIER_SETUP, + JOB_VERIFIER_SOLDER, JOB_VERIFIER_VALIDATE_GRAPH, OperatorStart, PHASE_COLLECT, PHASE_DELIVER, + PHASE_ENDORSE, PHASE_SETUP, PHASE_SIGNING, graph_pulled_event, graph_ref, outbox_id, retired, + slot_ref, +}; +use super::graph_fetch::{GraphLookup, fetch_and_store_graph}; +use super::graph_pegout::PEGOUT_JOBS; +use super::graph_pegout_jobs::run_pegout_step; +use super::graph_setup::{ + CandidateReadiness, build_babe_prover_state, build_signed_init_graph, + candidate_freeze_readiness, cut_circuits_owed, decode_soldering_proof_payload, + find_verifier_index_by_pubkey, freeze_operator_candidates, message_identity_matches, + record_candidate_gc_data, seal_selected_verifiers, selected_candidate_for_graph_index, + selected_gc_data, validate_verifier_slot, +}; +use super::jobs::{ + JobDeps, Plan, aggregate_uuid, author_peer, committee_pubkeys, decode_inbound, + ensure_committee_sender, gate_closed, instance_status, local_member, local_operator, + plan_outcome, signing_nonce_id, +}; +use crate::clock::current_time_secs; +use crate::engine_bridge::submit_event; +use crate::env::{ + get_babe_gc_asset_paths, get_bitvm_key, get_node_goat_address, get_peer_id, + get_soldering_proof_payload_store_path, get_verifier_candidate_collection_window_secs, +}; +use crate::error::{FailureClass, SpecialError, classify_failure}; +use crate::messages::{ + AggNonceConsensus, CommitteePresign, CreateGraph, CutCircuits, EndorseGraph, GOATMessage, + GOATMessageContent, GenCircuits, GraphFinalize, NonceGeneration, + VerifierGraphParamsEndorsement, verify_init_graph_signature, +}; +use crate::soldering_payload_store::{ + read_soldering_proof_store_payload, soldering_proof_payload_store_path, +}; +use crate::utils::{ + OperatorBabeSetupState, OperatorStakeStatus, VerifierBabeSetupState, + build_genesis_prekickoff_tx, build_graph_params, build_prekickoff_params, + derive_operator_static_input, endorse_graph, endorse_graph_params, + get_committee_agg_nonce_consensus_for_graph, get_committee_endorsements_for_graph, + get_committee_params_endorsements_for_graph, get_committee_partial_sigs_for_graph, + get_committee_partial_sigs_for_graph_member, get_committee_pub_nonces_for_graph, + get_current_prekickoff_tx, get_graph, get_guest_constant_value, get_instance_parameters, + get_verifier_graph_params_endorsements_for_graph, load_babe_setup_state, + mark_graph_as_endorsed, min_required_verifier, operator_stake_status, order_committee_values, + pending_graph_belongs_to_operator, refresh_graph, save_babe_setup_state, + save_soldering_proof_payload, sign_verifier_graph_params, + store_committee_agg_nonce_consensus_for_graph, store_committee_endorsement_for_graph, + store_committee_endorsements_for_graph, store_committee_partial_sigs_for_graph, + store_committee_pub_nonces_for_graph, store_finalized_graph_if_needed, + store_operator_presigned_graph, store_verifier_graph_params_endorsement, + update_babe_setup_state, validate_committee_with_evm_address, validate_finalized_graph, + validate_init_graph, validate_init_graph_base, validate_verifier_graph_params_endorsements, + verify_graph_endorsement, verify_graph_params_endorsement, + verify_verifier_graph_params_endorsement, +}; +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use bitcoin::hashes::Hash; +use bitcoin::{PublicKey, XOnlyPublicKey}; +use bitvm_lib::actors::Actor; +use bitvm_lib::babe_adapter::{ + BABE_M_CC, BABE_N_CC, build_real_setup_package, expand_compact_soldering_proof_payload, + extract_gc_circuit_data, open_real_setup_and_solder, verify_real_setup, +}; +use bitvm_lib::committee::{ + committee_nonce_consensus_hash, committee_pre_sign, nonces_aggregation, + push_committee_pre_signatures, signature_aggregation, verify_graph_committee_partial_sigs, + verify_nonce_signatures, +}; +use bitvm_lib::keys::{CommitteeMasterKey, VerifierMasterKey}; +use bitvm_lib::operator::{ + generate_bitvm_graph, operator_pre_sign, verify_graph_operator_pre_signatures, +}; +use bitvm_lib::types::{BitvmGcGraph, SimplifiedBitvmGcGraph}; +use client::goat_chain::PeginStatus; +use engine::{EngineBuilder, JobContext, JobOutcome, JobRunner}; +use goat::transactions::pre_signed::PreSignedTransaction; +use libp2p::PeerId; +use secp256k1::{Message as SecpMessage, SECP256K1}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use std::sync::Arc; +use store::{GraphStatus, SigningAuthorization}; +use tracing::{info, warn}; +use uuid::Uuid; + +pub fn register_graph_jobs(builder: EngineBuilder, deps: Arc) -> EngineBuilder { + let runner: Arc = Arc::new(GraphJobRunner { deps }); + let mut builder = builder; + for action in ALL_JOBS.iter().chain(PEGOUT_JOBS) { + builder = builder.job_runner(GRAPH_KIND, *action, runner.clone()); + } + builder +} + +pub(super) type Step = super::jobs::Step; + +struct GraphJobRunner { + deps: Arc, +} + +#[async_trait] +impl JobRunner for GraphJobRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let graph_id = match aggregate_uuid(&ctx, "a graph") { + Ok(graph_id) => graph_id, + Err(outcome) => return outcome, + }; + let params = ctx.job.params.clone().unwrap_or_default(); + let action = ctx.job.action.clone(); + let attempts = ctx.job.attempts; + let pegout = PEGOUT_JOBS.contains(&action.as_str()); + match run_step(&self.deps, &action, graph_id, ¶ms, attempts).await { + Ok(step) => step.into_outcome(), + Err(error) => match classify_failure(&error) { + FailureClass::Permanent(reason) if pegout => { + // A Goat revert on a peg-out call: retrying cannot help. + warn!(event = "graph_job", graph_id = %graph_id, action = %action, reason = %reason, "peg-out action rejected"); + self.deps.metrics_state.record_instance_alert("pegout_action_rejected"); + plan_outcome(&GraphPlan::noted(format!("rejected: {reason}"))) + } + FailureClass::Permanent(reason) => { + warn!(event = "graph_job", graph_id = %graph_id, action = %action, reason = %reason, "message rejected"); + self.deps.metrics_state.record_graph_validation(false); + plan_outcome(&GraphPlan::noted(format!("rejected: {reason}"))) + } + FailureClass::Transient => { + JobOutcome::Transient { error: format!("{error:#}"), retry_after_secs: None } + } + // A setup step failing for no recognised reason is given up on + // after a while: its session and slot are released and the + // failure is reported. Peg-out steps guard funds and keep + // retrying, with an alert every so often. + FailureClass::Unknown if !pegout && attempts >= UNKNOWN_FAILURE_RETRIES => { + warn!(event = "graph_job", graph_id = %graph_id, action = %action, attempts, error = %format!("{error:#}"), "setup step keeps failing for an unknown reason; giving up"); + self.deps.metrics_state.record_instance_alert("graph_job_gave_up"); + JobOutcome::Failed(format!("gave up after {attempts} attempts: {error:#}")) + } + FailureClass::Unknown => { + if attempts >= UNKNOWN_FAILURE_RETRIES + && attempts % UNKNOWN_FAILURE_RETRIES == 0 + { + warn!(event = "graph_job", graph_id = %graph_id, action = %action, attempts, error = %format!("{error:#}"), "peg-out step keeps failing for an unknown reason"); + self.deps.metrics_state.record_instance_alert("pegout_job_stuck"); + } + JobOutcome::Transient { error: format!("{error:#}"), retry_after_secs: None } + } + }, + } + } +} + +/// Attempts a step failing for an unrecognised reason gets before a setup +/// step is failed and a peg-out step raises an alert (then every so many). +const UNKNOWN_FAILURE_RETRIES: i64 = 20; + +async fn run_step( + deps: &JobDeps, + action: &str, + graph_id: Uuid, + params: &[u8], + attempts: i64, +) -> Result { + match action { + JOB_OPERATOR_INIT => operator_init_graph(deps, graph_id, params).await, + JOB_VERIFIER_SETUP => verifier_setup(deps, graph_id, params).await, + JOB_INGEST_GEN_CIRCUITS => ingest_gen_circuits(deps, graph_id, params).await, + JOB_FREEZE_CANDIDATES => freeze_candidates(deps, graph_id, params).await, + JOB_VERIFIER_SOLDER => verifier_solder(deps, graph_id, params).await, + JOB_INGEST_SOLDERING_PROOF => ingest_soldering_proof(deps, graph_id, params).await, + JOB_SEAL_PROOFS => seal_proofs(deps, graph_id, params).await, + JOB_BUILD_GRAPH => build_graph(deps, graph_id, params).await, + JOB_VERIFIER_VALIDATE_GRAPH => verifier_validate_graph(deps, graph_id, params).await, + JOB_COMMITTEE_INGEST_GRAPH => committee_ingest_graph(deps, graph_id, params).await, + JOB_INGEST_VERIFIER_ENDORSEMENT => { + ingest_verifier_endorsement(deps, graph_id, params).await + } + JOB_COMMITTEE_NONCES => committee_nonces(deps, graph_id, params).await, + JOB_INGEST_NONCES => ingest_nonces(deps, graph_id, params).await, + JOB_COMMITTEE_VOTE => committee_vote(deps, graph_id, params).await, + JOB_INGEST_VOTE => ingest_vote(deps, graph_id, params).await, + JOB_COMMITTEE_PRESIGN => committee_presign(deps, graph_id, params).await, + JOB_INGEST_PRESIGN => ingest_presign(deps, graph_id, params).await, + JOB_COMMITTEE_ENDORSE => committee_endorse(deps, graph_id, params).await, + JOB_INGEST_ENDORSEMENT => ingest_endorsement(deps, graph_id, params).await, + JOB_FINALIZE_GRAPH => finalize_graph(deps, graph_id, params).await, + JOB_INGEST_FINALIZED => ingest_finalized(deps, graph_id, params).await, + JOB_POST_GRAPH_DATA => post_graph_data(deps, graph_id, params).await, + JOB_FETCH_GRAPH => fetch_graph(deps, graph_id, params, attempts).await, + pegout if PEGOUT_JOBS.contains(&pegout) => { + run_pegout_step(deps, pegout, graph_id, params, attempts).await + } + other => bail!("unknown graph job {other}"), + } +} + +// ------------------------------------------------------------------ helpers + +/// Serialize an outbound message and plan it, computing the ack set for the +/// given recipients. +async fn planned( + graph_id: Uuid, + phase_ref: &str, + actor: Actor, + content: GOATMessageContent, + ack_peers: &[String], + qualifier: Option<&str>, +) -> Result { + let kind = content.event_type(); + let payload = GOATMessage::new(actor.clone(), content).serialize_message().await?; + let ack_set = if ack_peers.is_empty() { + None + } else { + let sender = get_peer_id(); + Some( + ack_peers + .iter() + .filter(|peer| **peer != sender) + .map(|peer| (peer.clone(), delivery_id(&sender, peer, &payload))) + .collect(), + ) + }; + Ok(PlannedMessage { + outbox_id: outbox_id(graph_id, kind, qualifier), + phase_ref: phase_ref.to_string(), + kind: kind.to_string(), + actor: actor.to_string(), + payload, + ack_set, + }) +} + +/// Committee peers from the registry, for the ack sets of the operator's +/// heavy messages. +fn committee_ack_peers() -> Vec { + crate::p2p_admission::peer_registry() + .committee_peer_ids(std::time::Instant::now()) + .into_iter() + .map(|peer| peer.to_string()) + .collect() +} + +/// Counts of committee endorsements, parameter endorsements, nonces and partials. +struct Collections { + endorsements: usize, + params_endorsements: usize, + pub_nonces: usize, + partial_sigs: usize, +} + +async fn collections(deps: &JobDeps, instance_id: Uuid, graph_id: Uuid) -> Result { + Ok(Collections { + endorsements: get_committee_endorsements_for_graph(&deps.local_db, instance_id, graph_id) + .await? + .len(), + params_endorsements: get_committee_params_endorsements_for_graph( + &deps.local_db, + instance_id, + graph_id, + ) + .await? + .len(), + pub_nonces: get_committee_pub_nonces_for_graph(&deps.local_db, instance_id, graph_id) + .await? + .len(), + partial_sigs: get_committee_partial_sigs_for_graph(&deps.local_db, instance_id, graph_id) + .await? + .len(), + }) +} + +async fn four_sets_complete(deps: &JobDeps, instance_id: Uuid, graph_id: Uuid) -> Result { + let committee = committee_pubkeys(deps, instance_id).await?.len(); + let collected = collections(deps, instance_id, graph_id).await?; + Ok(collected.endorsements == committee + && collected.params_endorsements == committee + && collected.pub_nonces == committee + && collected.partial_sigs == committee) +} + +fn invalid(message: impl Into) -> Result { + Err(SpecialError::InvalidGraph(message.into()).into()) +} + +// ----------------------------------------------------------------- operator + +async fn operator_init_graph(_deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let start: OperatorStart = bincode::deserialize(params).context("decode operator_start")?; + let (master, operator_pubkey) = local_operator()?; + let init = build_signed_init_graph(start.instance_id, graph_id, &master.master_keypair())?; + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SETUP, + Actor::Verifier, + GOATMessageContent::InitGraph(init), + &[], + None, + ) + .await?, + ); + plan.state = GraphStateUpdate { + instance_id: Some(start.instance_id), + operator_pubkey: Some(operator_pubkey.to_string()), + ..Default::default() + }; + Step::done(plan) +} + +async fn ingest_gen_circuits(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::GenCircuits(GenCircuits { + instance_id, + graph_id: message_graph_id, + verifier_pubkey, + setup_package, + }) = message.content + else { + return invalid("not a GenCircuits message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("GenCircuits identity mismatch"); + } + let (_, local_operator_pubkey) = local_operator()?; + if !pending_graph_belongs_to_operator( + &deps.local_db, + instance_id, + graph_id, + &local_operator_pubkey, + ) + .await? + { + return Step::ignore("no local pending graph session"); + } + if setup_package.commits.len() != BABE_N_CC { + return invalid(format!( + "invalid GenCircuits setup package commitment count: expected {BABE_N_CC}, got {}", + setup_package.commits.len() + )); + } + let verifier_peer = author_peer(&inbound)?; + let verifier_peer_id = verifier_peer.to_bytes(); + if !crate::p2p_admission::peer_registry().is_verifier(&verifier_peer) + && !deps.goat_client.committee_mana_is_verifier(&verifier_peer_id).await.with_context( + || format!("validate GenCircuits sender {verifier_peer} against the verifier registry"), + )? + { + return Step::ignore(format!("sender {verifier_peer} is not a registered verifier")); + } + + let mut state = + load_babe_setup_state(&deps.local_db, instance_id, graph_id)?.unwrap_or_default(); + let operator_state = state.operator.get_or_insert_with(|| OperatorBabeSetupState { + candidate_verifier_pubkeys: None, + candidates: vec![], + candidate_collection_started_at: None, + proof_collection_started_at: None, + selected_verifier_pubkeys: None, + asserted_operator_proof: None, + }); + let was_frozen = operator_state.candidate_verifier_pubkeys.is_some(); + if let Some(existing) = operator_state + .candidates + .iter() + .find(|candidate| candidate.verifier_peer_id == verifier_peer_id) + { + if existing.verifier_pubkey != verifier_pubkey { + return Step::ignore(format!( + "verifier peer {verifier_peer} already submitted a different public key" + )); + } + if existing.setup_package != setup_package { + return invalid(format!( + "conflicting GenCircuits setup package for verifier {verifier_pubkey}" + )); + } + } else if operator_state.candidates.iter().any(|candidate| { + XOnlyPublicKey::from(candidate.verifier_pubkey) == XOnlyPublicKey::from(verifier_pubkey) + }) { + return Step::ignore(format!( + "verifier public key {verifier_pubkey} is already bound to another peer" + )); + } else if was_frozen { + return Step::ignore("verifier membership is frozen"); + } else { + if operator_state.candidates.len() + >= min_required_verifier() + .saturating_add(crate::env::get_verifier_candidate_backup_count()) + { + return Step::ignore("candidate collection reached its configured limit"); + } + operator_state.candidates.push(crate::utils::OperatorVerifierCandidate { + verifier_peer_id, + verifier_pubkey, + setup_package, + candidate_index: None, + selected_circuit_indexes: vec![], + gc_data: None, + soldering_proof_ready: None, + }); + operator_state.candidate_collection_started_at.get_or_insert_with(current_time_secs); + } + let candidate_count = operator_state.candidates.len(); + let readiness = candidate_freeze_readiness(operator_state, current_time_secs()); + let mut plan = GraphPlan::default(); + match readiness { + CandidateReadiness::Frozen | CandidateReadiness::Ready => { + freeze_operator_candidates(operator_state)?; + plan_cut_circuits(graph_id, instance_id, operator_state, &mut plan).await?; + } + CandidateReadiness::Wait(secs) => { + plan.then( + JOB_FREEZE_CANDIDATES, + bincode::serialize(&FreezeParams { + candidate_count, + due_at: current_time_secs() + secs, + })?, + Some(secs), + ); + plan.note = Some(format!("collecting verifier candidates: {candidate_count} received")); + } + CandidateReadiness::Short => { + plan.note = Some(format!( + "collecting verifier candidates: {candidate_count}/{} received", + min_required_verifier() + )); + } + } + save_babe_setup_state(&deps.local_db, instance_id, graph_id, &state)?; + plan.state.instance_id = Some(instance_id); + Step::done(plan) +} + +/// Params of a delayed job: distinct per schedule, since a job id is derived +/// from its params and an existing id is not enqueued again. +#[derive(Serialize, Deserialize)] +struct FreezeParams { + candidate_count: usize, + due_at: i64, +} + +async fn freeze_candidates(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let _: FreezeParams = bincode::deserialize(params).context("decode freeze params")?; + let Some(instance_id) = pending_session(deps, graph_id).await? else { + return Step::ignore("no pending graph session"); + }; + let mut state = + load_babe_setup_state(&deps.local_db, instance_id, graph_id)?.unwrap_or_default(); + let Some(operator_state) = state.operator.as_mut() else { + return Step::ignore("no operator setup state"); + }; + let mut plan = GraphPlan::default(); + match candidate_freeze_readiness(operator_state, current_time_secs()) { + CandidateReadiness::Frozen | CandidateReadiness::Ready => { + freeze_operator_candidates(operator_state)?; + plan_cut_circuits(graph_id, instance_id, operator_state, &mut plan).await?; + save_babe_setup_state(&deps.local_db, instance_id, graph_id, &state)?; + } + CandidateReadiness::Wait(secs) => { + plan.then( + JOB_FREEZE_CANDIDATES, + bincode::serialize(&FreezeParams { + candidate_count: operator_state.candidates.len(), + due_at: current_time_secs() + secs, + })?, + Some(secs), + ); + } + CandidateReadiness::Short => { + plan.note = Some("not enough verifier candidates yet".to_string()); + } + } + Step::done(plan) +} + +/// The operator's pending session for a graph: `(instance_id, operator_pubkey)`. +/// The instance of the graph's pending setup session, if any. +async fn pending_session(deps: &JobDeps, graph_id: Uuid) -> Result> { + Ok(deps + .local_db + .acquire() + .await? + .find_pending_graph_init_by_graph_id(&graph_id) + .await? + .map(|pending| pending.instance_id)) +} + +/// CutCircuits for every frozen candidate still owed one, closing InitGraph. +async fn plan_cut_circuits( + graph_id: Uuid, + instance_id: Uuid, + operator_state: &OperatorBabeSetupState, + plan: &mut GraphPlan, +) -> Result<()> { + for candidate in cut_circuits_owed(operator_state, current_time_secs()) { + let verifier_peer = PeerId::from_bytes(&candidate.verifier_peer_id) + .context("decode candidate verifier peer id")? + .to_string(); + let content = GOATMessageContent::CutCircuits(CutCircuits { + instance_id, + graph_id, + verifier_pubkey: candidate.verifier_pubkey, + candidate_index: candidate.candidate_index.context("frozen candidate has no slot")?, + selected_circuit_indexes: candidate.selected_circuit_indexes.clone(), + }); + let qualifier = candidate.verifier_pubkey.to_string(); + plan.send( + planned( + graph_id, + PHASE_SETUP, + Actor::Verifier, + content, + &[verifier_peer], + Some(&qualifier), + ) + .await?, + ); + } + plan.close_rows.push(outbox_id(graph_id, "InitGraph", None)); + Ok(()) +} + +async fn ingest_soldering_proof(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::SolderingProofReady(soldering_proof_ready) = message.content else { + return invalid("not a SolderingProofReady message"); + }; + let instance_id = soldering_proof_ready.instance_id; + let candidate_index = soldering_proof_ready.candidate_index; + if soldering_proof_ready.graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("SolderingProofReady identity mismatch"); + } + if soldering_proof_ready.total_len == 0 { + return invalid("SolderingProofReady total_len must be greater than zero"); + } + let (_, local_operator_pubkey) = local_operator()?; + if !pending_graph_belongs_to_operator( + &deps.local_db, + instance_id, + graph_id, + &local_operator_pubkey, + ) + .await? + { + return Step::ignore("no local pending graph session"); + } + let mut state = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .ok_or_else(|| anyhow!("missing BABE setup state for pending graph {graph_id}"))?; + let operator_state = state + .operator + .as_mut() + .ok_or_else(|| anyhow!("missing operator BABE setup state for pending graph {graph_id}"))?; + let Some(frozen) = operator_state.candidate_verifier_pubkeys.clone() else { + return Step::retry("operator verifier membership is not frozen", 30); + }; + let verifier_pubkey = *frozen.get(candidate_index).ok_or_else(|| { + anyhow!("SolderingProofReady candidate index {candidate_index} out of range") + })?; + let candidate = operator_state + .candidates + .iter() + .find(|candidate| candidate.verifier_pubkey == verifier_pubkey) + .ok_or_else(|| anyhow!("selected verifier candidate is missing"))?; + if candidate.candidate_index != Some(candidate_index) { + return invalid( + "selected verifier candidate index does not match SolderingProofReady slot", + ); + } + if operator_state + .selected_verifier_pubkeys + .as_ref() + .is_some_and(|selected| !selected.contains(&verifier_pubkey)) + { + return Step::ignore( + "late SolderingProofReady from a verifier outside the sealed selection", + ); + } + let verifier_peer_id = author_peer(&inbound)?.to_bytes(); + if candidate.verifier_peer_id != verifier_peer_id { + return Step::ignore(format!( + "sender {} does not own verifier candidate {candidate_index}", + inbound.author + )); + } + if candidate.gc_data.is_some() { + // Already verified; only the collection window decides what is next. + return proof_collection_step(deps, graph_id, instance_id, &mut state).await; + } + + let store_base_path = get_soldering_proof_payload_store_path()?; + let payload_path = soldering_proof_payload_store_path( + &store_base_path, + instance_id, + graph_id, + candidate_index, + &soldering_proof_ready.payload_hash, + )?; + let payload = match read_soldering_proof_store_payload(&payload_path).await { + Ok(payload) => payload, + Err(error) => { + return Step::retry(format!("read soldering proof payload from store: {error}"), 30); + } + }; + let payload = decode_soldering_proof_payload(&soldering_proof_ready, &payload)?; + let setup_package = candidate.setup_package.clone(); + let claimed_finalized_indices = candidate.selected_circuit_indexes.clone(); + let (opened, finalized, soldering) = expand_compact_soldering_proof_payload(payload) + .context("expand compact soldering proof payload")?; + let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for BABE validation")?; + let static_input = derive_operator_static_input()?; + let soldering_builder = Arc::clone( + deps.soldering_builder + .as_ref() + .context("BABE soldering builder is not initialized for Operator")?, + ); + #[cfg(feature = "mock-gc")] + crate::mock_gc::simulate(crate::mock_gc::GcOp::Verify).await; + let verification = { + let package = setup_package.clone(); + let opened = opened.clone(); + let finalized = finalized.clone(); + let soldering = soldering.clone(); + tokio::task::spawn_blocking(move || { + verify_real_setup( + &soldering_builder, + &package, + &opened, + &finalized, + &soldering, + &vk, + &claimed_finalized_indices, + static_input, + ) + }) + .await + .context("real BABE setup verification task failed")? + }; + if let Err(error) = verification { + deps.metrics_state.record_pegin_graph_setup(false); + return invalid(format!("soldering proof verification failed: {error}")); + } + if finalized.len() != BABE_M_CC { + return invalid(format!( + "each verifier must contribute exactly {BABE_M_CC} finalized BABE instances" + )); + } + let epk = &setup_package.commits[finalized[0].index].epk; + let prover_state = build_babe_prover_state(&setup_package, finalized, soldering)?; + let gc_data = extract_gc_circuit_data(verifier_pubkey, epk, &prover_state.h_msgs)?; + record_candidate_gc_data( + operator_state, + verifier_pubkey, + candidate_index, + &setup_package, + gc_data, + &prover_state, + soldering_proof_ready.clone(), + )?; + deps.metrics_state.record_pegin_graph_setup(true); + info!( + event = "operator_graph_creation", + outcome = "soldering_verified", + graph_id = %graph_id, + candidate_index, + verifier_pubkey = %verifier_pubkey, + "verified verifier soldering proof" + ); + operator_state.proof_collection_started_at.get_or_insert_with(current_time_secs); + proof_collection_step(deps, graph_id, instance_id, &mut state).await +} + +/// See [`FreezeParams`]. +#[derive(Serialize, Deserialize)] +struct SealParams { + completed: usize, + due_at: i64, +} + +/// Persist the operator state and decide whether the proofs can be sealed now, +/// later (a delayed `seal_proofs`), or not yet. +async fn proof_collection_step( + deps: &JobDeps, + graph_id: Uuid, + instance_id: Uuid, + state: &mut crate::utils::BabeSetupState, +) -> Result { + let operator_state = state.operator.as_mut().context("operator setup state")?; + let completed = + operator_state.candidates.iter().filter(|candidate| candidate.gc_data.is_some()).count(); + let required = min_required_verifier(); + let started_at = + *operator_state.proof_collection_started_at.get_or_insert_with(current_time_secs); + save_babe_setup_state(&deps.local_db, instance_id, graph_id, state)?; + let elapsed = current_time_secs() - started_at; + let window = get_verifier_candidate_collection_window_secs(); + let mut plan = GraphPlan::default(); + plan.state.instance_id = Some(instance_id); + if completed < required { + plan.note = Some(format!("collecting verified soldering proofs: {completed}/{required}")); + } else if elapsed < window { + let remaining = (window - elapsed).max(1); + plan.then( + JOB_SEAL_PROOFS, + bincode::serialize(&SealParams { completed, due_at: current_time_secs() + remaining })?, + Some(remaining), + ); + plan.note = + Some(format!("{completed}/{required} proofs verified; sealing in {remaining}s")); + } else { + plan.then(JOB_BUILD_GRAPH, Vec::new(), None); + } + Step::done(plan) +} + +async fn seal_proofs(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let _: SealParams = bincode::deserialize(params).context("decode seal params")?; + let Some(instance_id) = pending_session(deps, graph_id).await? else { + return Step::ignore("no pending graph session"); + }; + let mut state = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .ok_or_else(|| anyhow!("missing BABE setup state for pending graph {graph_id}"))?; + if state.operator.is_none() { + return Step::ignore("no operator setup state"); + } + proof_collection_step(deps, graph_id, instance_id, &mut state).await +} + +async fn build_graph(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + let Some(instance_id) = pending_session(deps, graph_id).await? else { + return Step::ignore("no pending graph session"); + }; + let (operator_master_key, local_operator_pubkey) = local_operator()?; + let mut state = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .ok_or_else(|| anyhow!("missing BABE setup state for pending graph {graph_id}"))?; + let operator_state = state + .operator + .as_mut() + .ok_or_else(|| anyhow!("missing operator BABE setup state for pending graph {graph_id}"))?; + let required_slots = min_required_verifier(); + let selected_verifier_pubkeys = match &operator_state.selected_verifier_pubkeys { + Some(selected) => selected.clone(), + None => { + let mut selected = operator_state + .candidates + .iter() + .filter(|candidate| candidate.gc_data.is_some()) + .map(|candidate| candidate.verifier_pubkey) + .collect::>(); + selected.sort_by_key(|candidate| candidate.to_bytes()); + selected.truncate(required_slots); + selected + } + }; + seal_selected_verifiers(operator_state, selected_verifier_pubkeys)?; + let bitvm_gc_circuit_datas = selected_gc_data(operator_state)?; + let selected_verifier_count = operator_state + .selected_verifier_pubkeys + .as_ref() + .context("selected verifier set is missing after proof collection")? + .len(); + let selected_verifier_peer_ids = (0..selected_verifier_count) + .map(|verifier_index| { + let candidate = selected_candidate_for_graph_index(operator_state, verifier_index)?; + PeerId::from_bytes(&candidate.verifier_peer_id) + .context("decode selected verifier peer id for CreateGraph ACK") + .map(|peer_id| peer_id.to_string()) + }) + .collect::>>()?; + let candidate_pubkeys = operator_state + .candidates + .iter() + .map(|candidate| candidate.verifier_pubkey) + .collect::>(); + save_babe_setup_state(&deps.local_db, instance_id, graph_id, &state)?; + + // Reuse an existing graph for this id and resume CreateGraph delivery. + let (graph, already_finalized) = match get_graph(&deps.local_db, instance_id, graph_id).await? { + Some(stored) => { + let already_finalized = BitvmGcGraph::from_simplified(&stored)?.committee_pre_signed(); + info!(event = "operator_graph_creation", outcome = "resumed", graph_id = %graph_id, already_finalized, "graph was stored by an earlier run"); + (stored, already_finalized) + } + None => { + let instance_params = get_instance_parameters(&deps.local_db, instance_id) + .await? + .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; + let (graph_nonce, cur_prekickoff_txn) = + match get_current_prekickoff_tx(&deps.local_db, &local_operator_pubkey).await? { + Some((graph_nonce, prekickoff_tx)) => (graph_nonce, prekickoff_tx), + None => ( + 0, + build_genesis_prekickoff_tx(&deps.btc_client, &deps.goat_client).await?, + ), + }; + let prekickoff_params = + build_prekickoff_params(&deps.btc_client, graph_nonce, cur_prekickoff_txn).await?; + let mut graph_params = build_graph_params( + &deps.local_db, + &deps.goat_client, + instance_params, + prekickoff_params, + bitvm_gc_circuit_datas, + graph_nonce, + graph_id, + ) + .await?; + let challenge_init_txid = generate_bitvm_graph(graph_params.clone())? + .watchtower_challenge_init + .tx() + .compute_txid() + .to_byte_array(); + graph_params.pubin_disprove_constant = get_guest_constant_value( + graph_id, + challenge_init_txid, + &graph_params.watchtower_pubkeys, + )?; + let mut graph = generate_bitvm_graph(graph_params)?; + anyhow::ensure!( + graph.watchtower_challenge_init.tx().compute_txid().to_byte_array() + == challenge_init_txid, + "Operator constant unexpectedly changes the watchtower challenge init transaction" + ); + operator_pre_sign(operator_master_key.master_keypair(), &mut graph)?; + let graph = graph.to_simplified()?; + store_operator_presigned_graph(&deps.local_db, &graph).await?; + info!(event = "operator_graph_creation", outcome = "committed", graph_id = %graph_id, graph_nonce, "stored operator-pre-signed graph definition"); + (graph, false) + } + }; + let graph_nonce = graph.parameters.graph_nonce; + + let mut plan = GraphPlan::default(); + if !already_finalized { + let mut ack_peers = selected_verifier_peer_ids; + ack_peers.extend(committee_ack_peers()); + plan.send( + planned( + graph_id, + PHASE_COLLECT, + Actor::All, + GOATMessageContent::CreateGraph(CreateGraph { + instance_id, + graph_id, + graph_nonce, + graph, + }), + &ack_peers, + None, + ) + .await?, + ); + } + plan.close_rows.push(outbox_id(graph_id, "InitGraph", None)); + for pubkey in candidate_pubkeys { + plan.close_rows.push(outbox_id(graph_id, "CutCircuits", Some(&pubkey.to_string()))); + } + plan.delete_pending_session = true; + plan.wake_refs.push(slot_ref(&local_operator_pubkey.to_string(), graph_nonce)); + plan.state = GraphStateUpdate { + instance_id: Some(instance_id), + operator_pubkey: Some(local_operator_pubkey.to_string()), + graph_nonce: Some(graph_nonce), + finalized: Some(already_finalized), + }; + Step::done(plan) +} + +async fn ingest_nonces(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::NonceGeneration(NonceGeneration { + instance_id, + graph_id: message_graph_id, + committee_pubkey, + pub_nonces, + nonce_sigs, + }) = message.content + else { + return invalid("not a NonceGeneration message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("NonceGeneration identity mismatch"); + } + ensure_committee_sender(deps, &inbound, &committee_pubkey).await?; + if deps.actor == Actor::Committee && local_member(deps, instance_id).await?.is_none() { + return Step::ignore("not in this instance's committee"); + } + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::retry("graph definition not stored yet", 30); + }; + let watchtower_num = graph.parameters.watchtower_pubkeys.len(); + let verifier_num = graph.parameters.gc_data.len(); + if !verify_nonce_signatures( + &XOnlyPublicKey::from(committee_pubkey), + &pub_nonces, + &nonce_sigs, + watchtower_num, + verifier_num, + )? { + return invalid(format!("invalid pub_nonces or nonce_sigs from {committee_pubkey}")); + } + pub_nonces.validate_length(watchtower_num, verifier_num).map_err(|error| { + SpecialError::InvalidGraph(format!("invalid pub_nonces length: {error}")) + })?; + store_committee_pub_nonces_for_graph( + &deps.local_db, + instance_id, + graph_id, + committee_pubkey, + pub_nonces, + ) + .await?; + after_committee_value(deps, instance_id, graph_id, &graph).await +} + +/// What follows a stored committee value: the operator finalizes once the four +/// collections are complete; a committee member votes once every nonce is in +/// and pre-signs once every vote agrees. +async fn after_committee_value( + deps: &JobDeps, + instance_id: Uuid, + graph_id: Uuid, + graph: &SimplifiedBitvmGcGraph, +) -> Result { + let mut plan = GraphPlan::default(); + plan.state.instance_id = Some(instance_id); + plan.state.operator_pubkey = Some(graph.parameters.operator_pubkey.to_string()); + plan.state.graph_nonce = Some(graph.parameters.graph_nonce); + match deps.actor { + Actor::Operator => { + if four_sets_complete(deps, instance_id, graph_id).await? { + plan.then(JOB_FINALIZE_GRAPH, Vec::new(), None); + } + } + Actor::Committee => { + let committee = committee_pubkeys(deps, instance_id).await?; + let pub_nonces = + get_committee_pub_nonces_for_graph(&deps.local_db, instance_id, graph_id) + .await? + .len(); + if pub_nonces >= committee.len() { + plan.then(JOB_COMMITTEE_VOTE, Vec::new(), None); + } + } + _ => {} + } + Step::done(plan) +} + +async fn ingest_presign(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::CommitteePresign(CommitteePresign { + instance_id, + graph_id: message_graph_id, + committee_pubkey, + committee_partial_sigs, + agg_nonces: received_agg_nonces, + }) = message.content + else { + return invalid("not a CommitteePresign message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("CommitteePresign identity mismatch"); + } + ensure_committee_sender(deps, &inbound, &committee_pubkey).await?; + if deps.actor == Actor::Committee && local_member(deps, instance_id).await?.is_none() { + return Step::ignore("not in this instance's committee"); + } + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::retry("graph definition not stored yet", 30); + }; + let Some(consensus) = nonce_consensus(deps, instance_id, graph_id, &graph).await? else { + return Step::retry("waiting for committee public nonces", 30); + }; + let NonceConsensus { full_graph, committee, pub_nonces, agg_nonces, .. } = consensus; + let Some(received_pub_nonces) = committee + .iter() + .position(|pubkey| *pubkey == committee_pubkey) + .map(|index| pub_nonces[index].clone()) + else { + return invalid(format!("missing signer pub nonces for {committee_pubkey}")); + }; + if agg_nonces != received_agg_nonces { + return invalid(format!("agg_nonces mismatch from {committee_pubkey}")); + } + if let Err(error) = verify_graph_committee_partial_sigs( + &full_graph, + &committee, + &committee_pubkey, + &received_pub_nonces, + &agg_nonces, + &committee_partial_sigs, + ) { + return invalid(format!("invalid partial signatures from {committee_pubkey}: {error}")); + } + store_committee_partial_sigs_for_graph( + &deps.local_db, + instance_id, + graph_id, + committee_pubkey, + committee_partial_sigs, + ) + .await?; + match deps.actor { + Actor::Committee => { + let partials = + get_committee_partial_sigs_for_graph(&deps.local_db, instance_id, graph_id) + .await? + .len(); + let mut plan = GraphPlan::default(); + plan.state.instance_id = Some(instance_id); + if partials >= committee.len() { + plan.then(JOB_COMMITTEE_ENDORSE, Vec::new(), None); + } + Step::done(plan) + } + _ => after_committee_value(deps, instance_id, graph_id, &graph).await, + } +} + +async fn ingest_endorsement(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::EndorseGraph(EndorseGraph { + instance_id, + graph_id: message_graph_id, + committee_pubkey, + committee_sig_for_graph, + committee_sig_for_params, + committee_evm_address, + }) = message.content + else { + return invalid("not an EndorseGraph message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("EndorseGraph identity mismatch"); + } + let peer = author_peer(&inbound)?; + validate_committee_with_evm_address( + &deps.goat_client, + &peer, + instance_id, + &committee_pubkey, + &committee_evm_address, + ) + .await?; + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::ignore("graph not found, maybe belongs to another operator"); + }; + let full_graph = BitvmGcGraph::from_simplified(&graph)?; + if let Err(error) = verify_graph_endorsement( + &deps.goat_client, + &committee_evm_address, + &full_graph, + &committee_sig_for_graph, + ) + .await + { + return invalid(format!("invalid endorsement signature from {committee_pubkey}: {error}")); + } + match verify_graph_params_endorsement( + &committee_evm_address, + &full_graph, + &committee_sig_for_params, + ) { + Ok(true) => {} + Ok(false) => { + return invalid(format!( + "invalid params endorsement signature from {committee_pubkey}" + )); + } + Err(error) => { + return invalid(format!( + "invalid params endorsement signature from {committee_pubkey}: {error}" + )); + } + } + store_committee_endorsement_for_graph( + &deps.local_db, + instance_id, + graph_id, + committee_pubkey, + committee_evm_address, + committee_sig_for_graph, + committee_sig_for_params, + ) + .await?; + after_committee_value(deps, instance_id, graph_id, &graph).await +} + +async fn finalize_graph(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + let Some(row) = deps.local_db.acquire().await?.find_graph(&graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + // Given up at the abandon height while the signatures were still being + // collected: the slot is left for the Skip, nothing is finalized. + if retired(GraphStatus::from_str(&row.status).ok()) { + return Step::ignore(format!("graph is {}", row.status)); + } + let instance_id = row.instance_id; + let endorsements = + get_committee_endorsements_for_graph(&deps.local_db, instance_id, graph_id).await?; + let params_endorsements = + get_committee_params_endorsements_for_graph(&deps.local_db, instance_id, graph_id).await?; + let pub_nonces = + get_committee_pub_nonces_for_graph(&deps.local_db, instance_id, graph_id).await?; + let partial_sigs = + get_committee_partial_sigs_for_graph(&deps.local_db, instance_id, graph_id).await?; + let committee = committee_pubkeys(deps, instance_id).await?; + if endorsements.len() != committee.len() + || params_endorsements.len() != committee.len() + || pub_nonces.len() != committee.len() + || partial_sigs.len() != committee.len() + { + return Step::ignore(format!( + "collections incomplete: endorsements {}/{} params {}/{} nonces {}/{} partials {}/{}", + endorsements.len(), + committee.len(), + params_endorsements.len(), + committee.len(), + pub_nonces.len(), + committee.len(), + partial_sigs.len(), + committee.len() + )); + } + let stored = get_graph(&deps.local_db, instance_id, graph_id) + .await? + .ok_or_else(|| anyhow!("Graph not found for {instance_id}:{graph_id}"))?; + let mut graph = BitvmGcGraph::from_simplified(&stored)?; + if graph.parameters.instance_parameters.instance_id != instance_id + || graph.parameters.graph_id != graph_id + { + bail!("refuse to finalize graph {instance_id}:{graph_id} with mismatched graph parameters"); + } + // Reuse stored committee signatures when retrying graph finalization. + if !graph.committee_pre_signed() { + let pub_nonces = + order_committee_values(&committee, pub_nonces, "graph committee pub nonces")?; + let agg_nonces = nonces_aggregation(&pub_nonces)?; + let partial_sigs = + order_committee_values(&committee, partial_sigs, "graph committee partial sigs")?; + let committee_sig_for_graph = signature_aggregation(&partial_sigs, &agg_nonces, &graph)?; + push_committee_pre_signatures(&mut graph, &committee_sig_for_graph)?; + } + let simplified_graph = graph.to_simplified()?; + store_finalized_graph_if_needed(&deps.local_db, &simplified_graph).await?; + mark_graph_as_endorsed(&deps.local_db, instance_id, graph_id).await?; + let graph_nonce = graph.parameters.graph_nonce; + let operator_pubkey = graph.parameters.operator_pubkey.to_string(); + try_refresh_graph(deps, instance_id, graph_id, &graph).await; + + let mut ack_peers = committee_ack_peers(); + ack_peers.extend(selected_verifier_peers(deps, instance_id, graph_id).await?); + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_DELIVER, + Actor::All, + GOATMessageContent::GraphFinalize(GraphFinalize { + instance_id, + graph_id, + graph_nonce, + endorse_sigs: endorsements, + params_endorse_sigs: params_endorsements, + graph: simplified_graph, + }), + &ack_peers, + None, + ) + .await?, + ); + plan.notify_instance_endorsed = true; + plan.wake_refs.push(slot_ref(&operator_pubkey, graph_nonce)); + plan.state = GraphStateUpdate { + instance_id: Some(instance_id), + operator_pubkey: Some(operator_pubkey), + graph_nonce: Some(graph_nonce), + finalized: Some(true), + }; + Step::done(plan) +} + +/// Selected verifier peers of an operator's graph, from its BABE setup state. +async fn selected_verifier_peers( + deps: &JobDeps, + instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let Some(state) = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .and_then(|state| state.operator) + else { + return Ok(Vec::new()); + }; + let Some(selected) = state.selected_verifier_pubkeys.as_ref() else { + return Ok(Vec::new()); + }; + Ok(selected + .iter() + .filter_map(|pubkey| { + state.candidates.iter().find(|candidate| candidate.verifier_pubkey == *pubkey) + }) + .filter_map(|candidate| PeerId::from_bytes(&candidate.verifier_peer_id).ok()) + .map(|peer| peer.to_string()) + .collect()) +} + +/// The instance of a stored graph row, if any. +async fn graph_instance(deps: &JobDeps, graph_id: Uuid) -> Result> { + Ok(deps.local_db.acquire().await?.find_graph(&graph_id).await?.map(|row| row.instance_id)) +} + +/// Reconcile the finalized graph against both chains; a failure here is only +/// logged, the finalized definition is already stored. What the chain asks +/// of this node next comes from the graph's chain scans. +async fn try_refresh_graph( + deps: &JobDeps, + instance_id: Uuid, + graph_id: Uuid, + graph: &BitvmGcGraph, +) { + if let Err(error) = refresh_graph( + &deps.local_db, + &deps.btc_client, + &deps.goat_client, + instance_id, + graph_id, + graph, + ) + .await + { + warn!(event = "graph_refresh", graph_id = %graph_id, error = %error, "graph refresh failed"); + } +} + +// ----------------------------------------------------------------- verifier + +async fn verifier_setup(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::InitGraph(init) = message.content else { + return invalid("not an InitGraph message"); + }; + let instance_id = init.instance_id; + if init.graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("InitGraph identity mismatch"); + } + let from_peer = author_peer(&inbound)?; + if init.operator_peer_id != from_peer.to_bytes() { + return Step::ignore("signed operator peer id differs from the P2P source"); + } + if !verify_init_graph_signature(&init) { + return invalid("invalid operator signature on InitGraph"); + } + // The instance can fail while this message waits (or while a restart + // re-claims the job): the operator dropped its session, so the setup + // would only cost CPU and be ignored. + if instance_status(deps, instance_id).await?.is_some_and(|status| status.is_failed()) { + return Step::ignore("instance failed; verifier setup skipped"); + } + if !matches!( + operator_stake_status(&deps.goat_client, &init.operator_pubkey) + .await + .context("InitGraph operator stake lookup")?, + OperatorStakeStatus::Staked + ) { + return Step::ignore("operator does not meet stake requirements"); + } + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); + let verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); + let saved_verifier_state = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .and_then(|state| state.verifier) + .filter(|state| state.verifier_pubkey == verifier_pubkey); + let verifier_state = if let Some(saved) = saved_verifier_state { + if saved.operator_pubkey != init.operator_pubkey + || saved.operator_peer_id != init.operator_peer_id + { + return Step::ignore( + "a different authenticated operator identity owns this verifier setup", + ); + } + saved + } else { + get_babe_gc_asset_paths()?; + let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for BABE setup")?; + let static_input = derive_operator_static_input()?; + #[cfg(feature = "mock-gc")] + crate::mock_gc::simulate(crate::mock_gc::GcOp::Setup).await; + let (setup_package, private_state) = tokio::task::spawn_blocking(move || { + build_real_setup_package(BABE_N_CC, &vk, static_input) + }) + .await + .context("real BABE setup task failed")??; + info!(event = "verifier_gc_setup", graph_id = %graph_id, "verifier setup done"); + VerifierBabeSetupState { + verifier_pubkey, + operator_pubkey: init.operator_pubkey, + operator_peer_id: init.operator_peer_id.clone(), + setup_package, + private_state, + finalized_indices: vec![], + soldering_proof_ready: None, + } + }; + let setup_package = verifier_state.setup_package.clone(); + update_babe_setup_state(&deps.local_db, instance_id, graph_id, |state| { + state.verifier = Some(verifier_state); + })?; + let operator_peer = from_peer.to_string(); + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SETUP, + Actor::Operator, + GOATMessageContent::GenCircuits(GenCircuits { + instance_id, + graph_id, + verifier_pubkey, + setup_package, + }), + std::slice::from_ref(&operator_peer), + None, + ) + .await?, + ); + plan.state = GraphStateUpdate { + instance_id: Some(instance_id), + operator_pubkey: Some(init.operator_pubkey.to_string()), + ..Default::default() + }; + Step::done(plan) +} + +async fn verifier_solder(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::CutCircuits(CutCircuits { + instance_id, + graph_id: message_graph_id, + verifier_pubkey, + candidate_index, + selected_circuit_indexes, + }) = message.content + else { + return invalid("not a CutCircuits message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("CutCircuits identity mismatch"); + } + if instance_status(deps, instance_id).await?.is_some_and(|status| status.is_failed()) { + return Step::ignore("instance failed; verifier soldering skipped"); + } + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); + let local_verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); + if local_verifier_pubkey != verifier_pubkey { + return Step::ignore("target verifier pubkey does not match the local verifier"); + } + let Some(mut verifier_state) = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .and_then(|state| state.verifier) + else { + return Step::ignore("missing BABE verifier setup state"); + }; + if verifier_state.verifier_pubkey != verifier_pubkey { + return Step::ignore("verifier pubkey does not match the saved BABE setup state"); + } + if verifier_state.operator_peer_id != author_peer(&inbound)?.to_bytes() { + return Step::ignore("sender is not the operator that initiated this verifier setup"); + } + let operator_peer = PeerId::from_bytes(&verifier_state.operator_peer_id) + .context("decode operator peer id saved from InitGraph")? + .to_string(); + if selected_circuit_indexes.len() != BABE_M_CC { + return invalid(format!( + "invalid CutCircuits selection: expected {BABE_M_CC}, got {}", + selected_circuit_indexes.len() + )); + } + let soldering_proof_ready = if let Some(ready) = verifier_state.soldering_proof_ready.clone() { + if ready.candidate_index != candidate_index { + return invalid(format!( + "CutCircuits candidate index {candidate_index} conflicts with persisted slot {}", + ready.candidate_index + )); + } + if verifier_state.finalized_indices != selected_circuit_indexes { + return invalid("CutCircuits finalized indices conflict with persisted selection"); + } + ready + } else { + get_babe_gc_asset_paths()?; + let vk = + crate::vk::get_vk().await.context("load Groth16 verifying key for BABE opening")?; + let static_input = derive_operator_static_input()?; + let private_state = verifier_state.private_state.clone(); + let selected_indices = selected_circuit_indexes.clone(); + let package_for_opening = verifier_state.setup_package.clone(); + let soldering_builder = Arc::clone( + deps.soldering_builder + .as_ref() + .context("BABE soldering builder is not initialized for Verifier")?, + ); + #[cfg(feature = "mock-gc")] + crate::mock_gc::simulate(crate::mock_gc::GcOp::Solder).await; + let (opened, finalized, soldering) = tokio::task::spawn_blocking(move || { + open_real_setup_and_solder( + &soldering_builder, + &private_state, + &package_for_opening, + &selected_indices, + &vk, + static_input, + ) + }) + .await + .context("real BABE opening task failed")??; + let ready = save_soldering_proof_payload( + instance_id, + graph_id, + candidate_index, + &opened, + &finalized, + &soldering, + ) + .await?; + verifier_state.finalized_indices = selected_circuit_indexes; + verifier_state.soldering_proof_ready = Some(ready.clone()); + update_babe_setup_state(&deps.local_db, instance_id, graph_id, |state| { + state.verifier = Some(verifier_state); + })?; + ready + }; + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SETUP, + Actor::Operator, + GOATMessageContent::SolderingProofReady(soldering_proof_ready), + std::slice::from_ref(&operator_peer), + None, + ) + .await?, + ); + plan.state.instance_id = Some(instance_id); + Step::done(plan) +} + +async fn verifier_validate_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::CreateGraph(CreateGraph { + instance_id, + graph_id: message_graph_id, + graph_nonce, + graph, + }) = message.content + else { + return invalid("not a CreateGraph message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("CreateGraph identity mismatch"); + } + if !message_identity_matches( + "CreateGraph", + instance_id, + graph_id, + graph_nonce, + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return invalid("CreateGraph envelope and graph parameters name different graphs"); + } + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); + let local_verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); + let Some(verifier_index) = + find_verifier_index_by_pubkey(&graph.parameters.gc_data, &local_verifier_pubkey)? + else { + return Step::ignore("local verifier is not in this graph"); + }; + let full_graph = BitvmGcGraph::from_simplified(&graph)?; + validate_verifier_slot(&full_graph, verifier_index)?; + verify_graph_operator_pre_signatures(&full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!("invalid operator pre-signatures: {error}")) + })?; + let Some(verifier_state) = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .and_then(|state| state.verifier) + else { + return Step::ignore("missing local BABE verifier state"); + }; + if verifier_state.verifier_pubkey != local_verifier_pubkey { + return Step::ignore("local BABE verifier pubkey mismatch"); + } + let Some(soldering_proof_ready) = verifier_state.soldering_proof_ready.clone() else { + return Step::ignore("local verifier has no soldering proof"); + }; + let store_base_path = get_soldering_proof_payload_store_path()?; + let payload_path = soldering_proof_payload_store_path( + &store_base_path, + instance_id, + graph_id, + soldering_proof_ready.candidate_index, + &soldering_proof_ready.payload_hash, + )?; + let payload = read_soldering_proof_store_payload(&payload_path) + .await + .context("read local soldering proof payload for graph params endorsement")?; + let payload = decode_soldering_proof_payload(&soldering_proof_ready, &payload)?; + let (_opened, finalized, soldering) = expand_compact_soldering_proof_payload(payload) + .context("expand local soldering proof payload for graph params endorsement")?; + if finalized.len() != BABE_M_CC { + bail!("local verifier finalized BABE instance count mismatch"); + } + let epk = &verifier_state.setup_package.commits[finalized[0].index].epk; + let prover_state = + build_babe_prover_state(&verifier_state.setup_package, finalized, soldering)?; + let expected_gc_data = + extract_gc_circuit_data(local_verifier_pubkey, epk, &prover_state.h_msgs)?; + if graph.parameters.gc_data[verifier_index] != expected_gc_data { + return invalid( + "graph GC data for the local verifier does not match the local soldering proof", + ); + } + let canonical_graph_params_hash = graph.canonical_graph_params_hash()?; + let signature = sign_verifier_graph_params(verifier_master_key.master_keypair(), &graph)?; + deps.metrics_state.record_graph_validation(true); + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_ENDORSE, + Actor::Committee, + GOATMessageContent::VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement { + instance_id, + graph_id, + verifier_pubkey: local_verifier_pubkey, + verifier_index, + canonical_graph_params_hash, + signature, + }), + &[], + None, + ) + .await?, + ); + plan.close_rows.push(outbox_id(graph_id, "GenCircuits", None)); + plan.close_rows.push(outbox_id(graph_id, "SolderingProofReady", None)); + plan.state = GraphStateUpdate { + instance_id: Some(instance_id), + operator_pubkey: Some(graph.parameters.operator_pubkey.to_string()), + graph_nonce: Some(graph_nonce), + ..Default::default() + }; + Step::done(plan) +} + +// ---------------------------------------------------------------- committee + +async fn committee_ingest_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::CreateGraph(CreateGraph { + instance_id, + graph_id: message_graph_id, + graph_nonce, + graph, + }) = message.content + else { + return invalid("not a CreateGraph message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("CreateGraph identity mismatch"); + } + if !message_identity_matches( + "CreateGraph", + instance_id, + graph_id, + graph_nonce, + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return invalid("CreateGraph envelope and graph parameters name different graphs"); + } + if local_member(deps, instance_id).await?.is_none() { + return Step::ignore("not in this instance's committee"); + } + // Structure, policy, prekickoff continuity, instance parameters against + // both chains, watchtower set and operator stake; verifier endorsements + // may arrive later. + let validation = + validate_init_graph_base(&deps.local_db, &deps.btc_client, &deps.goat_client, &graph).await; + deps.metrics_state.record_graph_validation(validation.is_ok()); + validation?; + store_operator_presigned_graph(&deps.local_db, &graph).await?; + let verifier_count = graph.parameters.gc_data.len(); + let endorsements = + get_verifier_graph_params_endorsements_for_graph(&deps.local_db, instance_id, graph_id) + .await?; + let mut plan = GraphPlan { + state: GraphStateUpdate { + instance_id: Some(instance_id), + operator_pubkey: Some(graph.parameters.operator_pubkey.to_string()), + graph_nonce: Some(graph_nonce), + ..Default::default() + }, + ..Default::default() + }; + plan.wake_refs.push(slot_ref(&graph.parameters.operator_pubkey.to_string(), graph_nonce)); + if endorsements.len() >= verifier_count { + plan.then(JOB_COMMITTEE_NONCES, Vec::new(), None); + } + Step::done(plan) +} + +async fn ingest_verifier_endorsement( + deps: &JobDeps, + graph_id: Uuid, + params: &[u8], +) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement { + instance_id, + graph_id: message_graph_id, + verifier_pubkey, + verifier_index, + canonical_graph_params_hash, + signature, + }) = message.content + else { + return invalid("not a VerifierGraphParamsEndorsement message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("VerifierGraphParamsEndorsement identity mismatch"); + } + if local_member(deps, instance_id).await?.is_none() { + return Step::ignore("not in this instance's committee"); + } + let verifier_peer = author_peer(&inbound)?; + if !crate::p2p_admission::peer_registry().is_verifier(&verifier_peer) + && !deps.goat_client.committee_mana_is_verifier(&verifier_peer.to_bytes()).await? + { + return Step::ignore(format!("sender {verifier_peer} is not a registered verifier")); + } + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::retry("graph definition not stored yet", 30); + }; + if canonical_graph_params_hash != graph.canonical_graph_params_hash()? { + return invalid(format!("canonical graph params hash mismatch from {verifier_pubkey}")); + } + if verifier_index >= graph.parameters.gc_data.len() { + return invalid(format!("verifier index {verifier_index} out of range")); + } + if graph.parameters.gc_data[verifier_index].verifier_pubkey != verifier_pubkey { + return invalid(format!("verifier pubkey does not own slot {verifier_index}")); + } + if !verify_verifier_graph_params_endorsement(&verifier_pubkey, &graph, &signature)? { + return invalid(format!("invalid verifier endorsement signature from {verifier_pubkey}")); + } + store_verifier_graph_params_endorsement( + &deps.local_db, + instance_id, + graph_id, + verifier_pubkey, + verifier_index, + &verifier_peer.to_string(), + signature, + ) + .await?; + let endorsements = + get_verifier_graph_params_endorsements_for_graph(&deps.local_db, instance_id, graph_id) + .await?; + let mut plan = GraphPlan::default(); + plan.state.instance_id = Some(instance_id); + if endorsements.len() >= graph.parameters.gc_data.len() { + plan.then(JOB_COMMITTEE_NONCES, Vec::new(), None); + } + Step::done(plan) +} + +/// Start the committee's signing round for a validated, fully endorsed graph: +/// the deterministic nonces, stored before they are sent. +async fn committee_nonces(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + let Some(instance_id) = graph_instance(deps, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::ignore("graph definition missing"); + }; + let verifier_endorsements = + get_verifier_graph_params_endorsements_for_graph(&deps.local_db, instance_id, graph_id) + .await?; + if verifier_endorsements.len() < graph.parameters.gc_data.len() { + return Step::ignore(format!( + "waiting for verifier graph params endorsements ({}/{})", + verifier_endorsements.len(), + graph.parameters.gc_data.len() + )); + } + let validation = validate_init_graph( + &deps.local_db, + &deps.btc_client, + &deps.goat_client, + &graph, + &verifier_endorsements, + ) + .await; + deps.metrics_state.record_graph_validation(validation.is_ok()); + validation?; + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + let local_committee_pubkey: PublicKey = instance_keypair.public_key().into(); + if get_committee_partial_sigs_for_graph_member( + &deps.local_db, + instance_id, + graph_id, + &local_committee_pubkey, + ) + .await? + .is_some() + { + return Step::ignore("local committee partial signatures already exist"); + } + let full_graph = BitvmGcGraph::from_simplified(&graph)?; + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + let (pub_nonces, _, nonce_sigs) = + committee_master_key.nonces_for_graph_job_with_keypair(&full_graph, instance_keypair)?; + store_committee_pub_nonces_for_graph( + &deps.local_db, + instance_id, + graph_id, + local_committee_pubkey, + pub_nonces.clone(), + ) + .await?; + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SIGNING, + Actor::All, + GOATMessageContent::NonceGeneration(NonceGeneration { + instance_id, + graph_id, + committee_pubkey: local_committee_pubkey, + pub_nonces, + nonce_sigs, + }), + &[], + None, + ) + .await?, + ); + let committee = committee_pubkeys(deps, instance_id).await?; + let nonces = + get_committee_pub_nonces_for_graph(&deps.local_db, instance_id, graph_id).await?.len(); + plan.state.instance_id = Some(instance_id); + if nonces >= committee.len() { + plan.then(JOB_COMMITTEE_VOTE, Vec::new(), None); + } + Step::done(plan) +} + +/// The nonce transcript every member must agree on before signing. +struct NonceConsensus { + full_graph: BitvmGcGraph, + committee: Vec, + /// Every member's public nonces, in committee order. + pub_nonces: Vec, + agg_nonces: bitvm_lib::committee::CommitteeAggNonces, + consensus_hash: [u8; 32], +} + +async fn nonce_consensus( + deps: &JobDeps, + instance_id: Uuid, + graph_id: Uuid, + graph: &SimplifiedBitvmGcGraph, +) -> Result> { + let committee = committee_pubkeys(deps, instance_id).await?; + let pub_nonces_unchecked = + get_committee_pub_nonces_for_graph(&deps.local_db, instance_id, graph_id).await?; + if pub_nonces_unchecked.len() < committee.len() { + return Ok(None); + } + let full_graph = BitvmGcGraph::from_simplified(graph)?; + let verifier_num = full_graph.parameters.gc_data.len(); + let watchtower_num = full_graph.parameters.watchtower_pubkeys.len(); + let mut checked = Vec::with_capacity(pub_nonces_unchecked.len()); + for (pubkey, pub_nonces) in pub_nonces_unchecked { + pub_nonces.validate_length(watchtower_num, verifier_num).map_err(|error| { + anyhow!("committee public nonces from {pubkey} have invalid length: {error}") + })?; + checked.push((pubkey, pub_nonces)); + } + let ordered = order_committee_values(&committee, checked, "graph committee pub nonces")?; + let agg_nonces = nonces_aggregation(&ordered)?; + let consensus_hash = committee_nonce_consensus_hash(&full_graph, &committee, &ordered)?; + Ok(Some(NonceConsensus { + full_graph, + committee, + pub_nonces: ordered, + agg_nonces, + consensus_hash, + })) +} + +async fn committee_vote(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + let Some(instance_id) = graph_instance(deps, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::ignore("graph definition missing"); + }; + let Some(consensus) = nonce_consensus(deps, instance_id, graph_id, &graph).await? else { + return Step::ignore("waiting for committee public nonces"); + }; + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + let local_committee_pubkey: PublicKey = instance_keypair.public_key().into(); + let votes = + get_committee_agg_nonce_consensus_for_graph(&deps.local_db, instance_id, graph_id).await?; + let local_vote = votes.iter().find(|(pubkey, _, _)| *pubkey == local_committee_pubkey); + if let Some((_, stored_hash, _)) = local_vote + && *stored_hash != consensus.consensus_hash + { + return invalid(format!( + "local committee agg nonce consensus differs for graph {graph_id}" + )); + } + let signature = match local_vote { + Some((_, _, signature)) => *signature, + None => { + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + let signature = SECP256K1.sign_schnorr( + &SecpMessage::from_digest(consensus.consensus_hash), + &instance_keypair, + ); + store_committee_agg_nonce_consensus_for_graph( + &deps.local_db, + instance_id, + graph_id, + local_committee_pubkey, + consensus.consensus_hash, + signature, + ) + .await?; + signature + } + }; + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SIGNING, + Actor::Committee, + GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + instance_id, + graph_id, + committee_pubkey: local_committee_pubkey, + consensus_hash: consensus.consensus_hash, + signature, + }), + &[], + None, + ) + .await?, + ); + plan.state.instance_id = Some(instance_id); + if votes_complete(deps, instance_id, graph_id, &consensus).await? { + plan.then(JOB_COMMITTEE_PRESIGN, Vec::new(), None); + } + Step::done(plan) +} + +/// Every member signed the same ordered nonce set (the AggNonceConsensus barrier). +async fn votes_complete( + deps: &JobDeps, + instance_id: Uuid, + graph_id: Uuid, + consensus: &NonceConsensus, +) -> Result { + let votes = + get_committee_agg_nonce_consensus_for_graph(&deps.local_db, instance_id, graph_id).await?; + if votes.len() < consensus.committee.len() { + return Ok(false); + } + let ordered = order_committee_values( + &consensus.committee, + votes.into_iter().map(|(pubkey, hash, signature)| (pubkey, (hash, signature))).collect(), + "graph committee agg nonce consensus", + )?; + for (committee_pubkey, (received_hash, signature)) in consensus.committee.iter().zip(ordered) { + if received_hash != consensus.consensus_hash { + return invalid(format!( + "committee agg nonce consensus differs for graph {graph_id} and committee {committee_pubkey}" + )); + } + SECP256K1 + .verify_schnorr( + &signature, + &SecpMessage::from_digest(consensus.consensus_hash), + &XOnlyPublicKey::from(*committee_pubkey), + ) + .map_err(|error| { + SpecialError::InvalidGraph(format!( + "invalid agg nonce consensus signature for graph {graph_id} and committee {committee_pubkey}: {error}" + )) + })?; + } + Ok(true) +} + +async fn ingest_vote(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + instance_id, + graph_id: message_graph_id, + committee_pubkey, + consensus_hash: received_hash, + signature, + }) = message.content + else { + return invalid("not an AggNonceConsensus message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("AggNonceConsensus identity mismatch"); + } + ensure_committee_sender(deps, &inbound, &committee_pubkey).await?; + if deps.actor == Actor::Committee && local_member(deps, instance_id).await?.is_none() { + return Step::ignore("not in this instance's committee"); + } + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::retry("graph definition not stored yet", 30); + }; + let Some(consensus) = nonce_consensus(deps, instance_id, graph_id, &graph).await? else { + return Step::retry( + "waiting for committee public nonces before validating agg nonce consensus", + 30, + ); + }; + if received_hash != consensus.consensus_hash { + return invalid(format!("consensus hash mismatch from {committee_pubkey}")); + } + if SECP256K1 + .verify_schnorr( + &signature, + &SecpMessage::from_digest(consensus.consensus_hash), + &XOnlyPublicKey::from(committee_pubkey), + ) + .is_err() + { + return invalid(format!("invalid consensus signature from {committee_pubkey}")); + } + store_committee_agg_nonce_consensus_for_graph( + &deps.local_db, + instance_id, + graph_id, + committee_pubkey, + received_hash, + signature, + ) + .await?; + let mut plan = GraphPlan::default(); + plan.state.instance_id = Some(instance_id); + if votes_complete(deps, instance_id, graph_id, &consensus).await? { + plan.then(JOB_COMMITTEE_PRESIGN, Vec::new(), None); + } + Step::done(plan) +} + +async fn committee_presign(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + let Some(instance_id) = graph_instance(deps, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::ignore("graph definition missing"); + }; + let Some(mut consensus) = nonce_consensus(deps, instance_id, graph_id, &graph).await? else { + return Step::ignore("waiting for committee public nonces"); + }; + if !votes_complete(deps, instance_id, graph_id, &consensus).await? { + return Step::ignore("waiting for agg nonce consensus"); + } + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + let local_committee_pubkey: PublicKey = instance_keypair.public_key().into(); + let partial_sigs = match get_committee_partial_sigs_for_graph_member( + &deps.local_db, + instance_id, + graph_id, + &local_committee_pubkey, + ) + .await? + { + Some(stored) => stored, + None => { + // Recheck the persisted bindings at the actual signing boundary. + let verifier_endorsements = get_verifier_graph_params_endorsements_for_graph( + &deps.local_db, + instance_id, + graph_id, + ) + .await?; + validate_verifier_graph_params_endorsements( + &deps.goat_client, + &graph, + &verifier_endorsements, + ) + .await?; + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + let (pub_nonces, sec_nonces, _) = committee_master_key + .nonces_for_graph_job_with_keypair(&consensus.full_graph, instance_keypair)?; + // Authorization before the signature: this nonce, this transcript, + // once. The nonce consensus hash binds graph, committee and every + // public nonce, hence the aggregate nonces and the sighashes. + let transcript = hex::encode(consensus.consensus_hash); + let nonce = signing_nonce_id("graph-presign", &bincode::serialize(&pub_nonces)?); + let authorization = deps + .local_db + .acquire() + .await? + .authorize_signing( + "graph", + &graph_id.to_string(), + &nonce, + &transcript, + current_time_secs(), + ) + .await?; + if let SigningAuthorization::Conflict { existing_digest } = authorization { + return Err(SpecialError::InvalidGraph(format!( + "graph presign nonce already authorized for transcript {existing_digest}, refusing {transcript}" + )) + .into()); + } + let partial_sigs = committee_pre_sign( + instance_keypair, + sec_nonces, + consensus.agg_nonces.clone(), + &mut consensus.full_graph, + )?; + store_committee_partial_sigs_for_graph( + &deps.local_db, + instance_id, + graph_id, + local_committee_pubkey, + partial_sigs.clone(), + ) + .await?; + partial_sigs + } + }; + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SIGNING, + Actor::All, + GOATMessageContent::CommitteePresign(CommitteePresign { + instance_id, + graph_id, + committee_pubkey: local_committee_pubkey, + committee_partial_sigs: partial_sigs, + agg_nonces: consensus.agg_nonces.clone(), + }), + &[], + None, + ) + .await?, + ); + plan.state.instance_id = Some(instance_id); + let partials = + get_committee_partial_sigs_for_graph(&deps.local_db, instance_id, graph_id).await?.len(); + if partials >= consensus.committee.len() { + plan.then(JOB_COMMITTEE_ENDORSE, Vec::new(), None); + } + Step::done(plan) +} + +async fn committee_endorse(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + let Some(instance_id) = graph_instance(deps, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + let Some(graph) = get_graph(&deps.local_db, instance_id, graph_id).await? else { + return Step::ignore("graph definition missing"); + }; + let committee = committee_pubkeys(deps, instance_id).await?; + let partials = + get_committee_partial_sigs_for_graph(&deps.local_db, instance_id, graph_id).await?; + if partials.len() != committee.len() { + return Step::ignore("waiting for committee partial signatures"); + } + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + let full_graph = BitvmGcGraph::from_simplified(&graph)?; + let committee_sig_for_graph = endorse_graph(&deps.goat_client, &full_graph).await?; + let committee_sig_for_params = endorse_graph_params(&full_graph).await?; + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + let local_committee_pubkey: PublicKey = instance_keypair.public_key().into(); + let committee_evm_address = + get_node_goat_address().ok_or_else(|| anyhow!("failed to get node goat address"))?; + let mut plan = GraphPlan::default(); + plan.send( + planned( + graph_id, + PHASE_SIGNING, + Actor::All, + GOATMessageContent::EndorseGraph(EndorseGraph { + instance_id, + graph_id, + committee_pubkey: local_committee_pubkey, + committee_sig_for_graph: committee_sig_for_graph.as_bytes().to_vec(), + committee_sig_for_params: committee_sig_for_params.as_bytes().to_vec(), + committee_evm_address, + }), + &[], + None, + ) + .await?, + ); + plan.state.instance_id = Some(instance_id); + Step::done(plan) +} + +// ------------------------------------------------------------------ finalize + +async fn ingest_finalized(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::GraphFinalize(GraphFinalize { + instance_id, + graph_id: message_graph_id, + graph_nonce, + graph, + endorse_sigs, + params_endorse_sigs, + }) = message.content + else { + return invalid("not a GraphFinalize message"); + }; + if message_graph_id != graph_id || instance_id != inbound.instance_id { + return invalid("GraphFinalize identity mismatch"); + } + if !message_identity_matches( + "GraphFinalize", + instance_id, + graph_id, + graph_nonce, + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id, + graph.parameters.graph_nonce, + ) { + return invalid("GraphFinalize envelope and graph parameters name different graphs"); + } + let validation = validate_finalized_graph( + &deps.btc_client, + &deps.goat_client, + &graph, + &endorse_sigs, + ¶ms_endorse_sigs, + ) + .await; + deps.metrics_state.record_graph_validation(validation.is_ok()); + validation?; + store_finalized_graph_if_needed(&deps.local_db, &graph).await?; + store_committee_endorsements_for_graph( + &deps.local_db, + instance_id, + graph_id, + endorse_sigs, + params_endorse_sigs, + ) + .await?; + mark_graph_as_endorsed(&deps.local_db, instance_id, graph_id).await?; + let full_graph = BitvmGcGraph::from_simplified(&graph)?; + try_refresh_graph(deps, instance_id, graph_id, &full_graph).await; + let operator_pubkey = graph.parameters.operator_pubkey.to_string(); + let mut plan = GraphPlan { + notify_instance_endorsed: true, + wake_refs: vec![slot_ref(&operator_pubkey, graph_nonce)], + state: GraphStateUpdate { + instance_id: Some(instance_id), + operator_pubkey: Some(operator_pubkey), + graph_nonce: Some(graph_nonce), + finalized: Some(true), + }, + ..Default::default() + }; + if deps.relayer { + plan.then(JOB_POST_GRAPH_DATA, Vec::new(), None); + } + Step::done(plan) +} + +/// The relayer's `postGraphData` once the peg-in is withdrawable (Post phase). +async fn post_graph_data(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Result { + if !deps.relayer { + return Step::ignore("not a relayer"); + } + let Some(instance_id) = graph_instance(deps, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + if instance_status(deps, instance_id).await?.is_some_and(|status| status.is_failed()) { + return Step::ignore("instance failed; graph not posted"); + } + let graph_data = deps.goat_client.gateway_get_graph_data(&graph_id).await?; + if graph_data.operator_pubkey != [0u8; 32] { + return Step::ignore("graph data already posted"); + } + let pegin_data = deps.goat_client.gateway_get_pegin_data(&instance_id).await?; + if pegin_data.status != PeginStatus::Withdrawable { + let block_time = crate::utils::avg_block_time_secs(deps.btc_client.network()) as i64; + return Step::retry( + format!("pegin data is {}; not withdrawable yet", pegin_data.status), + block_time, + ); + } + let committee = committee_pubkeys(deps, instance_id).await?; + let endorsement_sigs = + get_committee_endorsements_for_graph(&deps.local_db, instance_id, graph_id) + .await? + .into_iter() + .map(|(_, _, sig)| sig) + .collect::>(); + if endorsement_sigs.len() != committee.len() { + return Step::retry( + format!( + "{} of {} committee endorsements for the graph", + endorsement_sigs.len(), + committee.len() + ), + 60, + ); + } + let graph = get_graph(&deps.local_db, instance_id, graph_id) + .await? + .ok_or_else(|| anyhow!("Graph not found for {instance_id}:{graph_id}"))?; + let graph = BitvmGcGraph::from_simplified(&graph)?; + let data = crate::utils::build_graph_data(&graph)?; + deps.goat_client + .gateway_post_graph_data(&instance_id, &graph_id, &data, &endorsement_sigs) + .await?; + info!(event = "graph_post", graph_id = %graph_id, instance_id = %instance_id, "postGraphData sent"); + Step::done(GraphPlan::default()) +} + +/// Pull attempts a minute apart before a missing definition is retried +/// slowly. The job id is stable, so a pull that stopped would never start +/// again for the parked messages that asked for it; it ends when they have +/// expired (`PARKED_MESSAGE_TTL_SECS`), or earlier when the graph retires. +const FETCH_GRAPH_FAST_ATTEMPTS: i64 = 5; +const FETCH_GRAPH_SLOW_RETRY_SECS: i64 = 600; +const FETCH_GRAPH_MAX_ATTEMPTS: i64 = FETCH_GRAPH_FAST_ATTEMPTS + + super::common::PARKED_MESSAGE_TTL_SECS / FETCH_GRAPH_SLOW_RETRY_SECS; + +async fn fetch_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8], attempts: i64) -> Result { + let lookup: GraphLookup = bincode::deserialize(params).context("decode graph lookup")?; + let give_up = |reason: String| { + if attempts + 1 >= FETCH_GRAPH_MAX_ATTEMPTS { + warn!(event = "graph_fetch", graph_id = %graph_id, ?lookup, attempts, reason, "giving up on the pull"); + Step::ignore(format!("gave up pulling {lookup:?}: {reason}")) + } else if attempts + 1 >= FETCH_GRAPH_FAST_ATTEMPTS { + Step::retry(reason, FETCH_GRAPH_SLOW_RETRY_SECS) + } else { + Step::retry(reason, 60) + } + }; + match fetch_and_store_graph( + &deps.local_db, + &deps.btc_client, + &deps.goat_client, + &deps.publisher, + &lookup, + ) + .await + { + Ok(Some(found)) => { + // The pulled graph's own aggregate decides what follows. + submit_event( + &deps.local_db, + graph_pulled_event(found.graph_id, found.instance_id, found.finalized)?, + ) + .await?; + let mut plan = GraphPlan::noted(format!( + "pulled graph {} (finalized: {})", + found.graph_id, found.finalized + )); + plan.wake_refs.push(slot_ref(&found.operator_pubkey, found.graph_nonce)); + plan.wake_refs.push(graph_ref(found.graph_id).wake_ref()); + if found.graph_id != graph_id { + plan.wake_refs.push(graph_ref(graph_id).wake_ref()); + } + Step::done(plan) + } + Ok(None) => give_up(format!("no peer served {lookup:?}")), + Err(error) => give_up(format!("{error:#}")), + } +} diff --git a/node/src/aggregates/graph_pegout.rs b/node/src/aggregates/graph_pegout.rs new file mode 100644 index 000000000..168345b35 --- /dev/null +++ b/node/src/aggregates/graph_pegout.rs @@ -0,0 +1,873 @@ +//! Graph withdraw and settlement decisions. +//! +//! Withdraw events, chain scans and Goat settlement events update graph state +//! and schedule jobs for this node's role. Kickoff waits for the previous slot +//! to settle; earlier slots are skipped only when the collected facts allow it. + +use super::graph::{GraphAggregate, GraphState, slot_ref}; +use super::graph_fetch::GraphLookup; +use super::instance::TxFact; +use crate::metrics_service::node_metrics_state; +use anyhow::anyhow; +use bitcoin::{OutPoint, Txid}; +use bitvm_lib::actors::Actor; +use client::goat_chain::DisproveTxType; +use engine::{ApplyContext, ApplyOutcome, EventSource, JobPool, JobSpec, NewEvent, WaitingFor}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use store::{Graph, GraphStatus, GraphStatusSource, GraphStatusTransitionOutcome}; +use store::{InstanceBridgeInStatus, localdb::GraphRuntimeUpdate}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +pub const EVENT_WITHDRAW_INIT: &str = "withdraw_init"; +pub const EVENT_WITHDRAW_CANCEL: &str = "withdraw_cancel"; +pub const EVENT_CHAIN_SCAN: &str = "chain_scan"; +/// A Goat settlement event changed the status; the aggregate mirrors it. +pub const EVENT_GOAT_STATUS: &str = "goat_status"; + +/// Job phase of the operator's withdraw: cancelled by `CancelWithdraw`. +pub const PHASE_KICKOFF: &str = "Kickoff"; +/// Job phase of everything the chain scan triggers. +pub const PHASE_SETTLEMENT: &str = "Settlement"; + +// Kickoff (operator). +pub const JOB_KICKOFF: &str = "kickoff"; +pub const JOB_SETTLE_PREDECESSORS: &str = "settle_predecessors"; +// Settlement, by the signal that triggers them. +pub const JOB_ENFORCE_PREVIOUS_SLOT: &str = "enforce_previous_slot"; +pub const JOB_PROCEED_WITHDRAW: &str = "proceed_withdraw"; +pub const JOB_CHALLENGE_KICKOFF: &str = "challenge_kickoff"; +pub const JOB_WATCHTOWER_CHALLENGE_INIT: &str = "watchtower_challenge_init"; +pub const JOB_WATCHTOWER_CHALLENGE: &str = "watchtower_challenge"; +pub const JOB_WATCHTOWER_PROOF_TIMEOUT: &str = "watchtower_proof_timeout"; +pub const JOB_CHALLENGE_ACK: &str = "challenge_ack"; +pub const JOB_WATCHTOWER_TIMEOUT: &str = "watchtower_timeout"; +pub const JOB_CHALLENGE_NACK: &str = "challenge_nack"; +pub const JOB_COMMIT_PUBIN: &str = "commit_pubin"; +pub const JOB_COMMIT_TIMEOUT: &str = "commit_timeout"; +pub const JOB_PROVER_ASSERT: &str = "prover_assert"; +pub const JOB_VERIFIER_ASSERT: &str = "verifier_assert"; +pub const JOB_WRONGLY_CHALLENGED: &str = "wrongly_challenged"; +pub const JOB_DISPROVE: &str = "disprove"; +pub const JOB_FINISH_DISPROVED: &str = "finish_disproved"; +pub const JOB_TAKE1: &str = "take1"; +pub const JOB_FINISH_HAPPY: &str = "finish_happy"; +pub const JOB_TAKE2: &str = "take2"; +pub const JOB_FINISH_UNHAPPY: &str = "finish_unhappy"; + +pub const PEGOUT_JOBS: &[&str] = &[ + JOB_KICKOFF, + JOB_SETTLE_PREDECESSORS, + JOB_ENFORCE_PREVIOUS_SLOT, + JOB_PROCEED_WITHDRAW, + JOB_CHALLENGE_KICKOFF, + JOB_WATCHTOWER_CHALLENGE_INIT, + JOB_WATCHTOWER_CHALLENGE, + JOB_WATCHTOWER_PROOF_TIMEOUT, + JOB_CHALLENGE_ACK, + JOB_WATCHTOWER_TIMEOUT, + JOB_CHALLENGE_NACK, + JOB_COMMIT_PUBIN, + JOB_COMMIT_TIMEOUT, + JOB_PROVER_ASSERT, + JOB_VERIFIER_ASSERT, + JOB_WRONGLY_CHALLENGED, + JOB_DISPROVE, + JOB_FINISH_DISPROVED, + JOB_TAKE1, + JOB_FINISH_HAPPY, + JOB_TAKE2, + JOB_FINISH_UNHAPPY, +]; + +/// WOTS and garbled-circuit work runs on the peg-out slots, apart from the +/// setup's CPU slots: a graph build takes hours, a challenge answers a +/// timelock. The rest broadcasts or calls Goat. +pub fn pegout_job_pool(action: &str) -> JobPool { + match action { + JOB_WATCHTOWER_CHALLENGE + | JOB_COMMIT_PUBIN + | JOB_PROVER_ASSERT + | JOB_VERIFIER_ASSERT + | JOB_WRONGLY_CHALLENGED => JobPool::Pegout, + _ => JobPool::Network, + } +} + +// ----------------------------------------------------------------- payloads + +/// `InitWithdraw` / `CancelWithdraw` on Goat. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct WithdrawEvent { + pub tx_hash: String, +} + +/// Params of the operator's kickoff jobs: the withdraw they serve, so a new +/// `InitWithdraw` after a cancel gets fresh jobs. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct KickoffParams { + pub tx_hash: String, +} + +/// Chain signals mapped to role-specific jobs with stable IDs for deduplication. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum PegoutSignal { + /// Pre-Kickoff of the graph is on chain. + PreKickoffSent, + /// Kickoff of the graph is on chain. + KickoffSent, + Take1Ready, + Take1Sent, + ChallengeSent { + challenge_txid: Txid, + }, + WatchtowerChallengeInitSent, + WatchtowerChallengeSent { + watchtower_index: usize, + }, + WatchtowerChallengeTimeout, + /// A watchtower's challenge confirmed, the operator neither ACKed nor + /// NACKed it and `T5` passed: one NACK per branch. + NackReady { + watchtower_index: usize, + }, + OperatorCommitPubinReady, + OperatorCommitPubinTimeout, + AssertReady, + AssertSent { + assert_txid: Txid, + }, + ChallengeAssertSent { + challenge_assert_txid: Txid, + verifier_index: usize, + }, + WronglyChallengeTimeout { + challenge_assert_txid: Txid, + verifier_index: usize, + }, + DisproveSent { + disprove_type: DisproveTxType, + index: usize, + challenge_finish_txid: Txid, + }, + Take2Ready, + Take2Sent, +} + +impl PegoutSignal { + pub fn encode(&self) -> anyhow::Result> { + Ok(serde_json::to_vec(self)?) + } + + pub fn decode(bytes: &[u8]) -> anyhow::Result { + Ok(serde_json::from_slice(bytes)?) + } +} + +/// A chain fact that makes the graph unusable before its Pre-Kickoff confirms. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Unusable { + /// Connector-0 was spent by another graph's Take-1 / Take-2. + ConnectorZeroTaken { spent_by: Txid }, + /// An input of this graph's Pre-Kickoff was spent by another transaction + /// (Challenge-Incomplete-Kickoff or another Pre-Kickoff): it can never + /// confirm, and neither can any later slot of the operator. + PrekickoffInputSpent { outpoint: OutPoint, spent_by: Txid }, + /// That happened to an earlier slot of the operator, which this graph's + /// Pre-Kickoff extends. + EarlierSlotDead { slot: u64 }, +} + +/// One scan of a graph at a Bitcoin height. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChainScan { + pub height: i64, + /// The persisted status the scan saw; part of the event id. + pub status_at_scan: String, + /// The stage both chains show; never `Obsoleted`. + pub stage: GraphStatus, + /// Challenge progress (`ChallengeSubStatus` JSON). + pub sub_status: Option, + pub challenge_txid: Option, + pub unusable: Option, + pub signals: Vec, +} + +impl ChainScan { + /// One event per graph, height and status seen: a rescan at the same tip + /// is deduplicated, a scan after a status change is not. + pub fn event(&self, graph_id: Uuid) -> anyhow::Result { + Ok(NewEvent::new( + format!("graph:{graph_id}:scan:{}:{}", self.height, self.status_at_scan), + super::graph::graph_ref(graph_id), + EVENT_CHAIN_SCAN, + EventSource::Height, + serde_json::to_vec(self)?, + )) + } +} + +pub fn withdraw_init_event(graph_id: Uuid, tx_hash: &str) -> anyhow::Result { + withdraw_event(EVENT_WITHDRAW_INIT, "withdraw-init", graph_id, tx_hash) +} + +pub fn withdraw_cancel_event(graph_id: Uuid, tx_hash: &str) -> anyhow::Result { + withdraw_event(EVENT_WITHDRAW_CANCEL, "withdraw-cancel", graph_id, tx_hash) +} + +fn withdraw_event( + kind: &str, + key: &str, + graph_id: Uuid, + tx_hash: &str, +) -> anyhow::Result { + Ok(NewEvent::new( + format!("graph:{graph_id}:{key}:{tx_hash}"), + super::graph::graph_ref(graph_id), + kind, + EventSource::Chain, + serde_json::to_vec(&WithdrawEvent { tx_hash: tx_hash.to_string() })?, + )) +} + +/// A Goat settlement event moved the graph's status. +pub fn goat_status_event( + graph_id: Uuid, + status: GraphStatus, + tx_hash: &str, +) -> anyhow::Result { + Ok(NewEvent::new( + format!("graph:{graph_id}:goat:{status}:{tx_hash}"), + super::graph::graph_ref(graph_id), + EVENT_GOAT_STATUS, + EventSource::Chain, + Vec::new(), + )) +} + +// ------------------------------------------------------------------ routing + +/// The job this role runs for a signal; `own` is whether the graph belongs +/// to the local operator. +pub fn route_signal( + signal: &PegoutSignal, + actor: &Actor, + relayer: bool, + own: bool, +) -> Option<&'static str> { + use Actor::{Committee, Operator, Verifier, Watchtower}; + use PegoutSignal::*; + Some(match (signal, actor) { + (PreKickoffSent, Verifier) => JOB_ENFORCE_PREVIOUS_SLOT, + (KickoffSent, Committee) if relayer => JOB_PROCEED_WITHDRAW, + (KickoffSent, Verifier) => JOB_CHALLENGE_KICKOFF, + (ChallengeSent { .. }, Operator) if own => JOB_WATCHTOWER_CHALLENGE_INIT, + (WatchtowerChallengeInitSent, Watchtower) => JOB_WATCHTOWER_CHALLENGE, + (AssertSent { .. }, Watchtower) => JOB_WATCHTOWER_PROOF_TIMEOUT, + (WatchtowerChallengeSent { .. }, Operator) if own => JOB_CHALLENGE_ACK, + (WatchtowerChallengeTimeout, Operator) if own => JOB_WATCHTOWER_TIMEOUT, + (NackReady { .. }, Verifier) => JOB_CHALLENGE_NACK, + (OperatorCommitPubinReady, Operator) if own => JOB_COMMIT_PUBIN, + (OperatorCommitPubinTimeout, Verifier) => JOB_COMMIT_TIMEOUT, + (AssertReady, Operator) if own => JOB_PROVER_ASSERT, + (AssertSent { .. }, Verifier) => JOB_VERIFIER_ASSERT, + (ChallengeAssertSent { .. }, Operator) if own => JOB_WRONGLY_CHALLENGED, + (WronglyChallengeTimeout { .. }, Verifier) => JOB_DISPROVE, + (DisproveSent { .. }, Committee) if relayer => JOB_FINISH_DISPROVED, + (Take1Ready, Operator) if own => JOB_TAKE1, + (Take1Sent, Committee) if relayer => JOB_FINISH_HAPPY, + (Take2Ready, Operator) if own => JOB_TAKE2, + (Take2Sent, Committee) if relayer => JOB_FINISH_UNHAPPY, + _ => return None, + }) +} + +// ---------------------------------------------------------------- decisions + +/// Derive status from chain progress. Confirmed Pre-Kickoff or later progress takes +/// precedence over local Obsoleted state. Before that, unusable graphs or unposted +/// graphs belonging to failed instances become Obsoleted. +pub fn decide_status(scan: &ChainScan, instance_failed: bool) -> Option { + use GraphStatus::*; + match scan.stage { + PreKickoff | OperatorKickOff | Challenge | Disprove | OperatorTake1 | OperatorTake2 + | Skipped => Some(scan.stage), + _ if scan.unusable.is_some() => Some(Obsoleted), + OperatorDataPushed => Some(OperatorDataPushed), + OperatorPresigned | CommitteePresigned if instance_failed => Some(Obsoleted), + OperatorPresigned | CommitteePresigned | Obsoleted => None, + } +} + +/// How a Kickoff Connector was spent. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum KickoffConnectorSpend { + /// Unspent, or its Pre-Kickoff is not on chain. + #[default] + Unspent, + ByKickoff, + /// Skip-Kickoff or Force-Skip-Kickoff. + ByOther, +} + +/// The local copy of a slot. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocalSlot { + pub graph_id: Uuid, + pub instance_id: Uuid, + pub status: GraphStatus, + /// Carries committee pre-signatures. + pub finalized: bool, + pub instance_status: Option, +} + +/// Facts about one slot of an operator's chain. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SlotFacts { + pub slot: u64, + pub local: Option, + pub kickoff_connector: KickoffConnectorSpend, + /// An input of the slot's Pre-Kickoff was spent by another transaction. + pub prekickoff_dead: bool, + /// L2 `GraphData` of the graph is posted. + pub posted: bool, + /// Connector-0 was spent by another graph's Take. + pub connector_zero_taken: bool, + /// Peg-In-Cancel of the instance is confirmed (Connector-Z spent by Cancel). + pub cancel_confirmed: bool, + /// Peg-In-Confirm of the instance. + pub confirm: TxFact, + /// The instance passed its abandon height `H_p + T0 + B`: no committee + /// member signs after `T0`, so an unfinished graph stays unfinished. + pub past_abandon_height: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SlotVerdict { + /// Taken, disproved or skipped. + Settled, + /// Fetch the definition first: a missing or unfinalized local copy is + /// never taken for an abandoned graph. + Pull(GraphLookup), + /// Pre-Kickoff + Skip-Kickoff, for the named premise. + Skip(&'static str), + /// Live, in peg-out, or undecided: look again later. + Wait(String), + /// Never skipped automatically; needs a human. + Stuck(String), +} + +/// Determine whether a slot is settled, skippable or still blocks its successor. +/// Use chain evidence rather than Obsoleted status alone. +pub fn decide_slot(facts: &SlotFacts, operator_pubkey: &str) -> SlotVerdict { + let slot = facts.slot; + let Some(local) = &facts.local else { + return SlotVerdict::Pull(GraphLookup::Slot { + operator_pubkey: operator_pubkey.to_string(), + graph_nonce: slot, + }); + }; + if facts.kickoff_connector == KickoffConnectorSpend::ByOther || local.status.is_closed() { + return SlotVerdict::Settled; + } + if facts.kickoff_connector == KickoffConnectorSpend::ByKickoff + || local.status.is_pegout_started() + { + return SlotVerdict::Wait(format!("slot {slot} is in peg-out")); + } + if facts.prekickoff_dead { + return SlotVerdict::Stuck(format!( + "Pre-Kickoff of slot {slot} can never confirm: an input is spent elsewhere" + )); + } + if facts.posted { + if !local.finalized { + return SlotVerdict::Pull(GraphLookup::Id { + instance_id: local.instance_id, + graph_id: local.graph_id, + finalized: true, + }); + } + if facts.connector_zero_taken { + return SlotVerdict::Skip("connector-0 taken by another graph"); + } + return SlotVerdict::Wait(format!("slot {slot} is posted and available for withdrawal")); + } + if facts.connector_zero_taken { + return SlotVerdict::Skip("connector-0 taken by another graph"); + } + if facts.cancel_confirmed { + return SlotVerdict::Skip("peg-in canceled"); + } + match local.instance_status { + Some(status) if status.is_failed() => { + // Checked again right before the skip: a Peg-In-Confirm that + // landed after the abandon height keeps the graph. + if facts.confirm.confirmed { + SlotVerdict::Stuck(format!( + "Peg-In-Confirm of slot {slot} is confirmed although its instance is {status}" + )) + } else if facts.confirm.in_mempool { + SlotVerdict::Wait(format!("Peg-In-Confirm of slot {slot} is in the mempool")) + } else if status == InstanceBridgeInStatus::Timeout { + SlotVerdict::Skip("instance timed out") + } else { + SlotVerdict::Skip("instance failed") + } + } + // After abandonment, an unfinished graph is skippable. A late finalized + // definition may have been stored after the row became Obsoleted. + Some(_) + if facts.past_abandon_height + && (!local.finalized || local.status == GraphStatus::Obsoleted) => + { + SlotVerdict::Skip("unfinished at the abandon height") + } + Some(status) => SlotVerdict::Wait(format!( + "slot {slot} ({}) belongs to a live instance ({status})", + if local.finalized { "finalized" } else { "unfinished" } + )), + None => SlotVerdict::Wait(format!("instance of slot {slot} is unknown")), + } +} + +/// Whether the local copy of slot `nonce − 1` is settled. +async fn predecessor_settled( + ctx: &mut ApplyContext<'_>, + operator_pubkey: &str, + graph_nonce: u64, +) -> anyhow::Result> { + if graph_nonce == 0 { + return Ok(None); + } + let previous = + ctx.storage.find_graph_by_operator_slot(operator_pubkey, graph_nonce as i64 - 1).await?; + Ok(match previous { + Some(previous) + if GraphStatus::from_str(&previous.status).is_ok_and(|status| status.is_closed()) => + { + None + } + Some(previous) => Some(format!( + "previous slot {} (graph {}) is {}", + previous.kickoff_index, previous.graph_id, previous.status + )), + None => Some(format!("previous slot {} is not stored", graph_nonce - 1)), + }) +} + +fn kickoff_job(ctx: &ApplyContext<'_>, action: &str, tx_hash: &str) -> anyhow::Result { + Ok(JobSpec::new( + &ctx.aggregate, + PHASE_KICKOFF, + action, + pegout_job_pool(action), + serde_json::to_vec(&KickoffParams { tx_hash: tx_hash.to_string() })?, + )) +} + +// ------------------------------------------------------------------- applies + +impl GraphAggregate { + /// `InitWithdraw`: the owning operator kicks off once the previous slot + /// settled; meanwhile a job settles the earlier slots and the event waits + /// on the previous slot. + pub(super) async fn apply_withdraw_init( + &self, + ctx: &mut ApplyContext<'_>, + state: &mut GraphState, + row: Option<&Graph>, + payload: &[u8], + ) -> anyhow::Result { + let withdraw: WithdrawEvent = match serde_json::from_slice(payload) { + Ok(withdraw) => withdraw, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!("malformed withdraw_init: {error}"))); + } + }; + let Some(row) = row else { + return Ok(ApplyOutcome::Applied); + }; + if self.actor() != Actor::Operator || !self.owns(row) { + return Ok(ApplyOutcome::Applied); + } + if state.retired_withdraws.contains(&withdraw.tx_hash) { + debug!(event = "graph_kickoff", graph_id = %row.graph_id, tx_hash = %withdraw.tx_hash, "withdraw was cancelled; ignored"); + return Ok(ApplyOutcome::Applied); + } + let status = GraphStatus::from_str(&row.status).map_err(|error| anyhow!("{error}"))?; + if status.is_closed() || status.is_pegout_started() { + state.withdraw = None; + return Ok(ApplyOutcome::Applied); + } + // A new init replaces the active one: Goat accepts it only after the + // previous withdraw was cancelled, so that cancel was missed. The jobs + // check the graph's Goat withdraw status, whichever hash they carry. + if let Some(active) = state.withdraw.take_if(|active| *active != withdraw.tx_hash) { + for action in [JOB_KICKOFF, JOB_SETTLE_PREDECESSORS] { + ctx.effects.cancel_job(kickoff_job(ctx, action, &active)?.job_id); + } + info!(event = "graph_kickoff", graph_id = %row.graph_id, replaced = %active, tx_hash = %withdraw.tx_hash, "withdraw replaced"); + state.retired_withdraws.push(active); + } + state.withdraw = Some(withdraw.tx_hash.clone()); + let graph_nonce = row.kickoff_index as u64; + let settle = kickoff_job(ctx, JOB_SETTLE_PREDECESSORS, &withdraw.tx_hash)?; + match predecessor_settled(ctx, &row.operator_pubkey, graph_nonce).await? { + None => { + let kickoff = kickoff_job(ctx, JOB_KICKOFF, &withdraw.tx_hash)?; + ctx.effects.cancel_job(settle.job_id); + ctx.effects.enqueue_job(kickoff); + info!(event = "graph_kickoff", graph_id = %row.graph_id, graph_nonce, "previous slot settled; kickoff queued"); + Ok(ApplyOutcome::Applied) + } + Some(detail) => { + ctx.effects.enqueue_job(settle); + Ok(ApplyOutcome::Waiting(WaitingFor::on_ref( + slot_ref(&row.operator_pubkey, graph_nonce - 1), + detail, + ))) + } + } + } + + /// `CancelWithdraw`: back to AwaitWithdraw; the kickoff jobs go. + pub(super) fn apply_withdraw_cancel( + &self, + ctx: &mut ApplyContext<'_>, + state: &mut GraphState, + ) -> ApplyOutcome { + if let Some(active) = state.withdraw.take() { + info!(event = "graph_kickoff", aggregate = %ctx.aggregate, tx_hash = %active, "withdraw cancelled"); + state.retired_withdraws.push(active); + ctx.effects.cancel_phase_jobs(PHASE_KICKOFF); + } + ApplyOutcome::Applied + } + + /// A chain scan: reconcile the status, mark dead slots, and hand every + /// signal to this role's job. + pub(super) async fn apply_chain_scan( + &self, + ctx: &mut ApplyContext<'_>, + graph_id: Uuid, + state: &mut GraphState, + row: Option<&Graph>, + payload: &[u8], + ) -> anyhow::Result { + let scan: ChainScan = match serde_json::from_slice(payload) { + Ok(scan) => scan, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!("malformed chain_scan: {error}"))); + } + }; + let Some(row) = row else { + return Ok(ApplyOutcome::Applied); + }; + let failed = super::common::instance_failed(ctx, row.instance_id).await?; + if let Some(target) = decide_status(&scan, failed) { + let outcome = ctx + .storage + .transition_graph_status( + row.instance_id, + graph_id, + target, + GraphStatusSource::ChainReconcile, + scan.sub_status.clone(), + ) + .await?; + if let GraphStatusTransitionOutcome::Rejected { current } = outcome { + debug!(event = "graph_scan", graph_id = %graph_id, target = %target, current = %current, "stale chain stage ignored"); + } + } + if let Some(challenge_txid) = scan.challenge_txid + && row.challenge_txid.as_ref().map(|txid| txid.0) != Some(challenge_txid) + { + ctx.storage + .update_graph_runtime( + &GraphRuntimeUpdate::new(row.instance_id, graph_id) + .with_challenge_txid(challenge_txid.into()), + ) + .await?; + } + if let Some(Unusable::PrekickoffInputSpent { outpoint, spent_by }) = &scan.unusable { + report_dead(state, row, outpoint, spent_by); + } + let own = self.owns(row); + for signal in &scan.signals { + if let Some(action) = route_signal(signal, &self.actor(), self.relayer(), own) { + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + PHASE_SETTLEMENT, + action, + pegout_job_pool(action), + signal.encode()?, + )); + } + // The operator asserted: the watchtower window is over. + if self.actor() == Actor::Watchtower + && matches!(signal, PegoutSignal::AssertSent { .. }) + { + ctx.effects.cancel_job( + JobSpec::new( + &ctx.aggregate, + PHASE_SETTLEMENT, + JOB_WATCHTOWER_CHALLENGE, + pegout_job_pool(JOB_WATCHTOWER_CHALLENGE), + PegoutSignal::WatchtowerChallengeInitSent.encode()?, + ) + .job_id, + ); + } + } + Ok(ApplyOutcome::Applied) + } +} + +/// Report a dead Pre-Kickoff once. Each later slot of the operator is +/// obsoleted by its own scan, which reports `EarlierSlotDead`. +fn report_dead(state: &mut GraphState, row: &Graph, outpoint: &OutPoint, spent_by: &Txid) { + if state.dead { + return; + } + state.dead = true; + warn!( + event = "graph_dead", + graph_id = %row.graph_id, + operator_pubkey = %row.operator_pubkey, + slot = row.kickoff_index, + outpoint = %outpoint, + spent_by = %spent_by, + "Pre-Kickoff can never confirm; this slot and every later slot of the operator are unusable" + ); + if let Some(metrics) = node_metrics_state() { + metrics.record_instance_alert("operator_chain_dead"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use GraphStatus::*; + + fn scan(stage: GraphStatus, unusable: Option) -> ChainScan { + ChainScan { + height: 1, + status_at_scan: String::new(), + stage, + sub_status: None, + challenge_txid: None, + unusable, + signals: Vec::new(), + } + } + + fn taken() -> Option { + Some(Unusable::ConnectorZeroTaken { spent_by: Txid::from_str(&"11".repeat(32)).unwrap() }) + } + + #[test] + fn chain_progress_beats_the_projection() { + let live = |stage, unusable| decide_status(&scan(stage, unusable), false); + assert_eq!(live(PreKickoff, taken()), Some(PreKickoff)); + assert_eq!(live(Challenge, taken()), Some(Challenge)); + assert_eq!(live(Skipped, None), Some(Skipped)); + assert_eq!(live(OperatorDataPushed, taken()), Some(Obsoleted)); + assert_eq!(live(CommitteePresigned, taken()), Some(Obsoleted)); + // A later slot of a dead chain stays Obsoleted on its own scans. + let behind = Some(Unusable::EarlierSlotDead { slot: 2 }); + assert_eq!(live(OperatorDataPushed, behind), Some(Obsoleted)); + // Posted overrides a local failure; nothing else is inferred. + assert_eq!(live(OperatorDataPushed, None), Some(OperatorDataPushed)); + assert_eq!(decide_status(&scan(OperatorDataPushed, None), true), Some(OperatorDataPushed)); + assert_eq!(live(CommitteePresigned, None), None); + assert_eq!(live(OperatorPresigned, None), None); + // A copy stored after its instance failed follows the failure. + assert_eq!(decide_status(&scan(OperatorPresigned, None), true), Some(Obsoleted)); + assert_eq!(decide_status(&scan(PreKickoff, None), true), Some(PreKickoff)); + } + + #[test] + fn signals_route_to_the_acting_role() { + use Actor::*; + use PegoutSignal::*; + assert_eq!(route_signal(&Take1Ready, &Operator, false, true), Some(JOB_TAKE1)); + assert_eq!(route_signal(&Take1Ready, &Operator, false, false), None); + assert_eq!(route_signal(&Take1Sent, &Committee, true, false), Some(JOB_FINISH_HAPPY)); + assert_eq!(route_signal(&Take1Sent, &Committee, false, false), None); + assert_eq!( + route_signal(&PreKickoffSent, &Verifier, false, false), + Some(JOB_ENFORCE_PREVIOUS_SLOT) + ); + assert_eq!( + route_signal(&KickoffSent, &Verifier, false, false), + Some(JOB_CHALLENGE_KICKOFF) + ); + assert_eq!(route_signal(&KickoffSent, &Committee, true, false), Some(JOB_PROCEED_WITHDRAW)); + assert_eq!(route_signal(&KickoffSent, &Watchtower, false, false), None); + let assert = AssertSent { assert_txid: Txid::from_str(&"22".repeat(32)).unwrap() }; + assert_eq!(route_signal(&assert, &Verifier, false, false), Some(JOB_VERIFIER_ASSERT)); + assert_eq!( + route_signal(&assert, &Watchtower, false, false), + Some(JOB_WATCHTOWER_PROOF_TIMEOUT) + ); + assert_eq!( + route_signal(&WatchtowerChallengeInitSent, &Watchtower, false, false), + Some(JOB_WATCHTOWER_CHALLENGE) + ); + let signal = DisproveSent { + disprove_type: DisproveTxType::QuickChallenge, + index: 0, + challenge_finish_txid: Txid::from_str(&"33".repeat(32)).unwrap(), + }; + assert_eq!(PegoutSignal::decode(&signal.encode().unwrap()).unwrap(), signal); + for action in PEGOUT_JOBS { + assert!(!super::super::graph::ALL_JOBS.contains(action), "{action} is a setup job"); + } + } + + #[test] + fn computing_pegout_jobs_have_their_own_pool() { + use super::super::graph::{JOB_BUILD_GRAPH, job_pool}; + for action in + [JOB_WATCHTOWER_CHALLENGE, JOB_COMMIT_PUBIN, JOB_PROVER_ASSERT, JOB_VERIFIER_ASSERT] + { + assert_eq!(job_pool(action), JobPool::Pegout, "{action}"); + } + assert_eq!(job_pool(JOB_WRONGLY_CHALLENGED), JobPool::Pegout); + assert_eq!(job_pool(JOB_TAKE1), JobPool::Network); + assert_eq!(job_pool(JOB_BUILD_GRAPH), JobPool::Cpu); + } + + fn local(status: GraphStatus, finalized: bool, instance: InstanceBridgeInStatus) -> SlotFacts { + SlotFacts { + slot: 3, + local: Some(LocalSlot { + graph_id: Uuid::from_u128(1), + instance_id: Uuid::from_u128(2), + status, + finalized, + instance_status: Some(instance), + }), + ..Default::default() + } + } + + fn skip(verdict: SlotVerdict) -> bool { + matches!(verdict, SlotVerdict::Skip(_)) + } + + #[test] + fn a_graph_confirmed_in_the_buffer_is_never_skipped() { + use InstanceBridgeInStatus::*; + // Peg-In-Confirm confirmed in the buffer: the instance moved on, the + // graph is live whether or not it is posted yet. + let mut facts = local(CommitteePresigned, true, RelayerL1Broadcasted); + facts.confirm = TxFact::confirmed_at(100); + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + facts.posted = true; + facts.local.as_mut().unwrap().status = OperatorDataPushed; + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + // Locally abandoned, but the confirm landed after all: alert, no skip. + let mut facts = local(Obsoleted, true, Timeout); + facts.confirm = TxFact::confirmed_at(100); + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Stuck(_))); + facts.confirm = TxFact::mempool(); + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + // Neither confirmed nor visible: the timed-out slot is skipped. + facts.confirm = TxFact::UNKNOWN; + assert_eq!(decide_slot(&facts, "op"), SlotVerdict::Skip("instance timed out")); + } + + #[test] + fn missing_or_unfinalized_copies_are_pulled_before_deciding() { + use InstanceBridgeInStatus::*; + let missing = SlotFacts { slot: 4, ..Default::default() }; + assert_eq!( + decide_slot(&missing, "op"), + SlotVerdict::Pull(GraphLookup::Slot { + operator_pubkey: "op".to_string(), + graph_nonce: 4 + }) + ); + // Posted on Goat but only operator pre-signed here: pull the + // finalized copy, never treat it as abandoned. + let mut facts = local(Obsoleted, false, UserCanceled); + facts.posted = true; + facts.cancel_confirmed = true; + assert_eq!( + decide_slot(&facts, "op"), + SlotVerdict::Pull(GraphLookup::Id { + instance_id: Uuid::from_u128(2), + graph_id: Uuid::from_u128(1), + finalized: true, + }) + ); + // Without evidence, an unfinished graph of a failed instance is + // abandoned. + let facts = local(Obsoleted, false, NoEnoughCommitteesAnswered); + assert_eq!(decide_slot(&facts, "op"), SlotVerdict::Skip("instance failed")); + // Of a live instance it is not, until the instance passed its + // abandon height: other graphs carried it, this one is never signed. + let mut facts = local(OperatorPresigned, false, RelayerL2Minted); + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + facts.past_abandon_height = true; + assert_eq!( + decide_slot(&facts, "op"), + SlotVerdict::Skip("unfinished at the abandon height") + ); + // Posted after all: pull the finalized copy first. + facts.posted = true; + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Pull(_))); + // A finalized graph past that height is still live ... + let mut facts = local(CommitteePresigned, true, RelayerL1Broadcasted); + facts.past_abandon_height = true; + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + // ... unless the operator gave it up at that height and the finalize + // only landed afterwards: Obsoleted under a finalized definition. + let mut facts = local(Obsoleted, true, RelayerL1Broadcasted); + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + facts.past_abandon_height = true; + assert_eq!( + decide_slot(&facts, "op"), + SlotVerdict::Skip("unfinished at the abandon height") + ); + } + + #[test] + fn skip_premises_are_chain_facts() { + use InstanceBridgeInStatus::*; + // Skipped or force-skipped on chain, or closed locally: settled. + let mut facts = local(Obsoleted, true, Timeout); + facts.kickoff_connector = KickoffConnectorSpend::ByOther; + assert_eq!(decide_slot(&facts, "op"), SlotVerdict::Settled); + assert_eq!( + decide_slot(&local(OperatorTake1, true, RelayerL2Minted), "op"), + SlotVerdict::Settled + ); + // Kicked off: wait for its settlement, never skip. + let mut facts = local(Obsoleted, true, RelayerL2Minted); + facts.kickoff_connector = KickoffConnectorSpend::ByKickoff; + facts.connector_zero_taken = true; + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + // Posted and available: the successor waits (withdraw it first). + let mut facts = local(OperatorDataPushed, true, RelayerL2Minted); + facts.posted = true; + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + // ... unless another graph took Connector-0. + facts.connector_zero_taken = true; + assert!(skip(decide_slot(&facts, "op"))); + // Canceled peg-in: Connector-Z is gone. + let mut facts = local(Obsoleted, true, UserCanceled); + facts.cancel_confirmed = true; + assert_eq!(decide_slot(&facts, "op"), SlotVerdict::Skip("peg-in canceled")); + // The Obsoleted projection alone proves nothing. + let facts = local(Obsoleted, true, Presigned); + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Wait(_))); + // A dead Pre-Kickoff cannot be skipped either. + let mut facts = local(Obsoleted, true, Timeout); + facts.prekickoff_dead = true; + assert!(matches!(decide_slot(&facts, "op"), SlotVerdict::Stuck(_))); + } +} diff --git a/node/src/aggregates/graph_pegout_jobs.rs b/node/src/aggregates/graph_pegout_jobs.rs new file mode 100644 index 000000000..42ad9088a --- /dev/null +++ b/node/src/aggregates/graph_pegout_jobs.rs @@ -0,0 +1,2098 @@ +//! Jobs for kickoff, challenges and withdraw settlement. +//! +//! Jobs query chain state before taking action. A mempool spend does not settle +//! a slot; pending transactions are checked again until confirmed, and local +//! broadcasts can be repeated if the transaction disappears. + +use super::graph::{GraphPlan, graph_pulled_event, slot_ref}; +use super::graph_fetch::fetch_and_store_graph; +use super::graph_jobs::Step; +use super::graph_pegout::{ + JOB_CHALLENGE_ACK, JOB_CHALLENGE_KICKOFF, JOB_CHALLENGE_NACK, JOB_COMMIT_PUBIN, + JOB_COMMIT_TIMEOUT, JOB_DISPROVE, JOB_ENFORCE_PREVIOUS_SLOT, JOB_FINISH_DISPROVED, + JOB_FINISH_HAPPY, JOB_FINISH_UNHAPPY, JOB_KICKOFF, JOB_PROCEED_WITHDRAW, JOB_PROVER_ASSERT, + JOB_SETTLE_PREDECESSORS, JOB_TAKE1, JOB_TAKE2, JOB_VERIFIER_ASSERT, JOB_WATCHTOWER_CHALLENGE, + JOB_WATCHTOWER_CHALLENGE_INIT, JOB_WATCHTOWER_PROOF_TIMEOUT, JOB_WATCHTOWER_TIMEOUT, + JOB_WRONGLY_CHALLENGED, KickoffConnectorSpend, KickoffParams, LocalSlot, PegoutSignal, + SlotFacts, SlotVerdict, decide_slot, +}; +use super::graph_scan::{connector_zero_taken, prekickoff_input_spent}; +use super::graph_setup::{ + build_babe_prover_state, decode_soldering_proof_payload, find_verifier_index_by_pubkey, + selected_candidate_for_graph_index, validate_verifier_slot, +}; +use super::instance::{PeginDeadlines, TxFact}; +use super::jobs::{JobDeps, Plan, local_operator}; +use crate::engine_bridge::submit_event; +use crate::env::{get_bitvm_key, get_soldering_proof_payload_store_path}; +use crate::metrics_service::node_metrics_state; +use crate::soldering_payload_store::{ + read_soldering_proof_store_payload, soldering_proof_payload_store_path, +}; +use crate::utils::*; +use anyhow::{Context, Result, anyhow, bail}; +use ark_serialize::CanonicalSerialize; +use bitcoin::{Amount, PublicKey, Txid}; +use bitvm_lib::babe_adapter::{ + BabeChallengeAssertWitness, ChallengeAssertWitnessRaw, TxAssertWitness, WOTS_SIG_COUNT, + assert_wots_message, build_assert_witness, build_real_challenge_assert_witness, + expand_compact_soldering_proof_payload, recover_operator_proof_from_assert_witness, + recover_real_wrongly_challenged_witness, +}; +use bitvm_lib::keys::{VerifierMasterKey, WatchtowerMasterKey}; +use bitvm_lib::operator::*; +use bitvm_lib::types::BitvmGcGraph; +use bitvm_lib::verifier::*; +use client::goat_chain::{DisproveTxType, WithdrawStatus}; +use goat::transactions::base::output_topology; +use goat::transactions::pre_signed::PreSignedTransaction; +use goat::wots::{Wots, Wots96}; +use std::str::FromStr; +use store::localdb::{GraphQuery, LocalDB}; +use store::{Graph, GraphStatus, InstanceBridgeInStatus, SerializableTxid}; +use tracing::{info, warn}; +use uuid::Uuid; + +pub(super) async fn run_pegout_step( + deps: &JobDeps, + action: &str, + graph_id: Uuid, + params: &[u8], + attempts: i64, +) -> Result { + match action { + JOB_KICKOFF => kickoff(deps, graph_id, params).await, + JOB_SETTLE_PREDECESSORS => settle_predecessors(deps, graph_id, params).await, + _ => { + let signal = PegoutSignal::decode(params).context("decode peg-out signal")?; + let Some((row, graph)) = load_graph(&deps.local_db, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + settlement_step(deps, action, &row, graph, signal, attempts).await + } + } +} + +async fn settlement_step( + deps: &JobDeps, + action: &str, + row: &Graph, + graph: BitvmGcGraph, + signal: PegoutSignal, + attempts: i64, +) -> Result { + use PegoutSignal::*; + // A closed graph's branches are over; it may still owe the relayer's Goat + // call, and its Pre-Kickoff still asks about the previous slot. + let after_close = matches!( + action, + JOB_PROCEED_WITHDRAW + | JOB_FINISH_HAPPY + | JOB_FINISH_UNHAPPY + | JOB_FINISH_DISPROVED + | JOB_ENFORCE_PREVIOUS_SLOT + ); + if closed(row) && !after_close { + return Step::ignore(format!("graph is {}", row.status)); + } + match (action, signal) { + (JOB_ENFORCE_PREVIOUS_SLOT, _) => enforce_previous_slot(deps, &graph).await, + (JOB_PROCEED_WITHDRAW, _) => proceed_withdraw(deps, row, &graph, attempts).await, + (JOB_CHALLENGE_KICKOFF, _) => challenge_kickoff(deps, &graph).await, + (JOB_WATCHTOWER_CHALLENGE_INIT, ChallengeSent { challenge_txid }) => { + watchtower_challenge_init(deps, row, graph, challenge_txid).await + } + (JOB_WATCHTOWER_CHALLENGE, _) => watchtower_challenge(deps, row, &graph).await, + (JOB_WATCHTOWER_PROOF_TIMEOUT, _) => watchtower_proof_timeout(row, &graph).await, + (JOB_CHALLENGE_ACK, WatchtowerChallengeSent { watchtower_index }) => { + challenge_ack(deps, row, &graph, watchtower_index).await + } + (JOB_WATCHTOWER_TIMEOUT, _) => watchtower_timeout(deps, row, graph).await, + (JOB_CHALLENGE_NACK, NackReady { watchtower_index }) => { + challenge_nack(deps, row, &graph, watchtower_index).await + } + (JOB_COMMIT_PUBIN, _) => commit_pubin(deps, &graph).await, + (JOB_COMMIT_TIMEOUT, _) => commit_timeout(deps, row, &graph).await, + (JOB_PROVER_ASSERT, _) => prover_assert(deps, graph).await, + (JOB_VERIFIER_ASSERT, AssertSent { assert_txid }) => { + verifier_assert(deps, &graph, assert_txid).await + } + (JOB_WRONGLY_CHALLENGED, ChallengeAssertSent { challenge_assert_txid, verifier_index }) => { + wrongly_challenged(deps, &graph, challenge_assert_txid, verifier_index).await + } + (JOB_DISPROVE, WronglyChallengeTimeout { challenge_assert_txid, verifier_index }) => { + disprove(deps, &graph, challenge_assert_txid, verifier_index).await + } + ( + JOB_FINISH_DISPROVED, + DisproveSent { disprove_type, index, challenge_finish_txid, .. }, + ) => { + finish_disproved(deps, &graph, disprove_type, index, challenge_finish_txid, attempts) + .await + } + (JOB_TAKE1, _) => take1(deps, row, graph).await, + (JOB_FINISH_HAPPY, _) => finish_happy(deps, &graph, attempts).await, + (JOB_TAKE2, _) => take2(deps, row, graph).await, + (JOB_FINISH_UNHAPPY, _) => finish_unhappy(deps, &graph, attempts).await, + (action, signal) => bail!("{action} does not act on {signal:?}"), + } +} + +// ------------------------------------------------------------------ helpers + +/// The graph row with its validated definition. +async fn load_graph(local_db: &LocalDB, graph_id: Uuid) -> Result> { + let mut storage = local_db.acquire().await?; + let Some(row) = storage.find_graph(&graph_id).await? else { + return Ok(None); + }; + let Some(definition) = + load_validated_graph_definition(&mut storage, row.instance_id, graph_id).await? + else { + return Ok(None); + }; + Ok(Some((row, BitvmGcGraph::from_simplified(&definition)?))) +} + +fn status_of(row: &Graph) -> Option { + GraphStatus::from_str(&row.status).ok() +} + +/// Closed graphs only owe the relayer's Goat call; their branches are over. +fn closed(row: &Graph) -> bool { + status_of(row).is_some_and(|status| status.is_closed()) +} + +fn block_secs(deps: &JobDeps) -> i64 { + avg_block_time_secs(deps.btc_client.network()) as i64 +} + +/// Retry after `blocks` blocks (at least one). +fn after_blocks(deps: &JobDeps, reason: impl Into, blocks: u64) -> Result { + Step::retry(reason, block_secs(deps) * blocks.max(1) as i64) +} + +/// What the chain says about an outpoint: a spend that is only in the mempool can still be +/// replaced, so a job waits on it instead of deciding on it. +enum Spend { + Unspent, + Pending(Txid), + Confirmed(Txid), +} + +async fn spend_of(btc: &client::btc_chain::BTCClient, txid: &Txid, vout: u64) -> Result { + Ok(match outpoint_spent_txid(btc, txid, vout).await? { + None => Spend::Unspent, + Some(spender) if btc.get_tx_status(&spender).await?.confirmed => Spend::Confirmed(spender), + Some(spender) => Spend::Pending(spender), + }) +} + +/// The step a connector's spend settles, `None` while it is unspent: a +/// confirmed spend ends the job, a pending one is looked at again. +fn settled_by(deps: &JobDeps, spend: &Spend, what: &str) -> Option> { + match spend { + Spend::Unspent => None, + Spend::Confirmed(by) => Some(Step::ignore(format!("{what} already spent by {by}"))), + Spend::Pending(by) => { + Some(after_blocks(deps, format!("{what} spent by unconfirmed {by}"), 1)) + } + } +} + +/// Whether one of the graph's transactions is confirmed, in the mempool or +/// nowhere. +enum Presence { + Absent, + Mempool, + Confirmed, +} + +async fn presence_of(btc: &client::btc_chain::BTCClient, txid: &Txid) -> Result { + Ok(match btc.get_tx_info(txid).await? { + None => Presence::Absent, + Some(tx) if tx.status.confirmed => Presence::Confirmed, + Some(_) => Presence::Mempool, + }) +} + +/// The step an own transaction's presence settles, `None` while it is absent. +fn present(deps: &JobDeps, presence: &Presence, what: &str) -> Option> { + match presence { + Presence::Absent => None, + Presence::Confirmed => Some(Step::ignore(format!("{what} is confirmed"))), + Presence::Mempool => Some(after_blocks(deps, format!("{what} is in the mempool"), 1)), + } +} + +/// After a broadcast: look again next block, so a transaction that fell out +/// of the mempool is sent again and a confirmed one ends the job. +fn broadcast_pending(deps: &JobDeps, what: &str) -> Result { + after_blocks(deps, format!("{what} broadcast; waiting for its confirmation"), 1) +} + +fn alert(reason: &str) { + if let Some(metrics) = node_metrics_state() { + metrics.record_instance_alert(reason); + } +} + +/// The step that ends an operator's kickoff job unless its withdraw is live +/// on Goat. `None` is a lagging Goat node (the `InitWithdraw` came from a +/// finalized block), so it is looked at again; any other status is over. +async fn step_unless_withdraw_live( + deps: &JobDeps, + graph_id: Uuid, + tx_hash: &str, +) -> Result> { + Ok(match deps.goat_client.gateway_get_withdraw_data(&graph_id).await?.status { + WithdrawStatus::Initialized => None, + WithdrawStatus::None => { + Some(after_blocks(deps, format!("withdraw {tx_hash} is not visible on Goat yet"), 1)?) + } + other => Some(Step::ignore(format!("withdraw {tx_hash} is {other:?}"))?), + }) +} + +// ------------------------------------------------------------------- kickoff + +/// Pre-Kickoff + Kickoff of the operator's graph; the apply queued it once +/// the previous slot settled. +async fn kickoff(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let params: KickoffParams = serde_json::from_slice(params).context("decode kickoff params")?; + let Some((_, mut graph)) = load_graph(&deps.local_db, graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + let (_, operator_pubkey) = local_operator()?; + if graph.parameters.operator_pubkey != operator_pubkey { + return Step::ignore("not this operator's graph"); + } + if let Some(step) = step_unless_withdraw_live(deps, graph_id, ¶ms.tx_hash).await? { + return Ok(step); + } + let kickoff_txid = graph.kickoff.tx().compute_txid(); + let prekickoff_txid = graph.cur_prekickoff.tx().compute_txid(); + let kickoff_connector = output_topology::prekickoff::kickoff_connector() as u64; + match outpoint_spent_txid(&deps.btc_client, &prekickoff_txid, kickoff_connector).await? { + Some(spent_by) if spent_by != kickoff_txid => { + if !deps.btc_client.get_tx_status(&spent_by).await?.confirmed { + return after_blocks( + deps, + format!("kickoff connector spent by unconfirmed {spent_by}"), + 1, + ); + } + return Step::ignore(format!("kickoff connector already spent by {spent_by}")); + } + // Broadcast but not confirmed: the retry below re-sends the CPFP + // children an earlier attempt may not have reached. + Some(_) if deps.btc_client.get_tx_status(&kickoff_txid).await?.confirmed => { + return Step::ignore("already kicked off"); + } + _ => {} + } + if let Some((outpoint, spent_by)) = prekickoff_input_spent(&deps.btc_client, &graph).await? { + alert("operator_chain_dead"); + return Step::ignore(format!( + "Pre-Kickoff can never confirm: {outpoint} was spent by {spent_by}" + )); + } + operator_kickoff(&deps.btc_client, &mut graph).await?; + info!(event = "graph_kickoff", graph_id = %graph_id, tx_hash = %params.tx_hash, "Pre-Kickoff and Kickoff broadcast"); + broadcast_pending(deps, &format!("Pre-Kickoff and Kickoff for withdraw {}", params.tx_hash)) +} + +/// Settle earlier operator slots in order: skip eligible slots, fetch missing +/// definitions and wait for unresolved slots. Completion unblocks kickoff. +async fn settle_predecessors(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> Result { + let params: KickoffParams = serde_json::from_slice(params).context("decode kickoff params")?; + let Some(row) = deps.local_db.acquire().await?.find_graph(&graph_id).await? else { + return Step::ignore("graph not stored locally"); + }; + let (_, operator_pubkey) = local_operator()?; + if row.operator_pubkey != operator_pubkey.to_string() { + return Step::ignore("not this operator's graph"); + } + if let Some(step) = step_unless_withdraw_live(deps, graph_id, ¶ms.tx_hash).await? { + return Ok(step); + } + let graph_nonce = row.kickoff_index as u64; + let chain = ChainClock { + height: i64::from(deps.btc_client.get_height().await?), + deadlines: PeginDeadlines::from_env(), + }; + for slot in first_open_slot(&deps.local_db, &row).await?..graph_nonce { + let mut facts = slot_facts(deps, &row.operator_pubkey, slot, &chain).await?; + let mut verdict = decide_slot(&facts, &row.operator_pubkey); + if let SlotVerdict::Pull(lookup) = &verdict { + match fetch_and_store_graph( + &deps.local_db, + &deps.btc_client, + &deps.goat_client, + &deps.publisher, + lookup, + ) + .await + { + Ok(Some(found)) => { + submit_event( + &deps.local_db, + graph_pulled_event(found.graph_id, found.instance_id, found.finalized)?, + ) + .await?; + facts = slot_facts(deps, &row.operator_pubkey, slot, &chain).await?; + verdict = decide_slot(&facts, &row.operator_pubkey); + } + Ok(None) => { + return Step::retry(format!("slot {slot}: no peer served {lookup:?}"), 60); + } + Err(error) => { + return Step::retry(format!("slot {slot}: pull failed: {error:#}"), 60); + } + } + } + match verdict { + SlotVerdict::Settled => continue, + SlotVerdict::Pull(lookup) => { + return Step::retry(format!("slot {slot}: still missing {lookup:?}"), 60); + } + SlotVerdict::Skip(premise) => { + let Some(local) = facts.local else { + bail!("slot {slot} has no local copy to skip"); + }; + let Some((_, mut skipped)) = load_graph(&deps.local_db, local.graph_id).await? + else { + bail!("slot {slot} definition disappeared"); + }; + operator_skip_graph(&deps.btc_client, &mut skipped).await?; + info!( + event = "graph_skip", + graph_id = %graph_id, + skipped_graph_id = %local.graph_id, + slot, + premise, + "Pre-Kickoff and Skip-Kickoff of an earlier slot broadcast" + ); + return after_blocks(deps, format!("skipped slot {slot} ({premise})"), 1); + } + SlotVerdict::Wait(reason) => return after_blocks(deps, reason, 1), + SlotVerdict::Stuck(reason) => { + warn!(event = "graph_skip", graph_id = %graph_id, slot, reason = %reason, "earlier slot needs manual handling"); + alert("pegout_predecessor_stuck"); + return after_blocks(deps, reason, 10); + } + } + } + let mut plan = GraphPlan::noted("every earlier slot is settled on chain"); + if graph_nonce > 0 { + plan.wake_refs.push(slot_ref(&row.operator_pubkey, graph_nonce - 1)); + } + Step::done(plan) +} + +/// One past the operator's highest closed slot below this graph: the slots +/// under it were settled when it advanced. +async fn first_open_slot(local_db: &LocalDB, row: &Graph) -> Result { + let closed: Vec = + GraphStatus::get_closed_status().iter().map(ToString::to_string).collect(); + let highest = local_db + .acquire() + .await? + .get_operator_graphs( + GraphQuery::default() + .with_operator_pubkey(row.operator_pubkey.clone()) + .with_statuses(closed) + .with_raw_condition(format!("kickoff_index < {}", row.kickoff_index)) + .with_order("kickoff_index DESC".to_string()) + .with_limit(1), + ) + .await? + .pop(); + Ok(highest.map(|graph| graph.kickoff_index as u64 + 1).unwrap_or(0)) +} + +/// Facts about one slot: the local copy, then the chain. +/// The Bitcoin height and peg-in deadlines a settle run judges every slot by. +struct ChainClock { + height: i64, + deadlines: PeginDeadlines, +} + +async fn slot_facts( + deps: &JobDeps, + operator_pubkey: &str, + slot: u64, + chain: &ChainClock, +) -> Result { + let mut facts = SlotFacts { slot, ..Default::default() }; + let (row, instance, definition) = { + let mut storage = deps.local_db.acquire().await?; + let Some(row) = storage.find_graph_by_operator_slot(operator_pubkey, slot as i64).await? + else { + return Ok(facts); + }; + let instance = storage.find_instance(&row.instance_id).await?; + let definition = + load_validated_graph_definition(&mut storage, row.instance_id, row.graph_id).await?; + (row, instance, definition) + }; + let Some(definition) = definition else { + return Ok(facts); + }; + let graph = BitvmGcGraph::from_simplified(&definition)?; + let btc = &deps.btc_client; + let prekickoff_txid = graph.cur_prekickoff.tx().compute_txid(); + let kickoff_connector = output_topology::prekickoff::kickoff_connector() as u64; + facts.kickoff_connector = + match outpoint_spent_txid(btc, &prekickoff_txid, kickoff_connector).await? { + Some(spent_by) if spent_by == graph.kickoff.tx().compute_txid() => { + KickoffConnectorSpend::ByKickoff + } + // Settled only once the Skip / Force-Skip confirmed. + Some(spent_by) if btc.get_tx_status(&spent_by).await?.confirmed => { + KickoffConnectorSpend::ByOther + } + _ => KickoffConnectorSpend::Unspent, + }; + if facts.kickoff_connector == KickoffConnectorSpend::Unspent + && !tx_visible(btc, &prekickoff_txid).await? + { + facts.prekickoff_dead = prekickoff_input_spent(btc, &graph).await?.is_some(); + } + facts.posted = + deps.goat_client.gateway_get_graph_data(&row.graph_id).await?.operator_pubkey != [0u8; 32]; + facts.connector_zero_taken = connector_zero_taken(btc, &graph).await?.is_some(); + if let Some(instance) = &instance { + if let Some(cancel) = &instance.pegin_cancel_txid { + facts.cancel_confirmed = btc.get_tx_status(&cancel.0).await?.confirmed; + } + if let Some(confirm) = &instance.pegin_confirm_txid { + facts.confirm = match btc.get_tx_info(&confirm.0).await? { + None => TxFact::UNKNOWN, + Some(tx) if tx.status.confirmed => TxFact { + confirmed: true, + block_height: tx.status.block_height.map(i64::from), + in_mempool: false, + }, + Some(_) => TxFact::mempool(), + }; + } + } + let finalized = graph.committee_pre_signed(); + if let Some(instance) = &instance { + facts.past_abandon_height = chain + .deadlines + .abandon_height(instance.btc_height) + .is_some_and(|abandon_height| chain.height >= abandon_height); + } + facts.local = Some(LocalSlot { + graph_id: row.graph_id, + instance_id: row.instance_id, + status: GraphStatus::from_str(&row.status).map_err(|error| anyhow!("{error}"))?, + finalized, + instance_status: instance + .as_ref() + .and_then(|instance| InstanceBridgeInStatus::from_str(&instance.status).ok()), + }); + Ok(facts) +} + +// ---------------------------------------------------------------- verifier + +/// Pre-Kickoff-N is on chain: the previous slot must have been kicked off and +/// settled, or skipped. Force-Skip it if Kickoff-(N−1) never appeared, +/// Quick-Challenge it if it is still unsettled. The previous definition comes +/// from the store or, if missing, from the relayer / committee. +async fn enforce_previous_slot(deps: &JobDeps, graph: &BitvmGcGraph) -> Result { + let graph_nonce = graph.parameters.graph_nonce; + if graph_nonce == 0 { + return Step::ignore("first slot of the operator"); + } + let btc = &deps.btc_client; + // The scan saw the Pre-Kickoff confirmed; a read that disagrees (a lagging + // backend, a reorg) is looked at again rather than ending the job. + let prekickoff_txid = graph.cur_prekickoff.tx().compute_txid(); + if !tx_confirmed(btc, &prekickoff_txid).await? { + return after_blocks(deps, "pre-kickoff is not confirmed yet", 1); + } + let operator_pubkey = graph.parameters.operator_pubkey.to_string(); + let previous_slot = graph_nonce - 1; + let previous_id = deps + .local_db + .acquire() + .await? + .find_graph_by_operator_slot(&operator_pubkey, previous_slot as i64) + .await? + .map(|row| row.graph_id); + let previous = match previous_id { + Some(previous_id) => load_graph(&deps.local_db, previous_id).await?, + None => None, + }; + let (previous_row, previous) = match previous { + Some(previous) => previous, + None => { + let lookup = super::graph_fetch::GraphLookup::Slot { + operator_pubkey: operator_pubkey.clone(), + graph_nonce: previous_slot, + }; + match fetch_and_store_graph( + &deps.local_db, + btc, + &deps.goat_client, + &deps.publisher, + &lookup, + ) + .await + { + Ok(Some(found)) => { + submit_event( + &deps.local_db, + graph_pulled_event(found.graph_id, found.instance_id, found.finalized)?, + ) + .await?; + match load_graph(&deps.local_db, found.graph_id).await? { + Some(previous) => previous, + None => return Step::retry("pulled previous graph is not readable", 60), + } + } + Ok(None) => return Step::retry("no peer served the previous graph", 60), + Err(error) => { + return Step::retry( + format!("pulling the previous graph failed: {error:#}"), + 60, + ); + } + } + } + }; + let stage = scan_graph_chain_state(btc, &deps.goat_client, &previous).await?.status; + let previous_kickoff = previous.kickoff.tx().compute_txid(); + if !tx_visible(btc, &previous_kickoff).await? { + let previous_prekickoff = previous.cur_prekickoff.tx().compute_txid(); + let kickoff_connector = output_topology::prekickoff::kickoff_connector() as u64; + let spend = spend_of(btc, &previous_prekickoff, kickoff_connector).await?; + if let Some(step) = settled_by(deps, &spend, "previous slot's kickoff connector") { + return step; + } + let txid = verifier_force_skip_kickoff(btc, &previous).await?; + info!(event = "graph_punish", graph_id = %previous_row.graph_id, txid = %txid, "Force-Skip-Kickoff broadcast"); + return broadcast_pending(deps, &format!("Force-Skip-Kickoff {txid}")); + } + if stage.is_closed() { + return Step::ignore(format!("previous slot is {stage}")); + } + let guardian = output_topology::kickoff::guardian_connector() as u64; + let spend = spend_of(btc, &previous_kickoff, guardian).await?; + if let Some(step) = settled_by(deps, &spend, "previous slot's guardian connector") { + return step; + } + let txid = verifier_quick_challenge(btc, &previous).await?; + info!(event = "graph_punish", graph_id = %previous_row.graph_id, txid = %txid, "Quick-Challenge broadcast"); + broadcast_pending(deps, &format!("Quick-Challenge {txid}")) +} + +/// Kickoff confirmed: challenge it when Goat shows no live withdraw. +async fn challenge_kickoff(deps: &JobDeps, graph: &BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let graph_id = graph.parameters.graph_id; + let kickoff_txid = graph.kickoff.tx().compute_txid(); + let Some(kickoff_height) = btc.get_tx_status(&kickoff_txid).await?.block_height else { + return after_blocks(deps, "kickoff is not confirmed yet", 1); + }; + let take1_txid = graph.take1.tx().compute_txid(); + let (challenge_tx, _) = export_challenge_tx(graph)?; + let outpoint = challenge_tx.input[0].previous_output; + if let Some(spent_txid) = + outpoint_spent_txid(btc, &outpoint.txid, u64::from(outpoint.vout)).await? + { + // Only a confirmed spend settles it: a Challenge in the mempool may be + // ours (resent below if it is evicted) or a decoy an operator replaces. + if !btc.get_tx_status(&spent_txid).await?.confirmed { + return after_blocks( + deps, + format!("challenge connector spent by unconfirmed {spent_txid}"), + 1, + ); + } + let spender = if spent_txid == take1_txid { "Take-1" } else { "Challenge" }; + return Step::ignore(format!( + "challenge connector already spent by {spender} {spent_txid}" + )); + } + match deps.goat_client.gateway_get_withdraw_data(&graph_id).await?.status { + WithdrawStatus::None | WithdrawStatus::Canceled => {} + // Passing: the relayer's proceedWithdraw makes it Processing, a + // cancel makes it Canceled. Decided on the next look, not dropped. + WithdrawStatus::Initialized => { + return after_blocks( + deps, + "withdraw is initialized; waiting for proceedWithdraw or a cancel", + 1, + ); + } + other => return Step::ignore(format!("withdraw is {other:?}")), + } + let spv_height = deps.goat_client.btc_spv_latest_height().await? as u32; + if kickoff_height >= spv_height { + return after_blocks( + deps, + "kickoff block is not available through GOAT SPV", + u64::from(kickoff_height - spv_height), + ); + } + // The connector is unspent, so no Challenge of ours is pending. + let txid = send_challenge_tx(btc, graph).await?; + info!(event = "graph_challenge", graph_id = %graph_id, txid = %txid, "Challenge broadcast"); + broadcast_pending(deps, &format!("Challenge {txid}")) +} + +fn local_verifier_pubkey() -> Result { + Ok(VerifierMasterKey::new(get_bitvm_key()?).master_keypair().public_key().into()) +} + +/// An ACK-Connector unspent past `T5` after a watchtower challenge: NACK. +async fn challenge_nack( + deps: &JobDeps, + row: &Graph, + graph: &BitvmGcGraph, + watchtower_index: usize, +) -> Result { + let btc = &deps.btc_client; + let watchtower_num = validate_watchtower_branches(graph)?; + if watchtower_index >= watchtower_num { + bail!("watchtower index {watchtower_index} out of range for {watchtower_num} slots"); + } + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let connector_f = + output_topology::watchtower_challenge_init::connector_f(watchtower_num) as u64; + if let Some(step) = + settled_by(deps, &spend_of(btc, &wci_txid, connector_f).await?, "connector-F") + { + return step; + } + // The scan saw this watchtower's challenge confirmed; a read that disagrees + // is looked at again. + let watchtower_vout = + output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) as u64; + let Some(watchtower_spent) = outpoint_spent_txid(btc, &wci_txid, watchtower_vout).await? else { + return after_blocks( + deps, + format!("watchtower connector {watchtower_index} is not spent"), + 1, + ); + }; + if watchtower_spent == graph.watchtower_challenge_timeouts[watchtower_index].tx().compute_txid() + { + return Step::ignore(format!("watchtower {watchtower_index} timed out")); + } + // ACK or NACK (ours included) spend the ACK-Connector: confirmed, the + // branch is settled; pending, it is looked at again. + let ack_vout = + output_topology::watchtower_challenge_init::ack_connector(watchtower_index) as u64; + if let Some(step) = settled_by( + deps, + &spend_of(btc, &wci_txid, ack_vout).await?, + &format!("ack connector {watchtower_index}"), + ) { + return step; + } + let nack = graph + .operator_challenge_nacks + .get(watchtower_index) + .ok_or_else(|| anyhow!("invalid watchtower index {watchtower_index}"))?; + let nack_tx = nack.tx().clone(); + let nack_txid = nack_tx.compute_txid(); + let input_amount = nack.prev_outs().iter().map(|o| o.value).sum::(); + broadcast_tx_with_cpfp(btc, nack_tx, input_amount).await?; + info!(event = "graph_challenge", graph_id = %row.graph_id, watchtower_index, txid = %nack_txid, "Operator-Challenge-NACK broadcast"); + broadcast_pending(deps, &format!("Operator-Challenge-NACK {nack_txid}")) +} + +/// Commit-pubin missing past `T6`: Operator-Commit-Timeout. +async fn commit_timeout(deps: &JobDeps, row: &Graph, graph: &BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let watchtower_num = graph.parameters.watchtower_pubkeys.len(); + let connector_e = + output_topology::watchtower_challenge_init::connector_e(watchtower_num) as u64; + let connector_f = + output_topology::watchtower_challenge_init::connector_f(watchtower_num) as u64; + for (vout, what) in [(connector_e, "connector-E"), (connector_f, "connector-F")] { + if let Some(step) = settled_by(deps, &spend_of(btc, &wci_txid, vout).await?, what) { + return step; + } + } + let timeout_tx = graph.operator_commit_timeout.tx().clone(); + let timeout_txid = timeout_tx.compute_txid(); + let input_amount = + graph.operator_commit_timeout.prev_outs().iter().map(|o| o.value).sum::(); + broadcast_tx_with_cpfp(btc, timeout_tx, input_amount).await?; + info!(event = "graph_challenge", graph_id = %row.graph_id, txid = %timeout_txid, "Operator-Commit-Timeout broadcast"); + broadcast_pending(deps, &format!("Operator-Commit-Timeout {timeout_txid}")) +} + +/// Prover-Assert confirmed: Pubin-Disprove if the committed pubin and the +/// asserted one disagree, otherwise Verifier-Assert unless the proof is valid. +async fn verifier_assert(deps: &JobDeps, graph: &BitvmGcGraph, assert_txid: Txid) -> Result { + let btc = &deps.btc_client; + let instance_id = graph.parameters.instance_parameters.instance_id; + let graph_id = graph.parameters.graph_id; + let Some(assert_tx) = btc.get_tx(&assert_txid).await? else { + return after_blocks(deps, format!("assert tx {assert_txid} is not available"), 1); + }; + let verifier_pubkey = local_verifier_pubkey()?; + let Some(verifier_index) = + find_verifier_index_by_pubkey(&graph.parameters.gc_data, &verifier_pubkey)? + else { + return Step::ignore("local verifier has no graph slot"); + }; + validate_verifier_slot(graph, verifier_index)?; + // Connector-D of the assert goes to Take-2 or a Pubin-Disprove, this + // verifier's connector to its ChallengeAssert: either on chain (ours + // included) ends the job, either in the mempool is looked at again, before + // the proof work below is repeated. + let verifier_num = graph.parameters.gc_data.len(); + for (vout, what) in [ + (output_topology::operator_assert::connector_d(verifier_num) as u64, "assert connector-D"), + ( + output_topology::operator_assert::verifier_connector(verifier_index) as u64, + "verifier connector", + ), + ] { + if let Some(step) = settled_by(deps, &spend_of(btc, &assert_txid, vout).await?, what) { + return step; + } + } + + // ChallengeAssert may only be skipped after this graph's commit-pubin is + // on-chain and proves consistent with the assert witness. OperatorAssert + // can otherwise be broadcast before connector-E is spent. + let connector_e = graph + .watchtower_challenge_init + .connector_e_input() + .map_err(|error| anyhow!("failed to resolve connector-E input: {error}"))? + .outpoint; + let Some(connector_e_spent) = + outpoint_spent_txid(btc, &connector_e.txid, u64::from(connector_e.vout)).await? + else { + return after_blocks(deps, "operator commit-pubin is not on chain yet", 1); + }; + + let mut pubin_consistent = false; + if connector_e_spent != graph.operator_commit_timeout.tx().compute_txid() { + if !btc.get_tx_status(&connector_e_spent).await?.confirmed { + return after_blocks(deps, "operator commit-pubin is not confirmed yet", 1); + } + let Some(commit_pubin_tx) = btc.get_tx(&connector_e_spent).await? else { + return after_blocks(deps, "operator commit-pubin is not available yet", 1); + }; + let assert_txin = assert_tx + .input + .first() + .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; + let commit_pubin_txin = commit_pubin_tx + .input + .first() + .ok_or_else(|| anyhow!("operator commit-pubin transaction has no input"))?; + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let timeout_txids = graph + .watchtower_challenge_timeouts + .iter() + .map(|tx| tx.tx().compute_txid()) + .collect::>(); + let ack_txins = match collect_ack_txins(btc, &wci_txid, &timeout_txids).await { + Ok(txins) => txins, + Err(error) => { + return after_blocks( + deps, + format!("operator ACK inputs are not ready: {error}"), + 1, + ); + } + }; + match validate_pubin_disprove(graph, commit_pubin_txin, assert_txin, &ack_txins) { + Ok(Some((witness_data, _))) => { + // PubinDisprove is not pre-signed, so fund it directly rather + // than relying on CPFP. + let input_amount = graph + .operator_assert + .connector_d_input() + .map_err(|e| anyhow!("failed to get connector-d input: {e}"))? + .amount; + let txin = build_pubin_disprove_txin(graph, witness_data)?; + let outputs = vec![ + goat::scripts::p2a_output(), + // Keep the transaction above Bitcoin Core's minimum + // non-witness relay size. + bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: goat::scripts::generate_opreturn_script( + PUBIN_DISPROVE_OP_RETURN_DATA.to_vec(), + ), + }, + ]; + let verifier_keypair = VerifierMasterKey::new(get_bitvm_key()?).master_keypair(); + let txid = build_sign_and_broadcast_tx( + btc, + verifier_keypair, + vec![txin], + input_amount, + outputs, + ) + .await?; + info!(event = "graph_challenge", graph_id = %graph_id, txid = %txid, "Pubin-Disprove broadcast"); + return broadcast_pending(deps, &format!("Pubin-Disprove {txid}")); + } + Ok(None) => pubin_consistent = true, + Err(error) => warn!( + "PubinDisprove validation failed for {instance_id}:{graph_id}: {error}; proceeding to ChallengeAssert" + ), + } + } + + let assert_txin = assert_tx + .input + .first() + .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; + let assert_witness = extract_operator_assert_witness_for_challenge(graph, assert_txin) + .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; + let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for operator assert")?; + let static_input = derive_operator_static_input()?; + if pubin_consistent + && bitvm_lib::babe_adapter::verify_operator_assert_proof(&assert_witness, &vk, static_input) + { + return Step::ignore("operator assert proof is valid"); + } + let broadcast = broadcast_verifier_challenge_assert_tx( + &deps.local_db, + btc, + graph, + instance_id, + graph_id, + &assert_tx, + false, + ) + .await?; + if broadcast.is_none() { + // Without the BABE verifier state the assert cannot be challenged by + // this node; repeating the proof work every block would not change that. + warn!(event = "graph_challenge", graph_id = %graph_id, "ChallengeAssert cannot be built: BABE verifier state or soldering proof reference missing"); + alert("verifier_assert_unavailable"); + return Step::ignore("ChallengeAssert not built: missing BABE verifier state"); + } + broadcast_pending(deps, "ChallengeAssert") +} + +const PUBIN_DISPROVE_OP_RETURN_DATA: &[u8] = b"pubin-disprove"; + +pub async fn broadcast_verifier_challenge_assert_tx( + local_db: &LocalDB, + btc_client: &client::btc_chain::BTCClient, + graph: &BitvmGcGraph, + instance_id: Uuid, + graph_id: Uuid, + assert_tx: &bitcoin::Transaction, + strict: bool, +) -> Result> { + let verifier_pubkey = local_verifier_pubkey()?; + let Some(verifier_index) = + find_verifier_index_by_pubkey(&graph.parameters.gc_data, &verifier_pubkey)? + else { + if strict { + bail!("local verifier has no graph slot for {instance_id}:{graph_id}"); + } + tracing::debug!( + "Ignore AssertSent for {instance_id}:{graph_id}: local verifier has no graph slot" + ); + return Ok(None); + }; + validate_verifier_slot(graph, verifier_index)?; + + let assert_txin = assert_tx + .input + .first() + .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; + let assert_witness = extract_operator_assert_witness_for_challenge(graph, assert_txin) + .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; + let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for operator assert")?; + let static_input = derive_operator_static_input()?; + + let Some(saved_verifier_state) = + load_babe_setup_state(local_db, instance_id, graph_id)?.and_then(|state| state.verifier) + else { + if strict { + bail!("missing BABE verifier setup state for {instance_id}:{graph_id}"); + } + warn!("Ignore AssertSent for {instance_id}:{graph_id}: missing BABE verifier setup state"); + return Ok(None); + }; + if saved_verifier_state.soldering_proof_ready.is_none() { + if strict { + bail!("missing soldering proof reference for {instance_id}:{graph_id}"); + } + warn!("Ignore AssertSent for {instance_id}:{graph_id}: missing soldering proof reference"); + return Ok(None); + } + #[cfg(feature = "mock-gc")] + crate::mock_gc::simulate(crate::mock_gc::GcOp::Challenge).await; + let challenge_witness = build_real_challenge_assert_witness( + &saved_verifier_state.private_state, + &saved_verifier_state.setup_package, + &saved_verifier_state.finalized_indices, + &vk, + static_input, + &assert_witness, + verifier_index, + )?; + let labels: [Vec; goat::assert_scripts::INPUT_WIRE_NUM] = challenge_witness + .witness + .input_labels + .iter() + .map(|label| label.to_vec()) + .collect::>() + .try_into() + .map_err(|labels: Vec>| { + anyhow!( + "BABE challenge witness exposes {} labels; expected {}", + labels.len(), + goat::assert_scripts::INPUT_WIRE_NUM + ) + })?; + let operator_assert_txin = assert_tx + .input + .first() + .cloned() + .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; + let challenge_assert_tx = + build_verifier_assert_tx(graph, operator_assert_txin, verifier_index, labels)?; + let challenge_assert_txid = challenge_assert_tx.compute_txid(); + if btc_client.get_tx(&challenge_assert_txid).await?.is_some() { + info!( + "Verifier ChallengeAssert already exists for {instance_id}:{graph_id}: verifier_index={verifier_index}, txid={challenge_assert_txid}" + ); + return Ok(Some((challenge_assert_txid, verifier_index))); + } + let input_amount = + graph.verifier_asserts[verifier_index].prev_outs().iter().map(|o| o.value).sum::(); + broadcast_tx_with_cpfp(btc_client, challenge_assert_tx, input_amount).await?; + info!(event = "graph_challenge", graph_id = %graph_id, verifier_index, txid = %challenge_assert_txid, "Verifier-ChallengeAssert broadcast"); + Ok(Some((challenge_assert_txid, verifier_index))) +} + +/// A Verifier-Assert unanswered past `T2`: Disprove-X by its own verifier. +async fn disprove( + deps: &JobDeps, + graph: &BitvmGcGraph, + challenge_assert_txid: Txid, + verifier_index: usize, +) -> Result { + let btc = &deps.btc_client; + validate_expected_challenge_assert_txid(graph, verifier_index, challenge_assert_txid)?; + if graph.parameters.gc_data[verifier_index].verifier_pubkey != local_verifier_pubkey()? { + return Step::ignore(format!("local verifier does not own slot {verifier_index}")); + } + if btc.get_tx(&challenge_assert_txid).await?.is_none() { + return after_blocks(deps, "ChallengeAssert is not available from the Bitcoin backend", 1); + } + let Some(assert_height) = btc.get_tx_status(&challenge_assert_txid).await?.block_height else { + return after_blocks(deps, "ChallengeAssert is not confirmed", 1); + }; + let disprove_height = u64::from(assert_height) + + disprove_timelock_with_config( + graph.parameters.instance_parameters.network, + &graph.parameters.timelock_config, + ) as u64; + let height = u64::from(btc.get_height().await?); + if height < disprove_height { + return after_blocks(deps, "disprove timelock has not expired", disprove_height - height); + } + if let Some(step) = settled_by( + deps, + &spend_of(btc, &challenge_assert_txid, 0).await?, + "verifier assertion output", + ) { + return step; + } + let disprove_tx = build_disprove_tx(graph, verifier_index, None)?; + let disprove_txid = disprove_tx.compute_txid(); + let input_amount = + graph.disproves[verifier_index].prev_outs().iter().map(|o| o.value).sum::(); + broadcast_tx_with_cpfp(btc, disprove_tx, input_amount).await?; + info!(event = "graph_challenge", graph_id = %graph.parameters.graph_id, verifier_index, txid = %disprove_txid, "Disprove broadcast"); + broadcast_pending(deps, &format!("Disprove of slot {verifier_index}")) +} + +// ---------------------------------------------------------------- operator + +/// Challenge on Connector-A: Watchtower-Challenge-Init. +async fn watchtower_challenge_init( + deps: &JobDeps, + row: &Graph, + mut graph: BitvmGcGraph, + challenge_txid: Txid, +) -> Result { + let btc = &deps.btc_client; + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + if let Some(step) = + present(deps, &presence_of(btc, &wci_txid).await?, "Watchtower-Challenge-Init") + { + return step; + } + let Some(challenge_tx) = btc.get_tx(&challenge_txid).await? else { + return after_blocks(deps, "challenge tx is not available", 1); + }; + let connector_a = graph + .kickoff + .connector_a_input() + .map_err(|e| anyhow!("failed to get connector-a input: {e}"))? + .outpoint; + if challenge_tx.input.first().map(|input| input.previous_output) != Some(connector_a) { + return Step::ignore("challenge does not spend connector-A"); + } + let (master, _) = local_operator()?; + let wci_tx = operator_sign_watchtower_challenge_init(master.master_keypair(), &mut graph)?; + let anchor_vout = wci_tx.output.len() as u64 - 1; + let input_amount = graph.watchtower_challenge_init.prev_outs().iter().map(|o| o.value).sum(); + match build_cpfp_txns(btc, &wci_tx, anchor_vout, input_amount).await? { + Some(child) => broadcast_package(btc, &[wci_tx, child], true).await?, + None => broadcast_tx(btc, &wci_tx).await?, + }; + info!(event = "graph_challenge", graph_id = %row.graph_id, txid = %wci_txid, "Watchtower-Challenge-Init broadcast"); + broadcast_pending(deps, &format!("Watchtower-Challenge-Init {wci_txid}")) +} + +/// A watchtower challenged: reveal the ACK preimage. +async fn challenge_ack( + deps: &JobDeps, + row: &Graph, + graph: &BitvmGcGraph, + watchtower_index: usize, +) -> Result { + let btc = &deps.btc_client; + let watchtower_num = validate_watchtower_branches(graph)?; + if watchtower_index >= watchtower_num { + bail!("watchtower index {watchtower_index} out of range for {watchtower_num} slots"); + } + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let watchtower_vout = + output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) as u64; + // The scan saw this watchtower's challenge confirmed; a read that disagrees + // is looked at again rather than ending the only ACK job of the branch. + let Some((spent_txid, vin, _)) = outpoint_spent_txin(btc, &wci_txid, watchtower_vout).await? + else { + return after_blocks( + deps, + format!("watchtower connector {watchtower_index} is not spent"), + 1, + ); + }; + if vin != 0 { + return Step::ignore(format!( + "watchtower connector {watchtower_index} spent by {spent_txid} at input {vin}" + )); + } + let timeout_txid = graph.watchtower_challenge_timeouts[watchtower_index].tx().compute_txid(); + if spent_txid == timeout_txid { + return Step::ignore(format!("watchtower {watchtower_index} timed out")); + } + let ack_vout = + output_topology::watchtower_challenge_init::ack_connector(watchtower_index) as u64; + if let Some(step) = settled_by( + deps, + &spend_of(btc, &wci_txid, ack_vout).await?, + &format!("ack connector {watchtower_index}"), + ) { + return step; + } + let (master, _) = local_operator()?; + let preimage = master.preimage_for_graph(row.graph_id, watchtower_index); + let ack_connector = graph + .watchtower_challenge_init + .ack_connector_input(watchtower_index) + .map_err(|e| anyhow!("failed to get ack connector input: {e}"))?; + let ack_input = operator_sign_challenge_ack(graph, watchtower_index, &preimage)?; + let ack_txid = build_sign_and_broadcast_tx( + btc, + master.master_keypair(), + vec![ack_input], + ack_connector.amount, + vec![goat::scripts::p2a_output()], + ) + .await?; + info!(event = "graph_challenge", graph_id = %row.graph_id, watchtower_index, txid = %ack_txid, "Operator-Challenge-ACK broadcast"); + broadcast_pending(deps, &format!("Operator-Challenge-ACK {ack_txid}")) +} + +/// Watchtowers silent past `T4`: their timeouts. +async fn watchtower_timeout(deps: &JobDeps, row: &Graph, mut graph: BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let watchtower_num = validate_watchtower_branches(&graph)?; + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let (master, _) = local_operator()?; + let mut sent = 0; + let mut pending = 0; + for index in 0..watchtower_num { + let watchtower_vout = + output_topology::watchtower_challenge_init::watchtower_connector(index) as u64; + let ack_vout = output_topology::watchtower_challenge_init::ack_connector(index) as u64; + // A challenge, or a timeout of ours, spends the watchtower connector: + // confirmed, the branch is settled; pending, it is looked at again. + let mut settled = false; + for vout in [watchtower_vout, ack_vout] { + match spend_of(btc, &wci_txid, vout).await? { + Spend::Confirmed(_) => settled = true, + Spend::Pending(_) => { + settled = true; + pending += 1; + } + Spend::Unspent => {} + } + } + if settled { + continue; + } + let timeout_tx = + operator_sign_watchtower_challenge_timeout(master.master_keypair(), &mut graph, index)?; + let timeout_txid = timeout_tx.compute_txid(); + let input_amount = graph.watchtower_challenge_timeouts[index] + .prev_outs() + .iter() + .map(|o| o.value) + .sum::(); + broadcast_tx_with_cpfp(btc, timeout_tx, input_amount).await?; + info!(event = "graph_challenge", graph_id = %row.graph_id, watchtower_index = index, txid = %timeout_txid, "Watchtower-Challenge-Timeout broadcast"); + sent += 1; + } + if sent + pending > 0 { + return after_blocks( + deps, + format!("{sent} watchtower timeouts broadcast, {pending} branches pending"), + 1, + ); + } + Step::ignore("every watchtower branch is settled") +} + +/// Every watchtower branch resolved: Operator-Commit-Pubin. +async fn commit_pubin(deps: &JobDeps, graph: &BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let graph_id = graph.parameters.graph_id; + let wci = graph.watchtower_challenge_init.tx().compute_txid(); + let watchtower_num = graph.parameters.watchtower_pubkeys.len(); + // Commit-Pubin and Commit-Timeout both spend Connector-E. + let connector_e = + output_topology::watchtower_challenge_init::connector_e(watchtower_num) as u64; + if let Some(step) = settled_by(deps, &spend_of(btc, &wci, connector_e).await?, "connector-E") { + return step; + } + let wci_txid = SerializableTxid::from(wci); + let timeout_txids: Vec = + graph.watchtower_challenge_timeouts.iter().map(|tx| tx.tx().compute_txid()).collect(); + let WatchtowerChallengeInfo { + included_watchtowers: included_bits, resolved_branch_txids, .. + } = match get_watchtower_challenge_info(btc, &wci_txid, &timeout_txids, watchtower_num).await { + Ok(info) => info, + Err(error) => { + return after_blocks(deps, format!("challenge info is not ready: {error}"), 1); + } + }; + let (best_block_hash, included_bitmap) = match compute_operator_pubin_blockhash_and_bitmap( + btc, + &resolved_branch_txids, + &included_bits, + ) + .await + { + Ok(inputs) => inputs, + Err(error) => { + return after_blocks(deps, format!("operator pubin inputs are not ready: {error}"), 1); + } + }; + let guest_pubin = build_operator_guest_pubin( + &best_block_hash, + &graph.parameters.pubin_disprove_constant, + &included_bitmap, + ); + let (master, _) = local_operator()?; + let wots_secret = master.commit_pubin_wots_keypair_for_graph(graph_id).0; + let signed_input = operator_sign_commit_pubin(graph, &wots_secret, &guest_pubin)?; + let connector_e_amount = graph + .watchtower_challenge_init + .connector_e_input() + .map_err(|e| anyhow!("failed to get connector-e input: {e}"))? + .amount; + let txid = build_sign_and_broadcast_tx( + btc, + master.master_keypair(), + vec![signed_input], + connector_e_amount, + vec![], + ) + .await?; + info!(event = "graph_challenge", graph_id = %graph_id, txid = %txid, "Operator-Commit-Pubin broadcast"); + broadcast_pending(deps, &format!("Operator-Commit-Pubin {txid}")) +} + +/// Watchtower flow over and pubin committed: prove and assert. +async fn prover_assert(deps: &JobDeps, mut graph: BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let instance_id = graph.parameters.instance_parameters.instance_id; + let graph_id = graph.parameters.graph_id; + // The assert's input on chain (an assert of ours included) ends the job. + let assert_input = graph + .operator_assert + .tx() + .input + .first() + .ok_or_else(|| anyhow!("operator assert has no input"))? + .previous_output; + if let Some(step) = settled_by( + deps, + &spend_of(btc, &assert_input.txid, u64::from(assert_input.vout)).await?, + "assert input", + ) { + return step; + } + let (proof, wait_secs) = + get_operator_proof(&deps.local_db, &deps.http_client, &graph, btc, instance_id, graph_id) + .await?; + let Some(proof) = proof else { + // Not ready yet, never "never": the assert must follow once the proof exists. + return Step::retry("operator proof is not ready", (wait_secs as i64).max(1)); + }; + let (master, _) = local_operator()?; + let assert_secret = master.assert_wots_keypair_for_graph(graph_id).0; + let dynamic_input = operator_assert_dynamic_input(&proof.public_inputs)?; + let witness = build_assert_witness(&proof.proof, &assert_secret, dynamic_input)?; + let message = assert_wots_message(&witness)?; + let mut asserted_proof = Vec::new(); + proof.proof.serialize_compressed(&mut asserted_proof)?; + let mut setup_state = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; + let operator_state = setup_state + .operator + .as_mut() + .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; + if let Some(existing) = &operator_state.asserted_operator_proof + && existing != &asserted_proof + { + bail!("operator assertion proof conflicts with persisted proof"); + } + operator_state.asserted_operator_proof = Some(asserted_proof); + save_babe_setup_state(&deps.local_db, instance_id, graph_id, &setup_state)?; + let assert_tx = + operator_sign_assert(&mut graph, &assert_secret, &message, &witness.pi2, &witness.pi3)?; + let input_amount = graph.operator_assert.prev_outs().iter().map(|o| o.value).sum::(); + let assert_txid = assert_tx.compute_txid(); + broadcast_tx_with_cpfp(btc, assert_tx, input_amount).await?; + info!(event = "graph_challenge", graph_id = %graph_id, txid = %assert_txid, "Prover-Assert broadcast"); + broadcast_pending(deps, &format!("Prover-Assert {assert_txid}")) +} + +/// Recovers a `BabeChallengeAssertWitness` from the on-chain challenge assert tx. +fn recover_challenge_assert_witness( + challenge_assert_tx: &bitcoin::Transaction, + verifier_index: usize, +) -> Result { + let txin = challenge_assert_tx + .input + .first() + .ok_or_else(|| anyhow!("challenge assert tx has no input"))?; + let items: Vec> = txin.witness.to_vec(); + let expected_len = 2 * WOTS_SIG_COUNT + goat::assert_scripts::INPUT_WIRE_NUM + 2; + if items.len() != expected_len { + bail!( + "unexpected challenge assert witness length: {} (expected {expected_len})", + items.len() + ); + } + let mut bitcoin_witness = bitcoin::Witness::new(); + for item in &items[..2 * WOTS_SIG_COUNT] { + bitcoin_witness.push(item); + } + let wots_sig = Wots96::raw_witness_to_signature(&bitcoin_witness).to_vec(); + let input_labels: Vec<[u8; 16]> = items + [2 * WOTS_SIG_COUNT..2 * WOTS_SIG_COUNT + goat::assert_scripts::INPUT_WIRE_NUM] + .iter() + .map(|item| item.as_slice().try_into()) + .collect::>() + .map_err(|_| anyhow!("challenge assert label item is not 16 bytes"))?; + Ok(BabeChallengeAssertWitness { + verifier_index, + witness: ChallengeAssertWitnessRaw { input_labels, wots_sig }, + }) +} + +/// A verifier asserted against the operator's proof: Wrongly-Challenged. +async fn wrongly_challenged( + deps: &JobDeps, + graph: &BitvmGcGraph, + challenge_assert_txid: Txid, + verifier_index: usize, +) -> Result { + let btc = &deps.btc_client; + let instance_id = graph.parameters.instance_parameters.instance_id; + let graph_id = graph.parameters.graph_id; + validate_expected_challenge_assert_txid(graph, verifier_index, challenge_assert_txid)?; + // Wrongly-Challenged and Disprove both spend the ChallengeAssert output: + // one on chain (ours included) ends the job. + if let Some(step) = settled_by( + deps, + &spend_of(btc, &challenge_assert_txid, 0).await?, + "verifier assertion output", + ) { + return step; + } + let Some(challenge_assert_tx) = btc.get_tx(&challenge_assert_txid).await? else { + return after_blocks(deps, "ChallengeAssert is not available from the Bitcoin backend", 1); + }; + let challenge_witness = recover_challenge_assert_witness(&challenge_assert_tx, verifier_index)?; + if challenge_witness.verifier_index != verifier_index { + bail!( + "challenge witness verifier index {} does not match {verifier_index}", + challenge_witness.verifier_index + ); + } + let labels: [Vec; goat::assert_scripts::INPUT_WIRE_NUM] = challenge_witness + .witness + .input_labels + .iter() + .map(|label| label.to_vec()) + .collect::>() + .try_into() + .map_err(|labels: Vec>| { + anyhow!( + "BABE challenge witness exposes {} labels; expected {}", + labels.len(), + goat::assert_scripts::INPUT_WIRE_NUM + ) + })?; + let operator_assert_txid = challenge_assert_tx + .input + .first() + .ok_or_else(|| anyhow!("published verifier assert transaction has no input"))? + .previous_output + .txid; + let operator_assert_tx = btc.get_tx(&operator_assert_txid).await?.ok_or_else(|| { + anyhow!("published operator assert transaction {operator_assert_txid} is unavailable") + })?; + let operator_assert_txin = operator_assert_tx + .input + .first() + .cloned() + .ok_or_else(|| anyhow!("published operator assert transaction has no input"))?; + let operator_assert_witness = extract_operator_assert_witness(graph, &operator_assert_txin) + .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; + let expected = + build_verifier_assert_tx(graph, operator_assert_txin.clone(), verifier_index, labels)?; + if expected.input.first().map(|input| &input.witness) + != challenge_assert_tx.input.first().map(|input| &input.witness) + { + bail!("published verifier assert witness does not match BABE challenge labels"); + } + let setup_state = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? + .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; + let operator_state = setup_state + .operator + .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; + let candidate = selected_candidate_for_graph_index(&operator_state, verifier_index)?; + let soldering_proof_ready = candidate.soldering_proof_ready.as_ref().ok_or_else(|| { + anyhow!("missing soldering proof reference for verifier slot {verifier_index}") + })?; + let store_base_path = get_soldering_proof_payload_store_path()?; + let payload_path = soldering_proof_payload_store_path( + &store_base_path, + instance_id, + graph_id, + soldering_proof_ready.candidate_index, + &soldering_proof_ready.payload_hash, + )?; + let payload = read_soldering_proof_store_payload(&payload_path) + .await + .with_context(|| format!("read soldering proof payload from {payload_path}"))?; + let payload = decode_soldering_proof_payload(soldering_proof_ready, &payload)?; + let (_, finalized, soldering) = expand_compact_soldering_proof_payload(payload) + .context("expand compact soldering proof payload for challenge")?; + let prover_state = build_babe_prover_state(&candidate.setup_package, finalized, soldering)?; + let assert_witness = TxAssertWitness { + wots_sig: challenge_witness.witness.wots_sig.clone(), + pi2: operator_assert_witness.pi2, + pi3: operator_assert_witness.pi3, + }; + let proof = recover_operator_proof_from_assert_witness(&assert_witness) + .context("recover asserted operator proof from on-chain assert witness")?; + let vk = crate::vk::get_vk() + .await + .context("load Groth16 verifying key for BABE wrongly challenged")?; + let (_, dyn_pubin) = assert_witness.try_recover_pi1_xd().ok_or_else(|| { + anyhow!("cannot recover dynamic input from challenge witness WOTS signature") + })?; + #[cfg(feature = "mock-gc")] + crate::mock_gc::simulate(crate::mock_gc::GcOp::Eval).await; + let wrongly_challenged_witness = recover_real_wrongly_challenged_witness( + &prover_state, + &challenge_witness, + &proof, + vk, + dyn_pubin, + )?; + let (input, input_amount) = operator_sign_wrongly_challenged( + graph, + verifier_index, + &wrongly_challenged_witness.final_msg, + )?; + let (master, _) = local_operator()?; + let txid = build_sign_and_broadcast_tx( + btc, + master.master_keypair(), + vec![input], + input_amount, + vec![ + goat::scripts::p2a_output(), + // This makes the non-witness transaction size safely exceed the + // relay minimum even before the locally funded fee input is added. + bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: goat::scripts::generate_opreturn_script( + WRONGLY_CHALLENGED_OP_RETURN_DATA.to_vec(), + ), + }, + ], + ) + .await?; + info!(event = "graph_challenge", graph_id = %graph_id, verifier_index, txid = %txid, "Wrongly-Challenged broadcast"); + broadcast_pending(deps, &format!("Wrongly-Challenged {txid}")) +} + +const WRONGLY_CHALLENGED_OP_RETURN_DATA: &[u8] = b"wrongly-challenged"; + +/// No challenge for `T1` after the kickoff: Take-1. +async fn take1(deps: &JobDeps, row: &Graph, mut graph: BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let kickoff_txid = graph.kickoff.tx().compute_txid(); + let connector_a = output_topology::kickoff::connector_a() as u64; + let guardian = output_topology::kickoff::guardian_connector() as u64; + for (vout, what) in [(connector_a, "connector-A"), (guardian, "guardian connector")] { + if let Some(step) = settled_by(deps, &spend_of(btc, &kickoff_txid, vout).await?, what) { + return step; + } + } + let Some(kickoff_height) = btc.get_tx_status(&kickoff_txid).await?.block_height else { + return after_blocks(deps, "kickoff is not confirmed yet", 1); + }; + if !is_take1_timelock_expired(btc, kickoff_height, &graph.parameters.timelock_config).await? { + return after_blocks(deps, "take-1 timelock has not expired", 1); + } + let (master, _) = local_operator()?; + let take1_tx = operator_sign_take1(master.master_keypair(), &mut graph)?; + let take1_txid = take1_tx.compute_txid(); + let anchor_vout = take1_tx.output.len() as u64 - 1; + let input_amount = graph.take1.prev_outs().iter().map(|o| o.value).sum(); + match build_cpfp_txns(btc, &take1_tx, anchor_vout, input_amount).await? { + Some(child) => broadcast_package(btc, &[take1_tx, child], true).await?, + None => broadcast_tx(btc, &take1_tx).await?, + }; + info!(event = "graph_take", graph_id = %row.graph_id, txid = %take1_txid, "Take-1 broadcast"); + broadcast_pending(deps, &format!("Take-1 {take1_txid}")) +} + +/// Challenge answered and `T3` / `T7` elapsed: Take-2. +async fn take2(deps: &JobDeps, row: &Graph, mut graph: BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + // Take-2 spends Connector-0, Connector-D, Connector-F and the Guardian + // Connector: recheck every input before signing. + for input in graph.take2.tx().input.iter().map(|txin| txin.previous_output) { + if let Some(step) = settled_by( + deps, + &spend_of(btc, &input.txid, u64::from(input.vout)).await?, + &format!("take-2 input {input}"), + ) { + return step; + } + } + let assert_txid = graph.operator_assert.tx().compute_txid(); + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let Some(assert_height) = btc.get_tx_status(&assert_txid).await?.block_height else { + return after_blocks(deps, "prover assert is not confirmed yet", 1); + }; + let Some(wci_height) = btc.get_tx_status(&wci_txid).await?.block_height else { + return after_blocks(deps, "watchtower challenge init is not confirmed yet", 1); + }; + if !is_take2_timelock_expired(btc, assert_height, wci_height, &graph.parameters.timelock_config) + .await? + { + return after_blocks(deps, "take-2 timelock has not expired", 1); + } + let (master, _) = local_operator()?; + let take2_tx = operator_sign_take2(master.master_keypair(), &mut graph)?; + let take2_txid = take2_tx.compute_txid(); + let anchor_vout = take2_tx.output.len() as u64 - 1; + let input_amount = graph.take2.prev_outs().iter().map(|o| o.value).sum(); + match build_cpfp_txns(btc, &take2_tx, anchor_vout, input_amount).await? { + Some(child) => broadcast_package(btc, &[take2_tx, child], true).await?, + None => broadcast_tx(btc, &take2_tx).await?, + }; + info!(event = "graph_take", graph_id = %row.graph_id, txid = %take2_txid, "Take-2 broadcast"); + broadcast_pending(deps, &format!("Take-2 {take2_txid}")) +} + +// --------------------------------------------------------------- watchtower + +fn watchtower_index(graph: &BitvmGcGraph) -> Result> { + let keypair = WatchtowerMasterKey::new(get_bitvm_key()?).master_keypair(); + let local = keypair.public_key().x_only_public_key().0; + Ok(graph.parameters.watchtower_pubkeys.iter().position(|pubkey| *pubkey == local)) +} + +/// Watchtower-Challenge-Init confirmed: challenge with the watchtower proof. +async fn watchtower_challenge(deps: &JobDeps, row: &Graph, graph: &BitvmGcGraph) -> Result { + let btc = &deps.btc_client; + let Some(index) = watchtower_index(graph)? else { + return Step::ignore("not a watchtower of this graph"); + }; + let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); + if !tx_visible(btc, &wci_txid).await? { + return after_blocks(deps, "watchtower challenge init is not on chain", 1); + } + let vout = output_topology::watchtower_challenge_init::watchtower_connector(index) as u64; + if let Some(step) = + settled_by(deps, &spend_of(btc, &wci_txid, vout).await?, "watchtower challenge connector") + { + return step; + } + let instance_id = graph.parameters.instance_parameters.instance_id; + let proof = match get_watchtower_commitment( + &deps.local_db, + btc, + &deps.http_client, + instance_id, + row.graph_id, + ) + .await? + { + (Some(proof), _) => proof, + (None, wait_secs) => { + return Step::retry( + "watchtower proof commitment is not ready", + (wait_secs as i64).max(1), + ); + } + }; + match send_watchtower_challenge_tx(btc, graph, index, proof).await { + Ok(txid) => { + info!(event = "graph_challenge", graph_id = %row.graph_id, watchtower_index = index, txid = %txid, "Watchtower-Challenge broadcast"); + broadcast_pending(deps, &format!("Watchtower-Challenge {txid}")) + } + Err(error) => Step::retry(format!("watchtower challenge broadcast failed: {error:#}"), 30), + } +} + +/// The operator asserted: stop the watchtower proof this node still waits on. +async fn watchtower_proof_timeout(row: &Graph, graph: &BitvmGcGraph) -> Result { + if watchtower_index(graph)?.is_none() { + return Step::ignore("not a watchtower of this graph"); + } + let cancelled = notify_watchtower_proof_timeout(row.instance_id, row.graph_id).await?; + Step::done(GraphPlan::noted(format!("watchtower proof timeout sent: {cancelled}"))) +} + +// ------------------------------------------------------------------ relayer + +/// Kickoff confirmed and covered by the SPV client: `proceedWithdraw`. +/// Blocks a relayer job waits for a lagging Goat node to show the withdraw +/// before it gives up on it. +const WITHDRAW_LAG_RETRIES: i64 = 30; +/// Runs (six blocks apart) a finish job waits for `proceedWithdraw` to land. +const WITHDRAW_PROCEED_RETRIES: i64 = 60; + +/// Goat does not show the withdraw: a lagging node (the `InitWithdraw` came +/// from a finalized block) is waited out; a kickoff without a withdraw is +/// given up on with an alert. These jobs run once per graph, so an early +/// `ignore` would leave the Goat side behind for good. +fn withdraw_missing(deps: &JobDeps, graph_id: Uuid, call: &str, attempts: i64) -> Result { + if attempts < WITHDRAW_LAG_RETRIES { + return after_blocks(deps, format!("withdraw is not visible on Goat yet; {call} waits"), 1); + } + warn!(event = "graph_withdraw", graph_id = %graph_id, call, attempts, "withdraw never appeared on Goat; giving up"); + alert("pegout_withdraw_missing"); + Step::ignore(format!("withdraw not on Goat after {attempts} attempts; {call} skipped")) +} + +/// `proceedWithdraw` has not landed: wait for it, but not forever. +fn withdraw_not_proceeded( + deps: &JobDeps, + graph_id: Uuid, + call: &str, + attempts: i64, +) -> Result { + if attempts < WITHDRAW_PROCEED_RETRIES { + return after_blocks(deps, "withdraw is initialized but the kickoff is not posted", 6); + } + warn!(event = "graph_withdraw", graph_id = %graph_id, call, attempts, "proceedWithdraw never landed; giving up"); + alert("pegout_withdraw_stuck"); + Step::ignore(format!("withdraw still Initialized after {attempts} attempts; {call} skipped")) +} + +/// Goat shows a status the chain contradicts: report it, nothing to send. +fn withdraw_inconsistent( + graph_id: Uuid, + call: &str, + status: WithdrawStatus, + settled_by: &str, +) -> Result { + warn!(event = "graph_withdraw", graph_id = %graph_id, call, status = ?status, "withdraw status on Goat contradicts the {settled_by} on Bitcoin"); + alert("pegout_withdraw_inconsistent"); + Step::ignore(format!("withdraw is {status:?} although {settled_by} confirmed; {call} skipped")) +} + +async fn proceed_withdraw( + deps: &JobDeps, + row: &Graph, + graph: &BitvmGcGraph, + attempts: i64, +) -> Result { + if !deps.relayer { + return Step::ignore("not a relayer"); + } + let btc = &deps.btc_client; + let graph_id = graph.parameters.graph_id; + let withdraw_status = deps.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; + match withdraw_status { + WithdrawStatus::Initialized => {} + WithdrawStatus::None if closed(row) => { + return Step::ignore("graph closed without a withdraw on Goat"); + } + WithdrawStatus::None => { + return withdraw_missing(deps, graph_id, "proceedWithdraw", attempts); + } + WithdrawStatus::Canceled => { + // Cancelling needs the committee; a cancelled withdraw is handled by hand. + info!(event = "graph_withdraw", graph_id = %graph_id, "withdraw cancelled on Goat; nothing to proceed"); + return Step::ignore("withdraw is Canceled"); + } + other => return Step::ignore(format!("withdraw is already {other:?}")), + } + let kickoff_txid = graph.kickoff.tx().compute_txid(); + let Some(kickoff_tx) = btc.get_tx(&kickoff_txid).await? else { + return after_blocks(deps, "kickoff tx is not available", 1); + }; + let Some(kickoff_height) = btc.get_tx_status(&kickoff_txid).await?.block_height else { + return after_blocks(deps, "kickoff is not confirmed yet", 1); + }; + let kickoff_height = u64::from(kickoff_height); + let spv_height = deps.goat_client.btc_spv_latest_height().await?; + if spv_height < kickoff_height { + return after_blocks( + deps, + "kickoff block is not available through GOAT SPV", + kickoff_height - spv_height, + ); + } + deps.goat_client.gateway_process_withdraw(btc, &graph_id, &kickoff_tx).await?; + info!(event = "graph_withdraw", graph_id = %graph_id, "proceedWithdraw sent"); + Step::done(GraphPlan::noted("proceedWithdraw sent")) +} + +/// The SPV height covering `txid`, or a retry while it does not. +async fn spv_covered(deps: &JobDeps, txid: &Txid, what: &str) -> Result> { + let Some(height) = deps.btc_client.get_tx_status(txid).await?.block_height else { + return after_blocks(deps, format!("{what} is not confirmed yet"), 1).map(Some); + }; + let height = u64::from(height); + let spv_height = deps.goat_client.btc_spv_latest_height().await?; + if spv_height < height { + return after_blocks( + deps, + format!("{what} block is not available through GOAT SPV"), + height - spv_height, + ) + .map(Some); + } + Ok(None) +} + +/// Take-1 confirmed: `finishWithdrawHappyPath`. +async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Result { + if !deps.relayer { + return Step::ignore("not a relayer"); + } + let graph_id = graph.parameters.graph_id; + let take1_txid = graph.take1.tx().compute_txid(); + let Some(take1_tx) = deps.btc_client.get_tx(&take1_txid).await? else { + return after_blocks(deps, "take-1 tx is not available", 1); + }; + let withdraw_status = deps.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; + match withdraw_status { + WithdrawStatus::Processing => {} + WithdrawStatus::Initialized => { + return withdraw_not_proceeded(deps, graph_id, "finishWithdrawHappyPath", attempts); + } + WithdrawStatus::None => { + return withdraw_missing(deps, graph_id, "finishWithdrawHappyPath", attempts); + } + WithdrawStatus::Complete => return Step::ignore("withdraw already complete on Goat"), + other => { + return withdraw_inconsistent(graph_id, "finishWithdrawHappyPath", other, "Take-1"); + } + } + if let Some(retry) = spv_covered(deps, &take1_txid, "take-1").await? { + return Ok(retry); + } + deps.goat_client + .gateway_finish_withdraw_happy_path(&deps.btc_client, &graph_id, &take1_tx) + .await?; + info!(event = "graph_withdraw", graph_id = %graph_id, "finishWithdrawHappyPath sent"); + Step::done(GraphPlan::noted("finishWithdrawHappyPath sent")) +} + +/// Take-2 confirmed: `finishWithdrawUnhappyPath`. +async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Result { + if !deps.relayer { + return Step::ignore("not a relayer"); + } + let graph_id = graph.parameters.graph_id; + let take2_txid = graph.take2.tx().compute_txid(); + let Some(take2_tx) = deps.btc_client.get_tx(&take2_txid).await? else { + return after_blocks(deps, "take-2 tx is not available", 1); + }; + let withdraw_status = deps.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; + match withdraw_status { + WithdrawStatus::Processing => {} + WithdrawStatus::Initialized => { + return withdraw_not_proceeded(deps, graph_id, "finishWithdrawUnhappyPath", attempts); + } + WithdrawStatus::None => { + return withdraw_missing(deps, graph_id, "finishWithdrawUnhappyPath", attempts); + } + WithdrawStatus::Complete => return Step::ignore("withdraw already complete on Goat"), + other => { + return withdraw_inconsistent(graph_id, "finishWithdrawUnhappyPath", other, "Take-2"); + } + } + if let Some(retry) = spv_covered(deps, &take2_txid, "take-2").await? { + return Ok(retry); + } + deps.goat_client + .gateway_finish_withdraw_unhappy_path(&deps.btc_client, &graph_id, &take2_tx) + .await?; + info!(event = "graph_withdraw", graph_id = %graph_id, "finishWithdrawUnhappyPath sent"); + Step::done(GraphPlan::noted("finishWithdrawUnhappyPath sent")) +} + +/// A challenge succeeded on Bitcoin: `finishWithdrawDisproved`. +async fn finish_disproved( + deps: &JobDeps, + graph: &BitvmGcGraph, + disprove_type: DisproveTxType, + index: usize, + challenge_finish_txid: Txid, + attempts: i64, +) -> Result { + if !deps.relayer { + return Step::ignore("not a relayer"); + } + let btc = &deps.btc_client; + let graph_id = graph.parameters.graph_id; + let withdraw_status = deps.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; + match withdraw_status { + WithdrawStatus::Disproved => return Step::ignore("already disproved on Goat"), + WithdrawStatus::None => { + return withdraw_missing(deps, graph_id, "finishWithdrawDisproved", attempts); + } + _ => {} + } + let kickoff_txid = graph.kickoff.tx().compute_txid(); + let take1_txid = graph.take1.tx().compute_txid(); + let connector_a = output_topology::kickoff::connector_a() as u64; + let challenge_start_tx = match outpoint_spent_txid(btc, &kickoff_txid, connector_a).await? { + Some(spent_txid) if spent_txid == take1_txid => { + return Step::ignore(format!("graph already settled by Take-1 {spent_txid}")); + } + Some(spent_txid) => btc.get_tx(&spent_txid).await?, + None => None, + }; + let Some(challenge_finish_tx) = btc.get_tx(&challenge_finish_txid).await? else { + return after_blocks( + deps, + format!("challenge finish tx {challenge_finish_txid} is not available"), + 1, + ); + }; + if let Some(retry) = spv_covered(deps, &challenge_finish_txid, "challenge finish").await? { + return Ok(retry); + } + let result = deps + .goat_client + .gateway_finish_withdraw_disproved( + btc, + &graph_id, + disprove_type, + index as u64, + challenge_start_tx.as_ref(), + &challenge_finish_tx, + ) + .await; + deps.metrics_state.record_withdraw_finalize(result.is_ok()); + result?; + info!(event = "graph_withdraw", graph_id = %graph_id, disprove_type = %disprove_type, index, "finishWithdrawDisproved sent"); + Step::done(GraphPlan::noted("finishWithdrawDisproved sent")) +} + +// -------------------------------------------------------------- validation + +fn validate_watchtower_branches(graph: &BitvmGcGraph) -> Result { + let watchtower_num = graph.parameters.watchtower_pubkeys.len(); + if watchtower_num == 0 { + bail!("graph has no watchtower slots"); + } + if graph.parameters.watchtower_ack_hashlocks.len() != watchtower_num + || graph.watchtower_challenge_timeouts.len() != watchtower_num + || graph.operator_challenge_nacks.len() != watchtower_num + { + bail!( + "graph watchtower branch lengths differ: watchtowers={}, ack_hashlocks={}, timeouts={}, nacks={}", + watchtower_num, + graph.parameters.watchtower_ack_hashlocks.len(), + graph.watchtower_challenge_timeouts.len(), + graph.operator_challenge_nacks.len() + ); + } + Ok(watchtower_num) +} + +fn validate_expected_challenge_assert_txid( + graph: &BitvmGcGraph, + verifier_index: usize, + challenge_assert_txid: Txid, +) -> Result<()> { + validate_verifier_slot(graph, verifier_index)?; + let expected_txid = graph.verifier_asserts[verifier_index].tx().compute_txid(); + if challenge_assert_txid != expected_txid { + bail!( + "challenge assert txid {challenge_assert_txid} does not match graph verifier assert txid {expected_txid}" + ); + } + Ok(()) +} + +/// The on-chain ACK inputs of every watchtower that ACKed. +async fn collect_ack_txins( + btc_client: &client::btc_chain::BTCClient, + wci_txid: &Txid, + watchtower_timeout_txids: &[Txid], +) -> Result> { + let mut ack_txins = Vec::new(); + for (i, timeout_txid) in watchtower_timeout_txids.iter().enumerate() { + let ack_vout = output_topology::watchtower_challenge_init::ack_connector(i) as u64; + let Some(ack_spent_txid) = outpoint_spent_txid(btc_client, wci_txid, ack_vout).await? + else { + continue; + }; + if &ack_spent_txid == timeout_txid { + continue; + } + let Some((_, _, txin)) = outpoint_spent_txin(btc_client, wci_txid, ack_vout).await? else { + bail!("ACK spending tx {ack_spent_txid} for watchtower {i} is unavailable"); + }; + ack_txins.push(txin); + } + Ok(ack_txins) +} + +#[cfg(test)] +mod tests { + use super::*; + use bitcoin::hashes::Hash; + + // ── collect_ack_txins ───────────────────────────────────────────────────── + + fn make_txid(byte: u8) -> Txid { + Txid::from_byte_array([byte; 32]) + } + + fn make_confirmed_ack_tx(ack_txid: Txid, wci_txid: Txid, ack_vout: u32) -> esplora_client::Tx { + use esplora_client::{TxStatus, Vin}; + let preimage = b"test-ack-preimage".to_vec(); + esplora_client::Tx { + txid: ack_txid, + version: 2, + locktime: 0, + vin: vec![Vin { + txid: wci_txid, + vout: ack_vout, + prevout: None, + scriptsig: bitcoin::ScriptBuf::default(), + witness: vec![preimage], + sequence: 1, + is_coinbase: false, + }], + vout: vec![], + size: 1, + weight: 1, + status: TxStatus { + confirmed: true, + block_height: Some(800_000), + block_hash: None, + block_time: Some(1_000_000), + }, + fee: 0, + } + } + + #[tokio::test] + async fn collect_ack_txins_returns_only_spent_connectors() { + let (btc_client, mock) = client::btc_chain::BTCClient::new_mock_client(); + let wci_txid = make_txid(0x00); + let ack_txid_0 = make_txid(0x01); + let timeout_txid_1 = make_txid(0x03); + let timeout_txids = vec![make_txid(0x10), timeout_txid_1, make_txid(0x12)]; + + // Watchtower 0 ACKed + let vout_0 = output_topology::watchtower_challenge_init::ack_connector(0) as u32; + mock.set_tx(ack_txid_0, make_confirmed_ack_tx(ack_txid_0, wci_txid, vout_0)); + + // Watchtower 1 timed out; the timeout spend must not count as an ACK. + let vout_1 = output_topology::watchtower_challenge_init::ack_connector(1) as u32; + mock.set_tx(timeout_txid_1, make_confirmed_ack_tx(timeout_txid_1, wci_txid, vout_1)); + + // Watchtower 2 ACKed + let ack_txid_2 = make_txid(0x02); + let vout_2 = output_topology::watchtower_challenge_init::ack_connector(2) as u32; + mock.set_tx(ack_txid_2, make_confirmed_ack_tx(ack_txid_2, wci_txid, vout_2)); + + let txins = collect_ack_txins(&btc_client, &wci_txid, &timeout_txids).await.unwrap(); + + assert_eq!(txins.len(), 2); + assert_eq!(txins[0].previous_output.vout, vout_0); + assert_eq!(txins[1].previous_output.vout, vout_2); + } + + // ── recover_challenge_assert_witness ────────────────────────────────────── + + fn build_challenge_assert_tx( + sig: &::Signature, + labels: &[[u8; 16]], + ) -> bitcoin::Transaction { + let mut witness = bitcoin::Witness::new(); + let raw_wots = Wots96::signature_to_raw_witness(sig); + for item in raw_wots.iter() { + witness.push(item); + } + for label in labels { + witness.push(label.as_slice()); + } + witness.push([0xde, 0xad]); // placeholder script + witness.push([0xbe, 0xef]); // placeholder control block + bitcoin::Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![bitcoin::TxIn { witness, ..Default::default() }], + output: vec![], + } + } + + #[test] + fn recover_challenge_assert_witness_extracts_wots_sig_and_labels() { + use bitvm_lib::operator::generate_assert_wots_key; + + let (sk, _) = generate_assert_wots_key("test-challenge-witness"); + let msg = [0x42u8; 96]; + let sig = Wots96::sign(&sk, &msg); + + let labels: Vec<[u8; 16]> = + (0..goat::assert_scripts::INPUT_WIRE_NUM).map(|i| [(i % 256) as u8; 16]).collect(); + + let tx = build_challenge_assert_tx(&sig, &labels); + let verifier_index = 3; + let recovered = recover_challenge_assert_witness(&tx, verifier_index).unwrap(); + + assert_eq!(recovered.verifier_index, verifier_index); + assert_eq!(recovered.witness.wots_sig.len(), WOTS_SIG_COUNT); + assert_eq!(recovered.witness.input_labels.len(), goat::assert_scripts::INPUT_WIRE_NUM); + + let mut bw = bitcoin::Witness::new(); + for item in Wots96::signature_to_raw_witness(&sig).iter() { + bw.push(item); + } + assert_eq!(recovered.witness.wots_sig, Wots96::raw_witness_to_signature(&bw).to_vec()); + + // labels are correctly extracted + for (i, label) in recovered.witness.input_labels.iter().enumerate() { + assert_eq!(label[0], (i % 256) as u8, "label {i} mismatch"); + } + } + + #[test] + fn recover_challenge_assert_witness_rejects_wrong_length() { + use bitvm_lib::operator::generate_assert_wots_key; + + let (sk, _) = generate_assert_wots_key("test-wrong-len"); + let sig = Wots96::sign(&sk, &[0u8; 96]); + let labels: Vec<[u8; 16]> = vec![[0u8; 16]; goat::assert_scripts::INPUT_WIRE_NUM]; + + // one label short + let mut short_labels = labels.clone(); + short_labels.pop(); + let bad_tx = build_challenge_assert_tx(&sig, &short_labels); + assert!(recover_challenge_assert_witness(&bad_tx, 0).is_err()); + + // no inputs at all + let empty_tx = bitcoin::Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![], + output: vec![], + }; + assert!(recover_challenge_assert_witness(&empty_tx, 0).is_err()); + } + + #[test] + fn wrongly_challenge_outputs_exceed_minimum_non_witness_size() { + // Bitcoin Core rejects standard transactions smaller than 65 non-witness bytes. + let tx = bitcoin::Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![bitcoin::TxIn::default()], + output: vec![ + goat::scripts::p2a_output(), + bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: goat::scripts::generate_opreturn_script( + WRONGLY_CHALLENGED_OP_RETURN_DATA.to_vec(), + ), + }, + ], + }; + + assert!( + tx.base_size() >= 65, + "wrongly-challenge transaction is {} non-witness bytes", + tx.base_size() + ); + } + + #[test] + fn pubin_disprove_outputs_exceed_minimum_non_witness_size() { + // Bitcoin Core rejects standard transactions smaller than 65 non-witness bytes. + let tx = bitcoin::Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![bitcoin::TxIn::default()], + output: vec![ + goat::scripts::p2a_output(), + bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: goat::scripts::generate_opreturn_script( + PUBIN_DISPROVE_OP_RETURN_DATA.to_vec(), + ), + }, + ], + }; + + assert!( + tx.base_size() >= 65, + "pubin-disprove transaction is {} non-witness bytes", + tx.base_size() + ); + } +} diff --git a/node/src/aggregates/graph_scan.rs b/node/src/aggregates/graph_scan.rs new file mode 100644 index 000000000..d7fd7b574 --- /dev/null +++ b/node/src/aggregates/graph_scan.rs @@ -0,0 +1,907 @@ +//! Collect graph chain facts outside the apply transaction. +//! +//! Maintenance scans active graphs once per Bitcoin height and stored status; +//! Obsoleted graphs only get a cheap probe until their Pre-Kickoff confirms. +//! Each result becomes a `chain_scan` event. Operator nodes also report setups +//! unfinished at their instance's abandon height. + +use super::common::ScanMarks; +use super::graph::{SetupAbandoned, setup_abandoned_event}; +use super::graph_pegout::{ChainScan, PegoutSignal, Unusable}; +use super::instance::PeginDeadlines; +use crate::clock::current_time_secs; +use crate::env::{get_network, get_node_pubkey}; +use crate::utils::{ + GraphChainScan, load_validated_graph_definition, outpoint_spent_confirmed, outpoint_spent_txid, + scan_graph_chain_state, +}; +use bitcoin::{OutPoint, Txid}; +use bitvm_lib::timelocks::{ + connector_f_timelock_blocks, disprove_timelock_blocks, operator_ack_timelock_blocks, + operator_commit_timelock_blocks, take1_timelock_blocks, take2_timelock_blocks, + validate_timelock_config, watchtower_challenge_timelock_blocks, +}; +use bitvm_lib::types::BitvmGcGraph; +use client::btc_chain::BTCClient; +use client::goat_chain::{DisproveTxType, GOATClient}; +use engine::EngineHandle; +use goat::constants::TimelockConfig; +use goat::transactions::base::output_topology; +use goat::transactions::pre_signed::PreSignedTransaction; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; +use std::str::FromStr; +use std::sync::{LazyLock, Mutex}; +use store::localdb::{GraphQuery, LocalDB}; +use store::{Graph, GraphBtcTxVoutMonitor, GraphStatus, InstanceBridgeInStatus, SerializableTxid}; +use strum::{Display, EnumString}; +use tracing::{debug, trace, warn}; +use uuid::Uuid; + +/// Graphs scanned per tick; the rest follow on the next ticks, which resume +/// after the slot the previous tick stopped at and wrap around, so no graph +/// starves at a height when there are more graphs than one tick covers. +const SCAN_BATCH: usize = 64; +/// Obsoleted graphs probed per tick, in their own rotation so they never hold +/// the active graphs back. +const OBSOLETED_SCAN_BATCH: usize = 16; +/// Every this many heights an Obsoleted graph gets a full scan anyway, so a +/// dead Pre-Kickoff input is still noticed and recorded for the later slots. +const OBSOLETED_FULL_SCAN_EVERY: i64 = 16; + +const MONITOR_KICKOFF: &str = "kickoff"; +const MONITOR_WATCHTOWER_INIT: &str = "watchtower_init"; +const MONITOR_PROVER_ASSERT: &str = "prover_assert"; +const MONITOR_VERIFIER_ASSERT: &str = "verifier_assert"; + +#[derive( + Copy, Clone, Debug, Serialize, Deserialize, Default, Eq, PartialEq, Display, EnumString, +)] +pub enum VerifierChallengeStatus { + #[default] + None, + VerifierAsserted, + ProverAnswered, + Disproved, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Default)] +pub struct ChallengeSubStatus { + pub watchtower_challenge_status: Vec, // true for challenge connector spend + pub verifier_challenge_status: Vec, + pub disprove_type: Option, + pub disprove_index: i32, +} + +// ------------------------------------------------------------------- marks + +static SCAN_MARKS: LazyLock> = LazyLock::new(ScanMarks::new); + +/// The slot `(operator_pubkey, kickoff_index)` the last tick stopped at, for +/// the active graphs and for the Obsoleted ones. +static SCAN_CURSOR: LazyLock>> = LazyLock::new(|| Mutex::new(None)); +static OBSOLETED_SCAN_CURSOR: LazyLock>> = + LazyLock::new(|| Mutex::new(None)); + +/// Where a scan over `slots` (sorted by operator and nonce) resumes: the +/// first slot after `cursor`, or the start when the cursor is unset or past +/// the last slot. +fn resume_index(slots: &[(String, i64)], cursor: Option<&(String, i64)>) -> usize { + let Some(cursor) = cursor else { return 0 }; + slots.iter().position(|slot| slot > cursor).unwrap_or(0) +} + +/// Lowest slot per operator whose Pre-Kickoff can never confirm. The later +/// slots extend it, so their own scans report it too and keep them Obsoleted +/// instead of following their own unspent outpoints. Graphs are scanned +/// lowest slot first, which rebuilds the map after a restart. +static DEAD_SLOTS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +fn dead_slot_below(operator_pubkey: &str, slot: u64) -> Option { + let dead = DEAD_SLOTS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + dead.get(operator_pubkey).copied().filter(|dead| *dead < slot) +} + +fn record_dead_slot(operator_pubkey: &str, slot: u64) { + let mut dead = DEAD_SLOTS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let lowest = dead.entry(operator_pubkey.to_string()).or_insert(slot); + *lowest = (*lowest).min(slot); +} + +/// Statuses with something left to happen on chain: everything but closed. +fn monitored_statuses() -> Vec { + use GraphStatus::*; + [ + OperatorPresigned, + CommitteePresigned, + OperatorDataPushed, + PreKickoff, + OperatorKickOff, + Challenge, + Obsoleted, + ] + .iter() + .map(ToString::to_string) + .collect() +} + +/// Record a chain scan for every monitored graph not yet scanned at this +/// height, the operator's lower slots first. +pub async fn scan_graphs_on_chain( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + engine: &EngineHandle, +) -> anyhow::Result<()> { + let graphs = local_db + .acquire() + .await? + .get_operator_graphs( + GraphQuery::default() + .with_statuses(monitored_statuses()) + .with_order("operator_pubkey, kickoff_index".to_string()), + ) + .await?; + if graphs.is_empty() { + return Ok(()); + } + let height = i64::from(btc_client.get_height().await?); + let (obsoleted, active): (Vec, Vec) = + graphs.into_iter().partition(|row| row.status == GraphStatus::Obsoleted.to_string()); + let scan = ScanPass { local_db, btc_client, goat_client, engine, height }; + scan.rows(&active, SCAN_BATCH, &SCAN_CURSOR, false).await?; + scan.rows(&obsoleted, OBSOLETED_SCAN_BATCH, &OBSOLETED_SCAN_CURSOR, true).await +} + +struct ScanPass<'a> { + local_db: &'a LocalDB, + btc_client: &'a BTCClient, + goat_client: &'a GOATClient, + engine: &'a EngineHandle, + height: i64, +} + +impl ScanPass<'_> { + /// Scan up to `batch` of `graphs`, resuming after the slot `cursor` points + /// at. The cursor moves per graph, so a tick cut short by the maintenance + /// timeout resumes where it stopped instead of restarting at the same slot. + async fn rows( + &self, + graphs: &[Graph], + batch: usize, + cursor: &Mutex>, + obsoleted: bool, + ) -> anyhow::Result<()> { + if graphs.is_empty() { + return Ok(()); + } + let slots: Vec<(String, i64)> = + graphs.iter().map(|row| (row.operator_pubkey.clone(), row.kickoff_index)).collect(); + let start = { + let cursor = cursor.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + resume_index(&slots, cursor.as_ref()) + }; + let mut scanned = 0; + for offset in 0..graphs.len() { + if scanned >= batch { + break; + } + let index = (start + offset) % graphs.len(); + let row = &graphs[index]; + *cursor.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = + Some(slots[index].clone()); + let Ok(status) = GraphStatus::from_str(&row.status) else { + continue; + }; + if SCAN_MARKS.seen(row.graph_id, self.height, status) { + continue; + } + scanned += 1; + if obsoleted && self.height % OBSOLETED_FULL_SCAN_EVERY != 0 { + match prekickoff_confirmed(self.btc_client, row).await { + Ok(true) => {} + Ok(false) => { + SCAN_MARKS.mark(row.graph_id, self.height, status); + continue; + } + Err(error) => { + warn!(event = "graph_scan", graph_id = %row.graph_id, status = %status, height = self.height, error = %error, "pre-kickoff probe failed; retried on the next tick"); + continue; + } + } + } + match gather_chain_scan( + self.local_db, + self.btc_client, + self.goat_client, + row, + status, + self.height, + ) + .await + { + Ok(Some(scan)) => { + self.engine.submit(scan.event(row.graph_id)?).await?; + SCAN_MARKS.mark(row.graph_id, self.height, status); + } + Ok(None) => SCAN_MARKS.mark(row.graph_id, self.height, status), + Err(error) => warn!( + event = "graph_scan", + graph_id = %row.graph_id, + status = %status, + height = self.height, + error = %error, + "chain scan failed; retried on the next tick" + ), + } + } + Ok(()) + } +} + +/// Nothing happens on chain for an Obsoleted graph until its Pre-Kickoff +/// confirms (a Skip-Kickoff needs it, and a confirmed Pre-Kickoff makes the +/// graph live again), so that is all the regular probe asks. +async fn prekickoff_confirmed(btc_client: &BTCClient, row: &Graph) -> anyhow::Result { + let Some(txid) = row.cur_prekickoff_txid.clone() else { + return Ok(false); + }; + Ok(btc_client.get_tx_status(&Txid::from(txid)).await?.confirmed) +} + +/// Report operator setups unfinished at the abandon height, including pending +/// sessions and built graphs without committee pre-signatures. Failed instances +/// are handled by their own cleanup. +pub async fn abandon_unfinished_setups( + local_db: &LocalDB, + btc_client: &BTCClient, + engine: &EngineHandle, +) -> anyhow::Result<()> { + let operator_pubkey = get_node_pubkey()?.to_string(); + let setups: BTreeMap = { + let mut storage = local_db.acquire().await?; + let sessions = storage.list_pending_graph_inits_by_operator(&operator_pubkey).await?; + let graphs = storage + .get_operator_graphs( + GraphQuery::default() + .with_operator_pubkey(operator_pubkey) + .with_statuses(vec![GraphStatus::OperatorPresigned.to_string()]), + ) + .await?; + sessions + .into_iter() + .map(|session| (session.graph_id, session.instance_id)) + .chain(graphs.into_iter().map(|graph| (graph.graph_id, graph.instance_id))) + .collect() + }; + if setups.is_empty() { + return Ok(()); + } + let height = i64::from(btc_client.get_height().await?); + let deadlines = PeginDeadlines::from_env(); + for (graph_id, instance_id) in setups { + let Some(instance) = local_db.acquire().await?.find_instance(&instance_id).await? else { + continue; + }; + if InstanceBridgeInStatus::from_str(&instance.status).is_ok_and(|status| status.is_failed()) + { + continue; + } + if let Some(abandon_height) = deadlines.abandon_height(instance.btc_height) + && height >= abandon_height + { + let abandoned = SetupAbandoned { instance_id, abandon_height, height }; + engine.submit(setup_abandoned_event(graph_id, &abandoned)?).await?; + } + } + Ok(()) +} + +/// The facts of one graph at `height`; `None` when its definition is not +/// stored (nothing on chain can be decided without it). +pub async fn gather_chain_scan( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + row: &Graph, + status: GraphStatus, + height: i64, +) -> anyhow::Result> { + let definition = { + let mut storage = local_db.acquire().await?; + load_validated_graph_definition(&mut storage, row.instance_id, row.graph_id).await? + }; + let Some(definition) = definition else { + return Ok(None); + }; + let graph = BitvmGcGraph::from_simplified(&definition)?; + let chain = scan_graph_chain_state(btc_client, goat_client, &graph).await?; + let unusable = if before_prekickoff(chain.status) { + unusable_fact(btc_client, &graph, row).await? + } else { + None + }; + let mut signals = stage_signals(&chain); + match chain.status { + GraphStatus::OperatorKickOff => { + let timelocks = graph_timelocks(&graph)?; + if let Some(signal) = take1_ready(btc_client, local_db, row, &timelocks, height).await? + { + signals.push(signal); + } + } + GraphStatus::Challenge => { + let timelocks = graph_timelocks(&graph)?; + signals.extend(challenge_signals(btc_client, local_db, row, &timelocks, height).await?); + } + _ => {} + } + signals.dedup(); + let scan = ChainScan { + height, + status_at_scan: status.to_string(), + stage: chain.status, + sub_status: Some(serde_json::to_string(&chain.sub_status)?), + challenge_txid: chain.challenge_txid, + unusable, + signals, + }; + debug!( + event = "graph_scan", + graph_id = %row.graph_id, + status = %status, + stage = %scan.stage, + height, + unusable = ?scan.unusable, + signals = ?scan.signals, + "chain scan" + ); + Ok(Some(scan)) +} + +/// Pre-Kickoff of the graph is not on chain yet. +fn before_prekickoff(stage: GraphStatus) -> bool { + matches!( + stage, + GraphStatus::OperatorPresigned + | GraphStatus::CommitteePresigned + | GraphStatus::OperatorDataPushed + ) +} + +/// A chain fact that rules the graph out before its Pre-Kickoff confirms. +async fn unusable_fact( + btc_client: &BTCClient, + graph: &BitvmGcGraph, + row: &Graph, +) -> anyhow::Result> { + let slot = row.kickoff_index as u64; + if let Some(dead) = dead_slot_below(&row.operator_pubkey, slot) { + return Ok(Some(Unusable::EarlierSlotDead { slot: dead })); + } + if let Some((outpoint, spent_by)) = prekickoff_input_spent(btc_client, graph).await? { + record_dead_slot(&row.operator_pubkey, slot); + return Ok(Some(Unusable::PrekickoffInputSpent { outpoint, spent_by })); + } + Ok(connector_zero_taken(btc_client, graph) + .await? + .map(|spent_by| Unusable::ConnectorZeroTaken { spent_by })) +} + +/// An input of the graph's Pre-Kickoff spent by another confirmed +/// transaction: the Pre-Kickoff can never confirm. A conflicting spend still +/// in the mempool may never confirm, and what follows is irreversible. +pub async fn prekickoff_input_spent( + btc_client: &BTCClient, + graph: &BitvmGcGraph, +) -> anyhow::Result> { + let prekickoff = graph.cur_prekickoff.tx(); + let prekickoff_txid = prekickoff.compute_txid(); + for input in &prekickoff.input { + let outpoint = input.previous_output; + if let Some(spent_by) = + outpoint_spent_txid(btc_client, &outpoint.txid, u64::from(outpoint.vout)).await? + && spent_by != prekickoff_txid + && btc_client.get_tx_status(&spent_by).await?.confirmed + { + return Ok(Some((outpoint, spent_by))); + } + } + Ok(None) +} + +/// Connector-0 spent by a confirmed transaction other than this graph's +/// Take-1 / Take-2: another graph of the instance took the peg-in. +pub async fn connector_zero_taken( + btc_client: &BTCClient, + graph: &BitvmGcGraph, +) -> anyhow::Result> { + let Some(input) = graph.take1.tx().input.first() else { + return Ok(None); + }; + let outpoint = input.previous_output; + let take1 = graph.take1.tx().compute_txid(); + let take2 = graph.take2.tx().compute_txid(); + let Some(spent_by) = outpoint_spent_txid(btc_client, &outpoint.txid, u64::from(outpoint.vout)) + .await? + .filter(|spent_by| *spent_by != take1 && *spent_by != take2) + else { + return Ok(None); + }; + Ok(btc_client.get_tx_status(&spent_by).await?.confirmed.then_some(spent_by)) +} + +/// Signals implied by chain progress, including stages crossed between scans. +fn stage_signals(chain: &GraphChainScan) -> Vec { + use GraphStatus::*; + let stage = chain.status; + let mut signals = Vec::new(); + if !before_prekickoff(stage) { + signals.push(PegoutSignal::PreKickoffSent); + } + if matches!(stage, OperatorKickOff | Challenge | Disprove | OperatorTake1 | OperatorTake2) { + signals.push(PegoutSignal::KickoffSent); + } + if let Some(challenge_txid) = chain.challenge_txid { + signals.push(PegoutSignal::ChallengeSent { challenge_txid }); + } + match stage { + OperatorTake1 => signals.push(PegoutSignal::Take1Sent), + OperatorTake2 => signals.push(PegoutSignal::Take2Sent), + Disprove => { + if let Some(disprove) = &chain.disprove { + signals.push(PegoutSignal::DisproveSent { + disprove_type: disprove.disprove_type, + index: disprove.index, + challenge_finish_txid: disprove.challenge_finish_txid, + }); + } + } + _ => {} + } + signals +} + +fn graph_timelocks(graph: &BitvmGcGraph) -> anyhow::Result { + validate_timelock_config( + graph.parameters.instance_parameters.network, + &graph.parameters.timelock_config, + )?; + Ok(graph.parameters.timelock_config) +} + +fn is_timelock_ready(confirmed_height: i64, lock_blocks: i64, current_height: i64) -> bool { + confirmed_height > 0 && confirmed_height + lock_blocks <= current_height +} + +/// Confirmation height of a transaction, cached in the graph's monitor table. +async fn confirmed_monitor( + btc_client: &BTCClient, + local_db: &LocalDB, + graph_id: Uuid, + txid: Txid, + tx_name: String, +) -> anyhow::Result> { + let txid_serial = SerializableTxid::from(txid); + let existing = { + let mut storage = local_db.acquire().await?; + storage.find_graph_btc_tx_vout_monitor(&graph_id, &txid_serial).await? + }; + if let Some(existing) = existing + && existing.height > 0 + && existing.vout_len > 0 + { + return Ok(Some(existing)); + } + let Some(tx_info) = btc_client.get_tx_info(&txid).await? else { + return Ok(None); + }; + let height = tx_info.status.block_height.unwrap_or_default() as i64; + if height <= 0 { + return Ok(None); + } + let now = current_time_secs(); + let monitor = GraphBtcTxVoutMonitor { + graph_id, + tx_name, + txid: txid_serial, + height, + vout_len: tx_info.vout.len() as i64, + monitor_data: String::new(), + created_at: now, + updated_at: now, + }; + local_db.acquire().await?.upsert_graph_btc_tx_vout_monitor(&monitor).await?; + Ok(Some(monitor)) +} + +/// Kicked off, Connector-A and the Guardian Connector unspent, `T1` elapsed. +async fn take1_ready( + btc_client: &BTCClient, + local_db: &LocalDB, + row: &Graph, + timelocks: &TimelockConfig, + height: i64, +) -> anyhow::Result> { + let Some(kickoff_txid) = row.kickoff_txid.clone().map(Txid::from) else { + warn!(graph_id = %row.graph_id, "kickoff txid is missing"); + return Ok(None); + }; + let connector_a = output_topology::kickoff::connector_a() as u64; + let guardian = output_topology::kickoff::guardian_connector() as u64; + if outpoint_spent_txid(btc_client, &kickoff_txid, connector_a).await?.is_some() + || outpoint_spent_txid(btc_client, &kickoff_txid, guardian).await?.is_some() + { + return Ok(None); + } + let Some(kickoff) = confirmed_monitor( + btc_client, + local_db, + row.graph_id, + kickoff_txid, + MONITOR_KICKOFF.to_string(), + ) + .await? + else { + return Ok(None); + }; + let lock_blocks = take1_timelock_blocks(get_network(), timelocks) as i64; + Ok(is_timelock_ready(kickoff.height, lock_blocks, height).then_some(PegoutSignal::Take1Ready)) +} + +/// The challenge sub-flows of a graph in Challenge: watchtower branches, +/// commit-pubin, asserts, and Take-2. +async fn challenge_signals( + btc_client: &BTCClient, + local_db: &LocalDB, + row: &Graph, + timelocks: &TimelockConfig, + height: i64, +) -> anyhow::Result> { + let mut signals = watchtower_flow(btc_client, local_db, row, timelocks, height).await?; + signals.extend(assert_flow(btc_client, local_db, row).await?); + signals.extend(wrongly_challenge_timeout(btc_client, local_db, row, timelocks, height).await?); + if take2_ready(btc_client, local_db, row, timelocks, height).await? { + signals.push(PegoutSignal::Take2Ready); + } + Ok(signals) +} + +/// Watchtower-Challenge-Init and its branches: each watchtower challenged, +/// timed out, ACKed or NACKed; then commit-pubin and the operator's assert. +async fn watchtower_flow( + btc_client: &BTCClient, + local_db: &LocalDB, + row: &Graph, + timelocks: &TimelockConfig, + height: i64, +) -> anyhow::Result> { + let Some(wci_txid) = row.watchtower_challenge_init_txid.clone().map(Txid::from) else { + return Ok(Vec::new()); + }; + let Some(monitor) = confirmed_monitor( + btc_client, + local_db, + row.graph_id, + wci_txid, + MONITOR_WATCHTOWER_INIT.to_string(), + ) + .await? + else { + trace!(graph_id = %row.graph_id, "watchtower challenge init not confirmed"); + return Ok(Vec::new()); + }; + let mut signals = vec![PegoutSignal::WatchtowerChallengeInitSent]; + let watchtower_num = output_topology::watchtower_challenge_init::watchtower_num( + monitor.vout_len.max(0) as usize, + ); + if watchtower_num == 0 { + return Ok(signals); + } + let network = get_network(); + let connector_e = + output_topology::watchtower_challenge_init::connector_e(watchtower_num) as u64; + let connector_f = + output_topology::watchtower_challenge_init::connector_f(watchtower_num) as u64; + let connector_e_spent = outpoint_spent_txid(btc_client, &wci_txid, connector_e).await?; + let connector_f_spent = outpoint_spent_txid(btc_client, &wci_txid, connector_f).await?; + let commit_timeout_txid: Option = + row.operator_commit_timeout_txid.clone().map(Into::into); + let commit_timed_out = matches!( + (connector_e_spent.as_ref(), commit_timeout_txid.as_ref()), + (Some(spent), Some(timeout)) if spent == timeout + ); + let pubin_committed = connector_e_spent.is_some() && !commit_timed_out; + + let mut all_resolved = true; + let mut timeout_ready = false; + let mut nack_ready = Vec::new(); + for index in 0..watchtower_num { + let watchtower_vout = + output_topology::watchtower_challenge_init::watchtower_connector(index) as u64; + let ack_vout = output_topology::watchtower_challenge_init::ack_connector(index) as u64; + let watchtower_spent = outpoint_spent_txid(btc_client, &wci_txid, watchtower_vout).await?; + let ack_spent = outpoint_spent_txid(btc_client, &wci_txid, ack_vout).await?; + let timeout_txid: Option = + row.watchtower_challenge_timeout_txids.get(index).cloned().map(Into::into); + let watchtower_confirmed = match watchtower_spent.as_ref() { + Some(txid) => btc_client.get_tx_status(txid).await?.confirmed, + None => false, + }; + let ack_confirmed = match ack_spent.as_ref() { + Some(txid) => btc_client.get_tx_status(txid).await?.confirmed, + None => false, + }; + let timed_out = matches!( + (watchtower_spent.as_ref(), timeout_txid.as_ref()), + (Some(spent), Some(timeout)) if spent == timeout && watchtower_confirmed + ); + match watchtower_spent { + Some(_) if !watchtower_confirmed => all_resolved = false, + Some(_) if !timed_out => { + signals.push(PegoutSignal::WatchtowerChallengeSent { watchtower_index: index }); + if !ack_confirmed { + all_resolved = false; + if connector_f_spent.is_none() + && ack_spent.is_none() + && is_timelock_ready( + monitor.height, + operator_ack_timelock_blocks(network, timelocks) as i64, + height, + ) + { + nack_ready.push(index); + } + } + } + Some(_) => {} + None => { + all_resolved = false; + if ack_spent.is_none() + && is_timelock_ready( + monitor.height, + watchtower_challenge_timelock_blocks(network, timelocks) as i64, + height, + ) + { + timeout_ready = true; + } + } + } + } + if timeout_ready { + signals.push(PegoutSignal::WatchtowerChallengeTimeout); + } + for watchtower_index in nack_ready { + signals.push(PegoutSignal::NackReady { watchtower_index }); + } + if all_resolved && connector_e_spent.is_none() && connector_f_spent.is_none() { + signals.push(PegoutSignal::OperatorCommitPubinReady); + if is_timelock_ready( + monitor.height, + operator_commit_timelock_blocks(network, timelocks) as i64, + height, + ) { + signals.push(PegoutSignal::OperatorCommitPubinTimeout); + } + } + if all_resolved + && pubin_committed + && let Some(assert_txid) = row.operator_assert_txid.clone() + && !btc_client.get_tx_status(&assert_txid.0).await?.confirmed + { + signals.push(PegoutSignal::AssertReady); + } + Ok(signals) +} + +/// Prover-Assert confirmed, and each Verifier-Assert confirmed after it. +async fn assert_flow( + btc_client: &BTCClient, + local_db: &LocalDB, + row: &Graph, +) -> anyhow::Result> { + let Some(assert_txid) = row.operator_assert_txid.clone().map(Txid::from) else { + return Ok(Vec::new()); + }; + if confirmed_monitor( + btc_client, + local_db, + row.graph_id, + assert_txid, + MONITOR_PROVER_ASSERT.to_string(), + ) + .await? + .is_none() + { + return Ok(Vec::new()); + } + let mut signals = vec![PegoutSignal::AssertSent { assert_txid }]; + for (verifier_index, txid) in row.verifier_assert_txids.iter().enumerate() { + let challenge_assert_txid: Txid = txid.clone().into(); + if confirmed_monitor( + btc_client, + local_db, + row.graph_id, + challenge_assert_txid, + format!("{MONITOR_VERIFIER_ASSERT}_{verifier_index}"), + ) + .await? + .is_some() + { + signals + .push(PegoutSignal::ChallengeAssertSent { challenge_assert_txid, verifier_index }); + } + } + Ok(signals) +} + +/// Every Verifier-Assert unanswered past `T2`: each of those verifiers may +/// disprove, and each only acts on its own slot. +async fn wrongly_challenge_timeout( + btc_client: &BTCClient, + local_db: &LocalDB, + row: &Graph, + timelocks: &TimelockConfig, + height: i64, +) -> anyhow::Result> { + let mut signals = Vec::new(); + let Some(assert_txid) = row.operator_assert_txid.clone().map(Txid::from) else { + return Ok(signals); + }; + if row.verifier_assert_txids.is_empty() { + return Ok(signals); + } + let connector_d = + output_topology::operator_assert::connector_d(row.verifier_assert_txids.len()) as u64; + // Confirmed spends only: an answer that is still in the mempool can be + // replaced, and the disprove job waits it out itself. + if outpoint_spent_confirmed(btc_client, &assert_txid, connector_d).await?.is_some() { + return Ok(signals); + } + let disprove_blocks = disprove_timelock_blocks(get_network(), timelocks) as i64; + for (index, txid) in row.verifier_assert_txids.iter().enumerate() { + let challenge_assert_txid: Txid = txid.clone().into(); + if outpoint_spent_confirmed(btc_client, &challenge_assert_txid, 0).await?.is_some() { + continue; + } + let Some(monitor) = confirmed_monitor( + btc_client, + local_db, + row.graph_id, + challenge_assert_txid, + format!("{MONITOR_VERIFIER_ASSERT}_{index}"), + ) + .await? + else { + continue; + }; + if monitor.height + disprove_blocks <= height { + signals.push(PegoutSignal::WronglyChallengeTimeout { + challenge_assert_txid, + verifier_index: index, + }); + } + } + Ok(signals) +} + +/// Connector-D and the Guardian Connector unspent, `T3` after the assert and +/// `T7` after Watchtower-Challenge-Init. +async fn take2_ready( + btc_client: &BTCClient, + local_db: &LocalDB, + row: &Graph, + timelocks: &TimelockConfig, + height: i64, +) -> anyhow::Result { + let (Some(kickoff_txid), Some(wci_txid), Some(assert_txid)) = ( + row.kickoff_txid.clone().map(Txid::from), + row.watchtower_challenge_init_txid.clone().map(Txid::from), + row.operator_assert_txid.clone().map(Txid::from), + ) else { + return Ok(false); + }; + let connector_d = + output_topology::operator_assert::connector_d(row.verifier_assert_txids.len()) as u64; + let guardian = output_topology::kickoff::guardian_connector() as u64; + if outpoint_spent_txid(btc_client, &assert_txid, connector_d).await?.is_some() + || outpoint_spent_txid(btc_client, &kickoff_txid, guardian).await?.is_some() + { + return Ok(false); + } + let network = get_network(); + let Some(assert) = confirmed_monitor( + btc_client, + local_db, + row.graph_id, + assert_txid, + MONITOR_PROVER_ASSERT.to_string(), + ) + .await? + else { + return Ok(false); + }; + let Some(wci) = confirmed_monitor( + btc_client, + local_db, + row.graph_id, + wci_txid, + MONITOR_WATCHTOWER_INIT.to_string(), + ) + .await? + else { + return Ok(false); + }; + Ok(is_timelock_ready(assert.height, take2_timelock_blocks(network, timelocks) as i64, height) + && is_timelock_ready( + wci.height, + connector_f_timelock_blocks(network, timelocks) as i64, + height, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::DetectedDisprove; + use GraphStatus::*; + + fn chain(stage: GraphStatus) -> GraphChainScan { + GraphChainScan::for_test(stage) + } + + #[test] + fn a_scan_resumes_after_the_slot_the_last_tick_stopped_at() { + let slot = |op: &str, nonce: i64| (op.to_string(), nonce); + let slots = [slot("a", 0), slot("a", 1), slot("b", 0)]; + assert_eq!(resume_index(&slots, None), 0); + assert_eq!(resume_index(&slots, Some(&slot("a", 0))), 1); + assert_eq!(resume_index(&slots, Some(&slot("a", 1))), 2); + // Past the end, or at a slot that disappeared: wrap, or the next one. + assert_eq!(resume_index(&slots, Some(&slot("b", 0))), 0); + assert_eq!(resume_index(&slots, Some(&slot("a", 5))), 2); + assert_eq!(resume_index(&[], Some(&slot("a", 0))), 0); + } + + #[test] + fn edges_follow_the_chain_stage() { + assert!(stage_signals(&chain(OperatorDataPushed)).is_empty()); + assert_eq!(stage_signals(&chain(PreKickoff)), vec![PegoutSignal::PreKickoffSent]); + assert_eq!(stage_signals(&chain(Skipped)), vec![PegoutSignal::PreKickoffSent]); + assert_eq!( + stage_signals(&chain(OperatorTake1)), + vec![PegoutSignal::PreKickoffSent, PegoutSignal::KickoffSent, PegoutSignal::Take1Sent] + ); + let challenge_txid = Txid::from_str(&"44".repeat(32)).unwrap(); + let mut disproved = chain(Disprove); + disproved.challenge_txid = Some(challenge_txid); + disproved.disprove = Some(DetectedDisprove { + disprove_type: DisproveTxType::Disprove, + index: 1, + challenge_finish_txid: challenge_txid, + }); + let signals = stage_signals(&disproved); + assert!(signals.contains(&PegoutSignal::ChallengeSent { challenge_txid })); + assert!(signals.iter().any(|signal| matches!( + signal, + PegoutSignal::DisproveSent { index: 1, disprove_type: DisproveTxType::Disprove, .. } + ))); + assert!(is_timelock_ready(10, 5, 15) && !is_timelock_ready(10, 5, 14)); + assert!(!is_timelock_ready(0, 0, 15)); + } + + #[test] + fn later_slots_report_the_lowest_dead_slot() { + let operator = "03dead-slot-test"; + assert_eq!(dead_slot_below(operator, 9), None); + record_dead_slot(operator, 5); + record_dead_slot(operator, 7); + assert_eq!(dead_slot_below(operator, 9), Some(5)); + assert_eq!(dead_slot_below(operator, 5), None, "the dead slot reports its own fact"); + record_dead_slot(operator, 3); + assert_eq!(dead_slot_below(operator, 5), Some(3)); + assert_eq!(dead_slot_below("03another-operator", 9), None); + } +} diff --git a/node/src/aggregates/graph_setup.rs b/node/src/aggregates/graph_setup.rs new file mode 100644 index 000000000..446a5d7eb --- /dev/null +++ b/node/src/aggregates/graph_setup.rs @@ -0,0 +1,360 @@ +//! Graph setup helpers for candidate selection, verifier indices, message +//! identity checks and signed `InitGraph` messages. + +use crate::env::{ + get_peer_id, get_verifier_candidate_backup_count, get_verifier_candidate_collection_window_secs, +}; +use crate::messages::{InitGraph, SolderingProofReady, sign_init_graph}; +use crate::utils::{ + OperatorBabeSetupState, OperatorVerifierCandidate, min_required_verifier, + soldering_payload_hash, soldering_payload_hash_hex, +}; +use anyhow::{Context, Result, anyhow, bail}; +use bitcoin::PublicKey; +use bitcoin::key::Keypair; +use bitvm_lib::babe_adapter::{ + BABE_M_CC, BabeProverState, CACSetupPackage, CompactSolderingProofPayload, + FinalizedInstanceData, SolderingData, derive_finalized_indices, +}; +use bitvm_lib::types::{BitvmGcCircuitData, BitvmGcGraph}; +use libp2p::PeerId; +use std::str::FromStr; +use uuid::Uuid; + +pub(crate) fn build_signed_init_graph( + instance_id: Uuid, + graph_id: Uuid, + operator_keypair: &Keypair, +) -> Result { + let operator_peer_id = PeerId::from_str(&get_peer_id()) + .context("decode local operator peer id for InitGraph")? + .to_bytes(); + Ok(sign_init_graph(instance_id, graph_id, operator_keypair, operator_peer_id)) +} + +/// A graph-bearing message has two identity representations: its envelope and +/// the signed graph parameters. Never use one to read/write local state while +/// using the other to reconstruct or scan the graph. +pub(crate) fn message_identity_matches( + message_kind: &str, + message_instance_id: Uuid, + message_graph_id: Uuid, + message_graph_nonce: u64, + graph_instance_id: Uuid, + graph_graph_id: Uuid, + graph_nonce: u64, +) -> bool { + if message_instance_id == graph_instance_id + && message_graph_id == graph_graph_id + && message_graph_nonce == graph_nonce + { + return true; + } + + tracing::warn!( + "Ignore {message_kind}: message identity {message_instance_id}:{message_graph_id}:{message_graph_nonce:?} does not match graph parameters {graph_instance_id}:{graph_graph_id}:{graph_nonce}" + ); + false +} + +/// Closes a bounded candidate collection window and assigns provisional slots. +/// Final graph slots are assigned only after valid soldering proofs are collected. +pub(crate) fn freeze_operator_candidates(state: &mut OperatorBabeSetupState) -> Result<()> { + if state.candidate_verifier_pubkeys.is_some() { + return Ok(()); + } + if state.candidates.len() < min_required_verifier() { + bail!( + "cannot freeze {} verifier candidates before reaching target {}", + state.candidates.len(), + min_required_verifier() + ); + } + let mut verifier_peer_ids = std::collections::HashSet::new(); + for candidate in &state.candidates { + if !verifier_peer_ids.insert(&candidate.verifier_peer_id) { + bail!("cannot freeze duplicate verifier peer id"); + } + } + state.candidates.sort_by_key(|candidate| candidate.verifier_pubkey.to_bytes()); + for (candidate_index, candidate) in state.candidates.iter_mut().enumerate() { + candidate.candidate_index = Some(candidate_index); + candidate.selected_circuit_indexes = + derive_finalized_indices(&candidate.setup_package, BABE_M_CC)?; + } + state.candidate_verifier_pubkeys = + Some(state.candidates.iter().map(|candidate| candidate.verifier_pubkey).collect()); + Ok(()) +} + +/// Whether the candidate pool may be frozen now: the required number of +/// candidates is present and either the pool is full or the collection +/// window elapsed. Returns the seconds still to wait otherwise. +pub(crate) fn candidate_freeze_readiness( + state: &OperatorBabeSetupState, + now: i64, +) -> CandidateReadiness { + if state.candidate_verifier_pubkeys.is_some() { + return CandidateReadiness::Frozen; + } + let required = min_required_verifier(); + let limit = required.saturating_add(get_verifier_candidate_backup_count()); + if state.candidates.len() < required { + return CandidateReadiness::Short; + } + if state.candidates.len() >= limit { + return CandidateReadiness::Ready; + } + let window = get_verifier_candidate_collection_window_secs(); + let started_at = state.candidate_collection_started_at.unwrap_or(now); + let elapsed = now - started_at; + if elapsed >= window { + CandidateReadiness::Ready + } else { + CandidateReadiness::Wait((window - elapsed).max(1)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CandidateReadiness { + Frozen, + Short, + Ready, + Wait(i64), +} + +pub(crate) fn seal_selected_verifiers( + state: &mut OperatorBabeSetupState, + selected_verifier_pubkeys: Vec, +) -> Result<()> { + if let Some(existing) = &state.selected_verifier_pubkeys { + if existing != &selected_verifier_pubkeys { + bail!("refuse to replace the sealed verifier selection"); + } + return Ok(()); + } + if selected_verifier_pubkeys.len() != min_required_verifier() { + bail!( + "selected verifier count {} does not match required {}", + selected_verifier_pubkeys.len(), + min_required_verifier() + ); + } + if selected_verifier_pubkeys.windows(2).any(|pair| pair[0].to_bytes() >= pair[1].to_bytes()) { + bail!("selected verifier public keys must be unique and canonically ordered"); + } + if selected_verifier_pubkeys.iter().any(|selected| { + !state.candidates.iter().any(|candidate| candidate.verifier_pubkey == *selected) + }) { + bail!("sealed verifier selection contains a non-candidate public key"); + } + state.selected_verifier_pubkeys = Some(selected_verifier_pubkeys); + Ok(()) +} + +pub(crate) fn selected_gc_data(state: &OperatorBabeSetupState) -> Result> { + let selected = state + .selected_verifier_pubkeys + .as_ref() + .ok_or_else(|| anyhow!("verifier selection is not sealed"))?; + selected + .iter() + .map(|verifier_pubkey| { + state + .candidates + .iter() + .find(|candidate| candidate.verifier_pubkey == *verifier_pubkey) + .and_then(|candidate| candidate.gc_data.clone()) + .ok_or_else(|| { + anyhow!("missing verified GC data for sealed verifier {verifier_pubkey}") + }) + }) + .collect() +} + +pub(crate) fn selected_candidate_for_graph_index( + state: &OperatorBabeSetupState, + verifier_index: usize, +) -> Result<&OperatorVerifierCandidate> { + let verifier_pubkey = state + .selected_verifier_pubkeys + .as_ref() + .and_then(|selected| selected.get(verifier_index)) + .ok_or_else(|| anyhow!("missing sealed verifier slot {verifier_index}"))?; + state + .candidates + .iter() + .find(|candidate| candidate.verifier_pubkey == *verifier_pubkey) + .ok_or_else(|| anyhow!("missing candidate for sealed verifier slot {verifier_index}")) +} + +/// Stores verified proof data against its immutable candidate slot. +pub(crate) fn record_candidate_gc_data( + state: &mut OperatorBabeSetupState, + verifier_pubkey: PublicKey, + candidate_index: usize, + setup_package: &CACSetupPackage, + gc_data: BitvmGcCircuitData, + prover_state: &BabeProverState, + soldering_proof_ready: SolderingProofReady, +) -> Result<()> { + if gc_data.verifier_pubkey != verifier_pubkey { + bail!("GC slot owner does not match soldering proof verifier"); + } + let candidate = state + .candidates + .iter_mut() + .find(|candidate| candidate.verifier_pubkey == verifier_pubkey) + .ok_or_else(|| anyhow!("selected verifier candidate is missing"))?; + if candidate.candidate_index != Some(candidate_index) { + bail!("candidate index does not match soldering proof slot"); + } + if candidate.setup_package != *setup_package { + bail!("soldering proof setup package does not match selected verifier candidate"); + } + if prover_state.package != *setup_package { + bail!("BABE prover state setup package does not match selected verifier candidate"); + } + if candidate.selected_circuit_indexes != prover_state.soldering.finalized_indices { + bail!("BABE prover state finalized indices do not match selected verifier cut"); + } + if soldering_proof_ready.candidate_index != candidate_index { + bail!("soldering proof reference candidate index does not match selected verifier"); + } + if prover_state.finalized.len() != BABE_M_CC || prover_state.h_msgs.len() != BABE_M_CC { + bail!("BABE prover state must contain exactly {BABE_M_CC} finalized instances and hashes"); + } + if prover_state.h_msgs != gc_data.final_msg_hashlocks { + bail!("BABE prover state hashes do not match GC slot hashes"); + } + if let Some(existing) = &candidate.gc_data + && existing != &gc_data + { + bail!("conflicting GC data received for verifier candidate"); + } + if let Some(existing) = &candidate.soldering_proof_ready + && existing != &soldering_proof_ready + { + bail!("conflicting soldering proof reference received for verifier candidate"); + } + if candidate.gc_data.is_none() { + candidate.gc_data = Some(gc_data); + } + if candidate.soldering_proof_ready.is_none() { + candidate.soldering_proof_ready = Some(soldering_proof_ready); + } + Ok(()) +} + +pub(crate) fn build_babe_prover_state( + setup_package: &CACSetupPackage, + finalized: Vec, + soldering: SolderingData, +) -> Result { + let h_msgs = finalized + .iter() + .map(|finalized| { + setup_package + .commits + .get(finalized.index) + .map(|commit| commit.h_msg) + .ok_or_else(|| anyhow!("finalized BABE instance index is out of range")) + }) + .collect::>>()?; + Ok(BabeProverState { package: setup_package.clone(), finalized, soldering, h_msgs }) +} + +pub(crate) fn validate_verifier_slot(graph: &BitvmGcGraph, verifier_index: usize) -> Result<()> { + let gc_data_len = graph.parameters.gc_data.len(); + let verifier_asserts_len = graph.verifier_asserts.len(); + let disproves_len = graph.disproves.len(); + if gc_data_len == 0 { + bail!("graph has no verifier GC slots"); + } + if gc_data_len != verifier_asserts_len || gc_data_len != disproves_len { + bail!( + "graph verifier branch lengths differ: gc_data={gc_data_len}, verifier_asserts={verifier_asserts_len}, disproves={disproves_len}" + ); + } + if verifier_index >= gc_data_len { + bail!("verifier index {verifier_index} out of range for {gc_data_len} slots"); + } + Ok(()) +} + +pub(crate) fn find_verifier_index_by_pubkey( + gc_data: &[BitvmGcCircuitData], + verifier_pubkey: &PublicKey, +) -> Result> { + let matches = gc_data + .iter() + .enumerate() + .filter_map(|(index, data)| (data.verifier_pubkey == *verifier_pubkey).then_some(index)) + .collect::>(); + match matches.as_slice() { + [] => Ok(None), + [index] => Ok(Some(*index)), + _ => bail!("graph contains duplicate slots for verifier {verifier_pubkey}"), + } +} + +/// Candidates still owed CutCircuits, excluding delivered proofs and closed selections. +pub(crate) fn cut_circuits_owed( + state: &OperatorBabeSetupState, + now: i64, +) -> Vec { + if state.candidate_verifier_pubkeys.is_none() || state.selected_verifier_pubkeys.is_some() { + return Vec::new(); + } + let delivered = state.candidates.iter().filter(|candidate| candidate.gc_data.is_some()).count(); + let window_over = state.proof_collection_started_at.is_some_and(|started_at| { + now - started_at >= get_verifier_candidate_collection_window_secs() + }); + if window_over && delivered >= min_required_verifier() { + return Vec::new(); + } + state + .candidates + .iter() + .filter(|candidate| { + candidate.gc_data.is_none() && candidate.soldering_proof_ready.is_none() + }) + .cloned() + .collect() +} + +pub(crate) fn decode_soldering_proof_payload( + soldering_proof_ready: &SolderingProofReady, + payload: &[u8], +) -> Result { + let SolderingProofReady { instance_id, graph_id, candidate_index, payload_hash, total_len } = + soldering_proof_ready; + if payload.len() != *total_len { + tracing::warn!( + instance_id = %instance_id, + graph_id = %graph_id, + candidate_index, + actual_len = payload.len(), + total_len, + payload_hash = %soldering_payload_hash_hex(payload_hash), + "SolderingProof payload length mismatch" + ); + bail!( + "SolderingProof payload length {} does not match total_len {total_len}", + payload.len() + ); + } + let actual_hash = soldering_payload_hash(payload); + if actual_hash != *payload_hash { + tracing::warn!( + instance_id = %instance_id, + graph_id = %graph_id, + candidate_index, + expected_hash = %soldering_payload_hash_hex(payload_hash), + actual_hash = %soldering_payload_hash_hex(&actual_hash), + "SolderingProof payload hash mismatch" + ); + bail!("SolderingProof payload hash mismatch"); + } + bincode::deserialize(payload).context("deserialize compact soldering proof payload") +} diff --git a/node/src/aggregates/graph_tests.rs b/node/src/aggregates/graph_tests.rs new file mode 100644 index 000000000..7c0be5883 --- /dev/null +++ b/node/src/aggregates/graph_tests.rs @@ -0,0 +1,1385 @@ +//! Graph coordination tests using the engine and SQLite. +//! +//! Scripted jobs exercise routing, slot rules, missing definitions, job plans +//! and delivery closing; they do not perform real graph cryptography. + +use super::common::{ + InboundMessage, MembershipProbe, NextJob, PlannedMessage, content_event_id, delivery_id, +}; +use super::graph::*; +use super::graph_fetch::GraphLookup; +use super::graph_pegout::*; +use super::jobs::Plan; +use async_trait::async_trait; +use bitvm_lib::actors::Actor; +use engine::{ + Backoff, EngineBuilder, EngineConfig, EngineHandle, EngineInboxState, EventSource, JobContext, + JobOutcome, JobRunner, NewEvent, +}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use store::localdb::LocalDB; +use store::{ + EngineJobState, EngineOutboxState, Graph, GraphStatus, GraphStatusSource, Instance, + InstanceBridgeInStatus, +}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +const WAIT: Duration = Duration::from_secs(10); +const LOCAL_PEER: &str = "12D3KooWLocalPeer"; +const OPERATOR: &str = "02operator"; + +/// Returns the scripted plan for an action and records what it was asked. +#[derive(Default)] +struct Scripted { + plans: Mutex>, + calls: Mutex)>>, + /// Runs of these actions block until the test releases them. + holds: Mutex>>, +} + +impl Scripted { + fn hold(&self, action: &str) { + self.holds.lock().unwrap().insert(action.to_string(), Arc::new(tokio::sync::Notify::new())); + } + + fn release(&self, action: &str) { + if let Some(hold) = self.holds.lock().unwrap().remove(action) { + hold.notify_waiters(); + hold.notify_one(); + } + } + + fn release_all(&self) { + let actions: Vec = self.holds.lock().unwrap().keys().cloned().collect(); + for action in actions { + self.release(&action); + } + } +} + +#[async_trait] +impl JobRunner for Scripted { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let params = ctx.job.params.clone().unwrap_or_default(); + self.calls.lock().unwrap().push((ctx.job.action.clone(), params)); + let hold = self.holds.lock().unwrap().get(&ctx.job.action).cloned(); + if let Some(hold) = hold { + hold.notified().await; + } + let plan = self.plans.lock().unwrap().remove(&ctx.job.action).unwrap_or_default(); + JobOutcome::Done(plan.encode().unwrap()) + } +} + +struct Harness { + db: LocalDB, + _dir: tempfile::TempDir, + handle: EngineHandle, + cancel: CancellationToken, + run: tokio::task::JoinHandle>, + jobs: Arc, +} + +fn fast_config() -> EngineConfig { + EngineConfig { + worker_count: 1, + thread_stack_bytes: 0, + apply_timeout: Duration::from_secs(5), + worker_poll_interval: Duration::from_millis(30), + recheck_interval: Duration::from_millis(250), + inbox_retry_backoff: Backoff::Steps(vec![1]), + job_lease_secs: 5, + job_poll_interval: Duration::from_millis(30), + outbox_poll_interval: Duration::from_millis(30), + ..EngineConfig::default() + } +} + +/// Every instance is this node's (the production probe looks for the key +/// envelope on disk). +struct Member; + +impl MembershipProbe for Member { + fn holds_instance_key(&self, _instance_id: Uuid) -> bool { + true + } +} + +/// A node outside every committee. +struct Outsider; + +impl MembershipProbe for Outsider { + fn holds_instance_key(&self, _instance_id: Uuid) -> bool { + false + } +} + +/// A member whose registry knows no peer. +struct Unacquainted; + +impl MembershipProbe for Unacquainted { + fn holds_instance_key(&self, _instance_id: Uuid) -> bool { + true + } + + fn is_registered_peer(&self, _peer: &str) -> bool { + false + } +} + +async fn start(actor: Actor, relayer: bool) -> Harness { + start_with(actor, relayer, Arc::new(Member)).await +} + +async fn start_with(actor: Actor, relayer: bool, membership: Arc) -> Harness { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("graph.db"); + let db = LocalDB::new_with_busy_timeout( + &format!("sqlite:{}", path.display()), + false, + store::DEFAULT_BUSY_TIMEOUT, + ) + .await; + db.migrate().await; + let jobs = Arc::new(Scripted::default()); + let local_operator = (actor == Actor::Operator).then(|| OPERATOR.to_string()); + let mut builder = EngineBuilder::new(db.clone(), fast_config()).handler( + GRAPH_KIND, + Arc::new( + GraphAggregate::new(actor, relayer, LOCAL_PEER.to_string()) + .with_membership(membership) + .with_local_operator(local_operator), + ), + ); + for action in ALL_JOBS.iter().chain(PEGOUT_JOBS) { + let runner: Arc = jobs.clone(); + builder = builder.job_runner(GRAPH_KIND, *action, runner); + } + let (engine, _acks) = builder.build(); + let handle = engine.handle(); + let cancel = CancellationToken::new(); + let run = tokio::spawn(engine.run(cancel.clone())); + Harness { db, _dir: dir, handle, cancel, run, jobs } +} + +impl Harness { + async fn stop(self) { + self.jobs.release_all(); + self.cancel.cancel(); + self.run.await.unwrap().unwrap(); + } + + fn script(&self, action: &str, plan: GraphPlan) { + self.jobs.plans.lock().unwrap().insert(action.to_string(), plan); + } + + fn calls(&self) -> Vec<(String, Vec)> { + self.jobs.calls.lock().unwrap().clone() + } + + /// The params the runner was handed for the first call of `action`. + fn job_params(&self, action: &str) -> Vec { + self.calls() + .into_iter() + .find(|(called, _)| called == action) + .map(|(_, params)| params) + .unwrap_or_else(|| panic!("{action} was never run")) + } + + async fn insert_graph(&self, instance_id: Uuid, nonce: i64, status: GraphStatus) -> Uuid { + let graph_id = Uuid::new_v4(); + let mut storage = self.db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&Graph { + graph_id, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_string(), + operator_pubkey: OPERATOR.to_string(), + kickoff_index: nonce, + ..Default::default() + }) + .await + .unwrap(); + if status != GraphStatus::OperatorPresigned { + self.set_status(instance_id, graph_id, status).await; + } + graph_id + } + + async fn insert_session(&self, instance_id: Uuid, graph_id: Uuid) { + let mut storage = self.db.acquire().await.unwrap(); + storage.upsert_pending_graph_init(&instance_id, OPERATOR, &graph_id).await.unwrap(); + } + + async fn set_instance_status(&self, instance_id: Uuid, status: InstanceBridgeInStatus) { + let mut storage = self.db.acquire().await.unwrap(); + storage + .update_instance( + &store::localdb::InstanceUpdate::new_with_instance_id(instance_id) + .with_status(status.to_string()), + ) + .await + .unwrap(); + } + + async fn insert_instance(&self, status: InstanceBridgeInStatus) -> Uuid { + let instance_id = Uuid::new_v4(); + let mut storage = self.db.acquire().await.unwrap(); + assert!( + storage + .insert_instance_if_absent(&Instance { + instance_id, + status: status.to_string(), + ..Default::default() + }) + .await + .unwrap() + ); + instance_id + } + + async fn set_status(&self, instance_id: Uuid, graph_id: Uuid, status: GraphStatus) { + let mut storage = self.db.acquire().await.unwrap(); + storage + .transition_graph_status( + instance_id, + graph_id, + status, + GraphStatusSource::ChainReconcile, + None, + ) + .await + .unwrap(); + } + + /// A network message event as `inbound_event` would build it, without a + /// real graph: the apply only reads the envelope. + fn message( + &self, + kind: &str, + author: &str, + instance_id: Uuid, + graph_id: Uuid, + nonce: Option, + ) -> NewEvent { + let inbound = InboundMessage { + author: author.to_string(), + instance_id, + graph_nonce: nonce, + operator_pubkey: Some(OPERATOR.to_string()), + message: format!("{kind}:{graph_id}").into_bytes(), + }; + let mut event = NewEvent::new( + content_event_id(author, kind, &graph_id.to_string(), &inbound.message), + graph_ref(graph_id), + kind, + EventSource::Network, + inbound.encode().unwrap(), + ) + .with_author(author); + if ack_required(kind) { + event = event.with_ack(author, delivery_id(author, LOCAL_PEER, &inbound.message)); + } + event + } + + async fn submit(&self, event: NewEvent) -> String { + let event_id = event.event_id.clone(); + self.handle.submit(event).await.unwrap(); + event_id + } + + /// `None` until the event is recorded (an apply may record it later). + async fn inbox(&self, event_id: &str) -> Option { + self.db.acquire().await.unwrap().find_engine_inbox_event(event_id).await.unwrap() + } + + async fn wait_inbox(&self, event_id: &str, state: EngineInboxState) -> store::EngineInboxEvent { + wait_for(|| async { + self.inbox(event_id).await.filter(|row| row.state == state.to_string()) + }) + .await + } + + async fn wait_waiting(&self, event_id: &str) -> store::EngineInboxEvent { + wait_for(|| async { self.inbox(event_id).await.filter(|row| row.waiting_ref.is_some()) }) + .await + } + + async fn jobs_of(&self, graph_id: Uuid) -> Vec { + let mut storage = self.db.acquire().await.unwrap(); + storage.list_engine_jobs_for_aggregate(GRAPH_KIND, &graph_id.to_string()).await.unwrap() + } + + async fn wait_job( + &self, + graph_id: Uuid, + action: &str, + state: EngineJobState, + ) -> store::EngineJob { + wait_for(|| async { + self.jobs_of(graph_id) + .await + .into_iter() + .find(|job| job.action == action && job.state == state.to_string()) + }) + .await + } + + /// The job is queued or running: enqueued by an apply, no longer gated. + async fn wait_job_live(&self, graph_id: Uuid, action: &str) -> store::EngineJob { + wait_for(|| async { + self.jobs_of(graph_id).await.into_iter().find(|job| { + job.action == action + && (job.state == EngineJobState::Pending.to_string() + || job.state == EngineJobState::Running.to_string()) + }) + }) + .await + } + + async fn outbox(&self, outbox_id: &str) -> Option { + self.db.acquire().await.unwrap().find_engine_outbox_message(outbox_id).await.unwrap() + } + + async fn wait_outbox( + &self, + outbox_id: &str, + state: EngineOutboxState, + ) -> store::EngineOutboxMessage { + wait_for(|| async { + self.outbox(outbox_id).await.filter(|row| row.state == state.to_string()) + }) + .await + } + + async fn aggregate(&self, graph_id: Uuid) -> store::EngineAggregate { + self.handle.aggregate(&graph_ref(graph_id)).await.unwrap().unwrap() + } + + async fn wait_phase(&self, graph_id: Uuid, phase: &str) -> store::EngineAggregate { + wait_for(|| async { + let row = self.aggregate(graph_id).await; + (row.phase == phase).then_some(row) + }) + .await + } +} + +async fn wait_for(mut probe: F) -> T +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let started = tokio::time::Instant::now(); + loop { + if let Some(value) = probe().await { + return value; + } + assert!(started.elapsed() < WAIT, "condition not met within {WAIT:?}"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn settle() { + tokio::time::sleep(Duration::from_millis(400)).await; +} + +/// The plan a sealed proof collection returns: build the definition. +fn build_plan() -> GraphPlan { + let mut plan = GraphPlan::noted("proofs sealed"); + plan.then(JOB_BUILD_GRAPH, Vec::new(), None); + plan +} + +fn build_ready_id(graph_id: Uuid) -> String { + build_ready_event(graph_id).event_id +} + +fn planned(graph_id: Uuid, kind: &str, phase: &str, ack: Option<&str>) -> PlannedMessage { + PlannedMessage { + outbox_id: outbox_id(graph_id, kind, None), + phase_ref: phase.to_string(), + kind: kind.to_string(), + actor: Actor::All.to_string(), + payload: kind.as_bytes().to_vec(), + ack_set: ack.map(|peer| BTreeMap::from([(peer.to_string(), "delivery".to_string())])), + } +} + +// ------------------------------------------------------------------ routing + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_message_becomes_an_ingest_job_and_its_plan_is_committed() { + let t = start(Actor::Committee, false).await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let follow_up = GraphPlan { + messages: vec![planned(graph_id, "NonceGeneration", PHASE_SIGNING, None)], + next: vec![NextJob { + action: JOB_COMMITTEE_VOTE.to_string(), + params: vec![9], + delay_secs: Some(600), + }], + state: GraphStateUpdate { + instance_id: Some(instance_id), + graph_nonce: Some(0), + ..Default::default() + }, + ..Default::default() + }; + t.script(JOB_COMMITTEE_INGEST_GRAPH, follow_up); + + // The committee's CreateGraph for slot 0 needs no predecessor. + let event_id = + t.submit(t.message("CreateGraph", "peer-op", instance_id, graph_id, Some(0))).await; + let row = t.wait_inbox(&event_id, EngineInboxState::Applied).await; + assert_eq!(row.ack_peer.as_deref(), Some("peer-op"), "heavy messages carry their ACK"); + t.wait_job(graph_id, JOB_COMMITTEE_INGEST_GRAPH, EngineJobState::Done).await; + let calls = t.calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, JOB_COMMITTEE_INGEST_GRAPH); + assert_eq!(InboundMessage::decode(&calls[0].1).unwrap().instance_id, instance_id); + + // The plan: a message in the outbox, the next job delayed, the state kept. + let sent = + t.wait_outbox(&outbox_id(graph_id, "NonceGeneration", None), EngineOutboxState::Open).await; + assert_eq!(sent.phase_ref, PHASE_SIGNING); + assert_eq!(sent.topic, crate::middleware::get_topic_name("All")); + let vote = t.wait_job(graph_id, JOB_COMMITTEE_VOTE, EngineJobState::Pending).await; + assert!(vote.next_at >= vote.created_at + 599, "run_after keeps the delay"); + assert_eq!(vote.params.as_deref(), Some(&[9][..])); + let row = t.aggregate(graph_id).await; + let state: GraphState = serde_json::from_str(row.state.as_deref().unwrap()).unwrap(); + assert_eq!(state.instance_id, Some(instance_id)); + // No graph row yet: the phase is still Setup. + assert_eq!(row.phase, PHASE_SETUP); + + // A copy of the same message is a duplicate, applied once. + let again = t.submit(t.message("CreateGraph", "peer-op", instance_id, graph_id, Some(0))).await; + assert_eq!(again, event_id); + settle().await; + assert_eq!(t.calls().len(), 1); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn own_messages_and_unhandled_roles_are_ignored() { + let t = start(Actor::Operator, false).await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + // From this node: nothing to do. + let own = t.submit(t.message("GenCircuits", LOCAL_PEER, instance_id, graph_id, None)).await; + t.wait_inbox(&own, EngineInboxState::Applied).await; + // An operator never ingests a CreateGraph. + let foreign = + t.submit(t.message("CreateGraph", "peer-x", instance_id, graph_id, Some(0))).await; + t.wait_inbox(&foreign, EngineInboxState::Failed).await; + // A GenCircuits without a pending session is dropped. + let stray = t.submit(t.message("GenCircuits", "peer-v", instance_id, graph_id, None)).await; + t.wait_inbox(&stray, EngineInboxState::Applied).await; + settle().await; + assert!(t.calls().is_empty()); + t.stop().await; +} + +// ------------------------------------------------------------------- slots + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_second_definition_for_a_slot_is_refused() { + let t = start(Actor::Committee, false).await; + let instance_id = Uuid::new_v4(); + let holder = t.insert_graph(instance_id, 1, GraphStatus::OperatorPresigned).await; + let newcomer = Uuid::new_v4(); + let event_id = + t.submit(t.message("CreateGraph", "peer-op", instance_id, newcomer, Some(1))).await; + let row = t.wait_inbox(&event_id, EngineInboxState::Failed).await; + assert!(row.last_error.unwrap_or_default().contains(&holder.to_string())); + settle().await; + assert!(t.calls().is_empty(), "no job for a refused definition"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_definition_waits_for_its_predecessor_and_pulls_it() { + let t = start(Actor::Committee, false).await; + let instance_id = Uuid::new_v4(); + let graph_id = Uuid::new_v4(); + let event_id = + t.submit(t.message("CreateGraph", "peer-op", instance_id, graph_id, Some(2))).await; + let row = t.wait_waiting(&event_id).await; + assert_eq!(row.waiting_ref.as_deref(), Some(slot_ref(OPERATOR, 1).as_str())); + // The pull is a job with the slot to look up (a finished job has its + // params cleared, so read what the runner was handed). + t.wait_job(graph_id, JOB_FETCH_GRAPH, EngineJobState::Done).await; + let lookup: GraphLookup = bincode::deserialize(&t.job_params(JOB_FETCH_GRAPH)).unwrap(); + assert_eq!(lookup, GraphLookup::Slot { operator_pubkey: OPERATOR.to_string(), graph_nonce: 1 }); + assert!(t.jobs_of(graph_id).await.iter().all(|job| job.action == JOB_FETCH_GRAPH)); + + // The predecessor arrives by any path; waking the slot releases the event. + t.insert_graph(Uuid::new_v4(), 1, GraphStatus::OperatorPresigned).await; + t.handle.notify(&[slot_ref(OPERATOR, 1)]).await.unwrap(); + t.wait_inbox(&event_id, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_COMMITTEE_INGEST_GRAPH, EngineJobState::Done).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_value_from_an_unregistered_author_waits_without_a_pull() { + let t = start_with(Actor::Committee, false, Arc::new(Unacquainted)).await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let event_id = + t.submit(t.message("NonceGeneration", "peer-x", instance_id, graph_id, None)).await; + t.wait_waiting(&event_id).await; + settle().await; + assert!(t.jobs_of(graph_id).await.is_empty(), "no GetGraph pull for an unknown author"); + t.stop().await; +} + +// ------------------------------------------------------------ round values + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn round_values_wait_for_the_definition_and_stop_once_the_round_is_over() { + let t = start(Actor::Committee, false).await; + let instance_id = Uuid::new_v4(); + let graph_id = Uuid::new_v4(); + // No definition yet: parked on the graph itself, and pulled by id. + let nonce = t.submit(t.message("NonceGeneration", "peer-c", instance_id, graph_id, None)).await; + let row = t.wait_waiting(&nonce).await; + assert_eq!(row.waiting_ref.as_deref(), Some(graph_ref(graph_id).wake_ref().as_str())); + t.wait_job(graph_id, JOB_FETCH_GRAPH, EngineJobState::Done).await; + let lookup: GraphLookup = bincode::deserialize(&t.job_params(JOB_FETCH_GRAPH)).unwrap(); + assert_eq!(lookup, GraphLookup::Id { instance_id, graph_id, finalized: false }); + assert!(t.calls().iter().all(|(action, _)| action == JOB_FETCH_GRAPH)); + + // The definition is stored (by the ingest job in production): the + // aggregate's own wake releases the value. + let mut storage = t.db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&Graph { + graph_id, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_string(), + operator_pubkey: OPERATOR.to_string(), + kickoff_index: 0, + ..Default::default() + }) + .await + .unwrap(); + drop(storage); + t.handle.notify(&[graph_ref(graph_id).wake_ref()]).await.unwrap(); + t.wait_inbox(&nonce, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_INGEST_NONCES, EngineJobState::Done).await; + let row = t.wait_phase(graph_id, PHASE_SIGNING).await; + assert_eq!(row.status, GraphStatus::OperatorPresigned.to_string()); + + // Once finalized, late round values are ignored and the round's + // deliveries are closed. + t.script( + JOB_INGEST_PRESIGN, + GraphPlan { + messages: vec![planned(graph_id, "CommitteePresign", PHASE_SIGNING, None)], + ..Default::default() + }, + ); + let presign = + t.submit(t.message("CommitteePresign", "peer-c", instance_id, graph_id, None)).await; + t.wait_inbox(&presign, EngineInboxState::Applied).await; + let sent = outbox_id(graph_id, "CommitteePresign", None); + t.wait_outbox(&sent, EngineOutboxState::Open).await; + t.set_status(instance_id, graph_id, GraphStatus::CommitteePresigned).await; + let late = + t.submit(t.message("AggNonceConsensus", "peer-c", instance_id, graph_id, None)).await; + t.wait_inbox(&late, EngineInboxState::Applied).await; + let closed = t.wait_outbox(&sent, EngineOutboxState::Closed).await; + assert!(closed.close_reason.unwrap_or_default().contains("CommitteePresigned")); + assert_eq!( + t.wait_phase(graph_id, PHASE_SIGNED).await.status, + GraphStatus::CommitteePresigned.to_string() + ); + assert!( + !t.calls().iter().any(|(action, _)| action == JOB_INGEST_VOTE), + "late vote never ingested" + ); + t.stop().await; +} + +// ---------------------------------------------------------- operator start + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_operator_start_runs_the_setup_and_posting_closes_every_delivery() { + let t = start(Actor::Operator, false).await; + let instance_id = Uuid::new_v4(); + let graph_id = Uuid::new_v4(); + t.script( + JOB_OPERATOR_INIT, + GraphPlan { + messages: vec![planned(graph_id, "InitGraph", PHASE_SETUP, None)], + state: GraphStateUpdate { instance_id: Some(instance_id), ..Default::default() }, + ..Default::default() + }, + ); + let start_id = t.submit(operator_start_event(instance_id, graph_id, OPERATOR).unwrap()).await; + t.wait_inbox(&start_id, EngineInboxState::Applied).await; + let init = + t.wait_outbox(&outbox_id(graph_id, "InitGraph", None), EngineOutboxState::Open).await; + assert_eq!(init.phase_ref, PHASE_SETUP); + let calls = t.calls(); + assert_eq!(calls.len(), 1); + let params: OperatorStart = bincode::deserialize(&calls[0].1).unwrap(); + assert_eq!(params.instance_id, instance_id); + + // The build job delivers CreateGraph with its ack set and closes InitGraph. + t.script( + JOB_BUILD_GRAPH, + GraphPlan { + messages: vec![planned(graph_id, "CreateGraph", PHASE_COLLECT, Some("peer-v"))], + close_rows: vec![outbox_id(graph_id, "InitGraph", None)], + state: GraphStateUpdate { graph_nonce: Some(0), ..Default::default() }, + ..Default::default() + }, + ); + // Reached through a plan of another job: GenCircuits needs a session. + let mut storage = t.db.acquire().await.unwrap(); + storage.upsert_pending_graph_init(&instance_id, OPERATOR, &graph_id).await.unwrap(); + drop(storage); + t.script( + JOB_INGEST_GEN_CIRCUITS, + GraphPlan { + next: vec![NextJob { + action: JOB_BUILD_GRAPH.to_string(), + params: vec![], + delay_secs: None, + }], + ..Default::default() + }, + ); + let circuits = t.submit(t.message("GenCircuits", "peer-v", instance_id, graph_id, None)).await; + t.wait_inbox(&circuits, EngineInboxState::Applied).await; + let create = + t.wait_outbox(&outbox_id(graph_id, "CreateGraph", None), EngineOutboxState::Open).await; + assert_eq!(create.ack_set.unwrap().get("peer-v").map(String::as_str), Some("delivery")); + t.wait_outbox(&outbox_id(graph_id, "InitGraph", None), EngineOutboxState::Closed).await; + + // Posted on Goat: the watcher's status write plus the posted event close + // everything that is left and move the phase on. + t.insert_graph_with_id(instance_id, graph_id).await; + t.set_status(instance_id, graph_id, GraphStatus::CommitteePresigned).await; + t.set_status(instance_id, graph_id, GraphStatus::OperatorDataPushed).await; + let posted = t.submit(graph_posted_event(graph_id).unwrap()).await; + t.wait_inbox(&posted, EngineInboxState::Applied).await; + t.wait_outbox(&outbox_id(graph_id, "CreateGraph", None), EngineOutboxState::Closed).await; + assert_eq!( + t.wait_phase(graph_id, PHASE_AWAIT_WITHDRAW).await.status, + GraphStatus::OperatorDataPushed.to_string() + ); + t.stop().await; +} + +impl Harness { + async fn insert_graph_with_id(&self, instance_id: Uuid, graph_id: Uuid) { + let mut storage = self.db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&Graph { + graph_id, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_string(), + operator_pubkey: OPERATOR.to_string(), + kickoff_index: 0, + ..Default::default() + }) + .await + .unwrap(); + } +} + +// ----------------------------------------------------------------- retired + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_obsoleted_graph_ignores_messages_and_cancels_its_jobs() { + let t = start(Actor::Committee, false).await; + let instance_id = Uuid::new_v4(); + let graph_id = t.insert_graph(instance_id, 0, GraphStatus::OperatorPresigned).await; + // A pending job of the graph, then the graph is obsoleted. + t.script( + JOB_INGEST_NONCES, + GraphPlan { + next: vec![NextJob { + action: JOB_COMMITTEE_VOTE.to_string(), + params: vec![], + delay_secs: Some(3_600), + }], + ..Default::default() + }, + ); + let nonce = t.submit(t.message("NonceGeneration", "peer-c", instance_id, graph_id, None)).await; + t.wait_inbox(&nonce, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_COMMITTEE_VOTE, EngineJobState::Pending).await; + t.set_status(instance_id, graph_id, GraphStatus::Obsoleted).await; + let late = t.submit(t.message("CommitteePresign", "peer-c", instance_id, graph_id, None)).await; + t.wait_inbox(&late, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_COMMITTEE_VOTE, EngineJobState::Cancelled).await; + assert_eq!( + t.wait_phase(graph_id, PHASE_OBSOLETED).await.status, + GraphStatus::Obsoleted.to_string() + ); + assert!(!t.calls().iter().any(|(action, _)| action == JOB_INGEST_PRESIGN)); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_plan_finished_after_the_graph_retired_is_dropped() { + let t = start(Actor::Committee, false).await; + let instance_id = Uuid::new_v4(); + let graph_id = t.insert_graph(instance_id, 0, GraphStatus::OperatorPresigned).await; + // The ingest is running when the graph is obsoleted (a chain scan, say); + // its plan would send a message and queue the next step. + t.jobs.hold(JOB_INGEST_NONCES); + t.script( + JOB_INGEST_NONCES, + GraphPlan { + messages: vec![planned(graph_id, "NonceGeneration", PHASE_SIGNING, None)], + next: vec![NextJob { + action: JOB_COMMITTEE_VOTE.to_string(), + params: vec![], + delay_secs: None, + }], + wake_refs: vec![slot_ref(OPERATOR, 1)], + ..Default::default() + }, + ); + let nonce = t.submit(t.message("NonceGeneration", "peer-c", instance_id, graph_id, None)).await; + t.wait_inbox(&nonce, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_INGEST_NONCES, EngineJobState::Running).await; + t.set_status(instance_id, graph_id, GraphStatus::Obsoleted).await; + t.jobs.release(JOB_INGEST_NONCES); + t.wait_job(graph_id, JOB_INGEST_NONCES, EngineJobState::Done).await; + t.wait_phase(graph_id, PHASE_OBSOLETED).await; + settle().await; + assert!(t.outbox(&outbox_id(graph_id, "NonceGeneration", None)).await.is_none()); + assert!(!t.jobs_of(graph_id).await.iter().any(|job| job.action == JOB_COMMITTEE_VOTE)); + t.stop().await; +} + +// ------------------------------------------------------------ predecessor + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_start_and_its_build_do_not_wait_for_the_previous_slot() { + let t = start(Actor::Operator, false).await; + // Slot 0 is built but not finalized and its instance is alive: the next setup starts at + // once and builds on its definition. + let previous_instance = + t.insert_instance(InstanceBridgeInStatus::UserBroadcastPeginPrepare).await; + t.insert_graph(previous_instance, 0, GraphStatus::OperatorPresigned).await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + t.insert_session(instance_id, graph_id).await; + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_id = t.submit(operator_start_event(instance_id, graph_id, OPERATOR).unwrap()).await; + assert!(t.wait_inbox(&start_id, EngineInboxState::Applied).await.waiting_ref.is_none()); + t.wait_job(graph_id, JOB_OPERATOR_INIT, EngineJobState::Done).await; + t.wait_inbox(&build_ready_id(graph_id), EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_BUILD_GRAPH, EngineJobState::Done).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failed_instance_settles_its_slot_and_drops_its_own_pending_start() { + let t = start(Actor::Operator, false).await; + // The previous slot's graph never got past OperatorPresigned, but its instance failed: the + // slot is settled. + let failed_instance = t.insert_instance(InstanceBridgeInStatus::UserCanceled).await; + t.insert_graph(failed_instance, 3, GraphStatus::OperatorPresigned).await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + t.script(JOB_OPERATOR_INIT, GraphPlan::noted("started")); + let start_id = t.submit(operator_start_event(instance_id, graph_id, OPERATOR).unwrap()).await; + t.wait_inbox(&start_id, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_OPERATOR_INIT, EngineJobState::Done).await; + + // A start for an instance that failed meanwhile drops its session. + let doomed_instance = t.insert_instance(InstanceBridgeInStatus::UserDiscarded).await; + let doomed_graph = Uuid::new_v4(); + let mut storage = t.db.acquire().await.unwrap(); + storage.upsert_pending_graph_init(&doomed_instance, OPERATOR, &doomed_graph).await.unwrap(); + drop(storage); + let start_id = + t.submit(operator_start_event(doomed_instance, doomed_graph, OPERATOR).unwrap()).await; + t.wait_inbox(&start_id, EngineInboxState::Applied).await; + assert!( + t.db.acquire() + .await + .unwrap() + .find_pending_graph_init_by_graph_id(&doomed_graph) + .await + .unwrap() + .is_none() + ); + assert_eq!(t.calls().len(), 1, "no job for the doomed graph"); + t.stop().await; +} + +// ------------------------------------------------------------- membership + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_committee_node_outside_the_committee_ignores_all_but_graph_finalize() { + let t = start_with(Actor::Committee, false, Arc::new(Outsider)).await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let create = + t.submit(t.message("CreateGraph", "peer-op", instance_id, graph_id, Some(0))).await; + t.wait_inbox(&create, EngineInboxState::Applied).await; + let nonce = t.submit(t.message("NonceGeneration", "peer-c", instance_id, graph_id, None)).await; + t.wait_inbox(&nonce, EngineInboxState::Applied).await; + settle().await; + assert!(t.calls().is_empty(), "neither ingested nor pulled"); + assert!(t.jobs_of(graph_id).await.is_empty()); + + let finalize = + t.submit(t.message("GraphFinalize", "peer-op", instance_id, graph_id, Some(0))).await; + t.wait_inbox(&finalize, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_INGEST_FINALIZED, EngineJobState::Done).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_round_value_of_a_failed_instance_is_dropped_instead_of_parked() { + let t = start(Actor::Committee, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::Timeout).await; + let graph_id = Uuid::new_v4(); + let nonce = t.submit(t.message("NonceGeneration", "peer-c", instance_id, graph_id, None)).await; + let row = t.wait_inbox(&nonce, EngineInboxState::Applied).await; + assert!(row.waiting_ref.is_none()); + settle().await; + assert!(t.jobs_of(graph_id).await.is_empty(), "nothing is pulled for a failed instance"); + t.stop().await; +} + +// -------------------------------------------------------------------- pull + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_pulled_finalized_graph_is_posted_by_the_relayer() { + let t = start(Actor::Committee, true).await; + let instance_id = Uuid::new_v4(); + let graph_id = t.insert_graph(instance_id, 0, GraphStatus::CommitteePresigned).await; + let pulled = t.submit(graph_pulled_event(graph_id, instance_id, true).unwrap()).await; + t.wait_inbox(&pulled, EngineInboxState::Applied).await; + let post = t.wait_job(graph_id, JOB_POST_GRAPH_DATA, EngineJobState::Done).await; + assert_eq!(post.phase, PHASE_POST); + assert_eq!( + t.wait_phase(graph_id, PHASE_POST).await.status, + GraphStatus::CommitteePresigned.to_string() + ); + + // A pre-signed copy is not posted, and a non-relayer never posts. + let other = t.insert_graph(instance_id, 1, GraphStatus::OperatorPresigned).await; + let pulled = t.submit(graph_pulled_event(other, instance_id, false).unwrap()).await; + t.wait_inbox(&pulled, EngineInboxState::Applied).await; + settle().await; + assert!(t.jobs_of(other).await.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn builds_of_one_operator_run_one_at_a_time() { + let t = start(Actor::Operator, false).await; + let instance_a = t.insert_instance(InstanceBridgeInStatus::UserBroadcastPeginPrepare).await; + let instance_b = t.insert_instance(InstanceBridgeInStatus::UserBroadcastPeginPrepare).await; + let (graph_a, graph_b) = (Uuid::new_v4(), Uuid::new_v4()); + t.insert_session(instance_a, graph_a).await; + t.insert_session(instance_b, graph_b).await; + t.jobs.hold(JOB_BUILD_GRAPH); + + // A starts on an empty chain and builds slot 0. + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_a = t.submit(operator_start_event(instance_a, graph_a, OPERATOR).unwrap()).await; + t.wait_inbox(&start_a, EngineInboxState::Applied).await; + t.wait_job(graph_a, JOB_BUILD_GRAPH, EngineJobState::Running).await; + // B starts meanwhile; only its build waits, on the slot A is taking. + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_b = t.submit(operator_start_event(instance_b, graph_b, OPERATOR).unwrap()).await; + assert!(t.wait_inbox(&start_b, EngineInboxState::Applied).await.waiting_ref.is_none()); + let ready_b = t.wait_waiting(&build_ready_id(graph_b)).await; + assert_eq!(ready_b.waiting_ref.as_deref(), Some(slot_ref(OPERATOR, 0).as_str())); + settle().await; + assert_eq!(t.called(JOB_BUILD_GRAPH).len(), 1, "one build at a time"); + + // A stores slot 0: its row wakes the slot and B builds slot 1. + t.insert_graph_with_id(instance_a, graph_a).await; + t.jobs.release(JOB_BUILD_GRAPH); + t.wait_job(graph_a, JOB_BUILD_GRAPH, EngineJobState::Done).await; + t.wait_inbox(&build_ready_id(graph_b), EngineInboxState::Applied).await; + t.wait_job(graph_b, JOB_BUILD_GRAPH, EngineJobState::Done).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_build_of_a_failed_instance_does_not_hold_the_next_build() { + let t = start(Actor::Operator, false).await; + let instance_a = t.insert_instance(InstanceBridgeInStatus::UserBroadcastPeginPrepare).await; + let instance_b = t.insert_instance(InstanceBridgeInStatus::UserBroadcastPeginPrepare).await; + let (graph_a, graph_b) = (Uuid::new_v4(), Uuid::new_v4()); + t.insert_session(instance_a, graph_a).await; + t.insert_session(instance_b, graph_b).await; + t.jobs.hold(JOB_BUILD_GRAPH); + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_a = t.submit(operator_start_event(instance_a, graph_a, OPERATOR).unwrap()).await; + t.wait_inbox(&start_a, EngineInboxState::Applied).await; + t.wait_job(graph_a, JOB_BUILD_GRAPH, EngineJobState::Running).await; + + // A's instance times out while its build runs (the session row is still + // there, as on a node that applied the failure without this check): B's + // build does not wait on it. + t.set_instance_status(instance_a, InstanceBridgeInStatus::Timeout).await; + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_b = t.submit(operator_start_event(instance_b, graph_b, OPERATOR).unwrap()).await; + t.wait_inbox(&start_b, EngineInboxState::Applied).await; + let ready_b = t.wait_inbox(&build_ready_id(graph_b), EngineInboxState::Applied).await; + assert!(ready_b.waiting_ref.is_none()); + t.wait_job_live(graph_b, JOB_BUILD_GRAPH).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_unfinished_graph_is_given_up_at_the_abandon_height() { + let t = start(Actor::Operator, false).await; + // Other operators' graphs carried the instance; this operator's graph + // never got its committee signatures. Past the abandon height it is + // retired and its slot left for the Skip. + let carried = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let unfinished = t.insert_graph(carried, 0, GraphStatus::OperatorPresigned).await; + let abandoned = SetupAbandoned { instance_id: carried, abandon_height: 166, height: 167 }; + t.submit(setup_abandoned_event(unfinished, &abandoned).unwrap()).await; + t.wait_status(unfinished, GraphStatus::Obsoleted).await; + assert!(t.graph_state(unfinished).await.abandoned); + + // A finalized graph, or another operator's, is never given up. + let finalized = t.insert_graph(carried, 1, GraphStatus::CommitteePresigned).await; + let foreign = + t.insert_graph_of(OTHER_OPERATOR, carried, 0, GraphStatus::OperatorPresigned).await; + for graph_id in [finalized, foreign] { + let event_id = t.submit(setup_abandoned_event(graph_id, &abandoned).unwrap()).await; + t.wait_inbox(&event_id, EngineInboxState::Applied).await; + } + assert_eq!(t.status(finalized).await, GraphStatus::CommitteePresigned); + assert_eq!(t.status(foreign).await, GraphStatus::OperatorPresigned); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_session_unfinished_at_the_abandon_height_is_released() { + let t = start(Actor::Operator, false).await; + let instance_a = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let instance_b = t.insert_instance(InstanceBridgeInStatus::Presigned).await; + let instance_c = t.insert_instance(InstanceBridgeInStatus::Presigned).await; + let (graph_a, graph_b, graph_c) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + for (instance_id, graph_id) in + [(instance_a, graph_a), (instance_b, graph_b), (instance_c, graph_c)] + { + t.insert_session(instance_id, graph_id).await; + } + let abandon = |instance_id| SetupAbandoned { instance_id, abandon_height: 166, height: 166 }; + t.jobs.hold(JOB_BUILD_GRAPH); + + // A's build is running and B's build waits behind it when A's instance + // passes the abandon height: A's session goes and B builds. + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_a = t.submit(operator_start_event(instance_a, graph_a, OPERATOR).unwrap()).await; + t.wait_inbox(&start_a, EngineInboxState::Applied).await; + t.wait_job(graph_a, JOB_BUILD_GRAPH, EngineJobState::Running).await; + t.script(JOB_OPERATOR_INIT, build_plan()); + let start_b = t.submit(operator_start_event(instance_b, graph_b, OPERATOR).unwrap()).await; + t.wait_inbox(&start_b, EngineInboxState::Applied).await; + t.wait_waiting(&build_ready_id(graph_b)).await; + t.submit(setup_abandoned_event(graph_a, &abandon(instance_a)).unwrap()).await; + t.wait_inbox(&build_ready_id(graph_b), EngineInboxState::Applied).await; + t.wait_job_live(graph_b, JOB_BUILD_GRAPH).await; + assert!(t.session(graph_a).await.is_none()); + assert!(t.graph_state(graph_a).await.abandoned); + + // A session abandoned before its start applies is dropped. + let abandoned = t.submit(setup_abandoned_event(graph_c, &abandon(instance_c)).unwrap()).await; + t.wait_inbox(&abandoned, EngineInboxState::Applied).await; + let start_c = t.submit(operator_start_event(instance_c, graph_c, OPERATOR).unwrap()).await; + t.wait_inbox(&start_c, EngineInboxState::Applied).await; + settle().await; + assert!(t.session(graph_c).await.is_none()); + assert_eq!(t.called(JOB_OPERATOR_INIT).len(), 2, "C never starts"); + t.stop().await; +} + +// ------------------------------------------------------------------ peg-out + +const OTHER_OPERATOR: &str = "02other"; + +impl Harness { + async fn insert_graph_of( + &self, + operator: &str, + instance_id: Uuid, + nonce: i64, + status: GraphStatus, + ) -> Uuid { + let graph_id = Uuid::new_v4(); + let mut storage = self.db.acquire().await.unwrap(); + storage + .upsert_graph_definition(&Graph { + graph_id, + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_string(), + operator_pubkey: operator.to_string(), + kickoff_index: nonce, + ..Default::default() + }) + .await + .unwrap(); + drop(storage); + if status != GraphStatus::OperatorPresigned { + self.set_status(instance_id, graph_id, status).await; + } + graph_id + } + + async fn status(&self, graph_id: Uuid) -> GraphStatus { + let row = self.db.acquire().await.unwrap().find_graph(&graph_id).await.unwrap().unwrap(); + row.status.parse().unwrap() + } + + async fn wait_status(&self, graph_id: Uuid, status: GraphStatus) { + wait_for(|| async { (self.status(graph_id).await == status).then_some(()) }).await + } + + async fn graph_state(&self, graph_id: Uuid) -> GraphState { + serde_json::from_str(self.aggregate(graph_id).await.state.as_deref().unwrap()).unwrap() + } + + async fn session(&self, graph_id: Uuid) -> Option { + let mut storage = self.db.acquire().await.unwrap(); + storage.find_pending_graph_init_by_graph_id(&graph_id).await.unwrap() + } + + fn called(&self, action: &str) -> Vec> { + self.calls() + .into_iter() + .filter(|(called, _)| called == action) + .map(|(_, params)| params) + .collect() + } +} + +fn scan( + height: i64, + status_at_scan: GraphStatus, + stage: GraphStatus, + signals: Vec, + unusable: Option, +) -> ChainScan { + ChainScan { + height, + status_at_scan: status_at_scan.to_string(), + stage, + sub_status: None, + challenge_txid: None, + unusable, + signals, + } +} + +fn txid(byte: u8) -> bitcoin::Txid { + use bitcoin::hashes::Hash; + bitcoin::Txid::from_byte_array([byte; 32]) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_withdraw_waits_for_the_previous_slot_and_kicks_off_once_it_settles() { + let t = start(Actor::Operator, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let previous = t.insert_graph(instance_id, 0, GraphStatus::OperatorDataPushed).await; + let graph_id = t.insert_graph(instance_id, 1, GraphStatus::OperatorDataPushed).await; + + let init = t.submit(withdraw_init_event(graph_id, "0xinit").unwrap()).await; + let row = t.wait_waiting(&init).await; + assert_eq!(row.waiting_ref.as_deref(), Some(slot_ref(OPERATOR, 0).as_str())); + // Meanwhile a job settles the earlier slots from chain facts. + t.wait_job(graph_id, JOB_SETTLE_PREDECESSORS, EngineJobState::Done).await; + let params: KickoffParams = + serde_json::from_slice(&t.job_params(JOB_SETTLE_PREDECESSORS)).unwrap(); + assert_eq!(params.tx_hash, "0xinit"); + t.wait_phase(graph_id, PHASE_KICKOFF).await; + assert!(t.called(JOB_KICKOFF).is_empty(), "no kickoff over an unsettled slot"); + + // The previous slot settles on Goat: mirroring it wakes the withdraw. + t.set_status(instance_id, previous, GraphStatus::OperatorTake1).await; + t.submit(goat_status_event(previous, GraphStatus::OperatorTake1, "0xtake1").unwrap()).await; + t.wait_inbox(&init, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_KICKOFF, EngineJobState::Done).await; + let params: KickoffParams = serde_json::from_slice(&t.job_params(JOB_KICKOFF)).unwrap(); + assert_eq!(params.tx_hash, "0xinit"); + assert_eq!(t.wait_phase(previous, PHASE_CLOSED).await.status, "OperatorTake1"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_cancelled_withdraw_never_kicks_off_and_a_new_one_starts_over() { + let t = start(Actor::Operator, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let previous = t.insert_graph(instance_id, 0, GraphStatus::OperatorKickOff).await; + let graph_id = t.insert_graph(instance_id, 1, GraphStatus::OperatorDataPushed).await; + + let init = t.submit(withdraw_init_event(graph_id, "0xfirst").unwrap()).await; + t.wait_waiting(&init).await; + let cancel = t.submit(withdraw_cancel_event(graph_id, "0xcancel").unwrap()).await; + t.wait_inbox(&cancel, EngineInboxState::Applied).await; + t.wait_phase(graph_id, PHASE_AWAIT_WITHDRAW).await; + + // The previous slot settles; the cancelled withdraw stays cancelled. + t.set_status(instance_id, previous, GraphStatus::OperatorTake1).await; + t.submit(goat_status_event(previous, GraphStatus::OperatorTake1, "0xtake1").unwrap()).await; + t.wait_inbox(&init, EngineInboxState::Applied).await; + settle().await; + assert!(t.called(JOB_KICKOFF).is_empty()); + + // A new InitWithdraw gets jobs of its own. + let again = t.submit(withdraw_init_event(graph_id, "0xsecond").unwrap()).await; + t.wait_inbox(&again, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_KICKOFF, EngineJobState::Done).await; + let params: KickoffParams = serde_json::from_slice(&t.job_params(JOB_KICKOFF)).unwrap(); + assert_eq!(params.tx_hash, "0xsecond"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_new_withdraw_replaces_the_active_one() { + let t = start(Actor::Operator, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let previous = t.insert_graph(instance_id, 0, GraphStatus::OperatorKickOff).await; + let graph_id = t.insert_graph(instance_id, 1, GraphStatus::OperatorDataPushed).await; + + let first = t.submit(withdraw_init_event(graph_id, "0xfirst").unwrap()).await; + t.wait_waiting(&first).await; + // Its cancel never reached this node: the next init takes over. + let second = t.submit(withdraw_init_event(graph_id, "0xsecond").unwrap()).await; + t.wait_waiting(&second).await; + let state = t.graph_state(graph_id).await; + assert_eq!(state.withdraw.as_deref(), Some("0xsecond")); + assert_eq!(state.retired_withdraws, vec!["0xfirst".to_string()]); + + // The previous slot settles: only the live withdraw kicks off. + t.set_status(instance_id, previous, GraphStatus::OperatorTake1).await; + t.submit(goat_status_event(previous, GraphStatus::OperatorTake1, "0xtake1").unwrap()).await; + t.wait_inbox(&first, EngineInboxState::Applied).await; + t.wait_inbox(&second, EngineInboxState::Applied).await; + t.wait_job(graph_id, JOB_KICKOFF, EngineJobState::Done).await; + settle().await; + let kickoffs = t.called(JOB_KICKOFF); + assert_eq!(kickoffs.len(), 1); + let params: KickoffParams = serde_json::from_slice(&kickoffs[0]).unwrap(); + assert_eq!(params.tx_hash, "0xsecond"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn only_the_owning_operator_acts_on_a_withdraw() { + let t = start(Actor::Operator, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let foreign = + t.insert_graph_of(OTHER_OPERATOR, instance_id, 0, GraphStatus::OperatorDataPushed).await; + let init = t.submit(withdraw_init_event(foreign, "0xother").unwrap()).await; + t.wait_inbox(&init, EngineInboxState::Applied).await; + // The first slot has no predecessor: the own graph kicks off at once. + let own = t.insert_graph(instance_id, 0, GraphStatus::OperatorDataPushed).await; + let init = t.submit(withdraw_init_event(own, "0xown").unwrap()).await; + t.wait_inbox(&init, EngineInboxState::Applied).await; + t.wait_job(own, JOB_KICKOFF, EngineJobState::Done).await; + assert!(t.jobs_of(foreign).await.is_empty()); + t.stop().await; + + let t = start(Actor::Verifier, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let graph_id = t.insert_graph(instance_id, 0, GraphStatus::OperatorDataPushed).await; + let init = t.submit(withdraw_init_event(graph_id, "0xv").unwrap()).await; + t.wait_inbox(&init, EngineInboxState::Applied).await; + settle().await; + assert!(t.calls().is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_chain_scan_records_the_stage_and_runs_each_roles_job() { + let signals = vec![PegoutSignal::PreKickoffSent, PegoutSignal::KickoffSent]; + for (actor, relayer, expected) in [ + (Actor::Verifier, false, vec![JOB_ENFORCE_PREVIOUS_SLOT, JOB_CHALLENGE_KICKOFF]), + (Actor::Committee, true, vec![JOB_PROCEED_WITHDRAW]), + (Actor::Committee, false, vec![]), + (Actor::Watchtower, false, vec![]), + ] { + let t = start(actor.clone(), relayer).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let graph_id = t.insert_graph(instance_id, 1, GraphStatus::OperatorDataPushed).await; + let event = scan( + 100, + GraphStatus::OperatorDataPushed, + GraphStatus::OperatorKickOff, + signals.clone(), + None, + ) + .event(graph_id) + .unwrap(); + let event_id = t.submit(event).await; + t.wait_inbox(&event_id, EngineInboxState::Applied).await; + t.wait_status(graph_id, GraphStatus::OperatorKickOff).await; + t.wait_phase(graph_id, PHASE_SETTLEMENT).await; + for action in &expected { + t.wait_job(graph_id, action, EngineJobState::Done).await; + } + settle().await; + let mut called: Vec = t.calls().into_iter().map(|(action, _)| action).collect(); + called.sort(); + let mut expected: Vec = expected.iter().map(ToString::to_string).collect(); + expected.sort(); + assert_eq!(called, expected, "{actor} relayer={relayer}"); + if actor == Actor::Verifier { + let check = PegoutSignal::decode(&t.job_params(JOB_ENFORCE_PREVIOUS_SLOT)).unwrap(); + assert_eq!(check, PegoutSignal::PreKickoffSent); + } + t.stop().await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_closed_graph_keeps_the_relayers_goat_call() { + let t = start(Actor::Committee, true).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let graph_id = t.insert_graph(instance_id, 0, GraphStatus::OperatorKickOff).await; + let event = scan( + 200, + GraphStatus::OperatorKickOff, + GraphStatus::OperatorTake1, + vec![PegoutSignal::PreKickoffSent, PegoutSignal::KickoffSent, PegoutSignal::Take1Sent], + None, + ) + .event(graph_id) + .unwrap(); + let event_id = t.submit(event).await; + t.wait_inbox(&event_id, EngineInboxState::Applied).await; + t.wait_phase(graph_id, PHASE_CLOSED).await; + // Closing the graph cancels setup work, not the Goat call it still owes. + t.wait_job(graph_id, JOB_FINISH_HAPPY, EngineJobState::Done).await; + t.wait_job(graph_id, JOB_PROCEED_WITHDRAW, EngineJobState::Done).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_dead_prekickoff_obsoletes_the_slot_and_every_later_one() { + let t = start(Actor::Committee, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::RelayerL2Minted).await; + let earlier = t.insert_graph(instance_id, 1, GraphStatus::OperatorDataPushed).await; + let dead = t.insert_graph(instance_id, 2, GraphStatus::CommitteePresigned).await; + let later = t.insert_graph(instance_id, 3, GraphStatus::OperatorDataPushed).await; + let foreign = + t.insert_graph_of(OTHER_OPERATOR, instance_id, 5, GraphStatus::OperatorPresigned).await; + let unusable = Unusable::PrekickoffInputSpent { + outpoint: bitcoin::OutPoint { txid: txid(1), vout: 2 }, + spent_by: txid(9), + }; + let event = scan( + 300, + GraphStatus::CommitteePresigned, + GraphStatus::CommitteePresigned, + Vec::new(), + Some(unusable), + ) + .event(dead) + .unwrap(); + let event_id = t.submit(event).await; + t.wait_inbox(&event_id, EngineInboxState::Applied).await; + t.wait_status(dead, GraphStatus::Obsoleted).await; + // A later slot follows from its own scans, which report the dead slot at + // every height: posted on Goat, it still stays Obsoleted. + let mut status_at_scan = GraphStatus::OperatorDataPushed; + for height in [300, 301] { + let behind = scan( + height, + status_at_scan, + GraphStatus::OperatorDataPushed, + Vec::new(), + Some(Unusable::EarlierSlotDead { slot: 2 }), + ); + let event_id = t.submit(behind.event(later).unwrap()).await; + t.wait_inbox(&event_id, EngineInboxState::Applied).await; + assert_eq!(t.status(later).await, GraphStatus::Obsoleted); + status_at_scan = GraphStatus::Obsoleted; + } + assert_eq!(t.status(earlier).await, GraphStatus::OperatorDataPushed); + assert_eq!(t.status(foreign).await, GraphStatus::OperatorPresigned); + assert!(t.graph_state(dead).await.dead); + assert!(!t.graph_state(later).await.dead, "only the dead slot raises the alert"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_obsoleted_graph_stays_monitored_and_follows_the_chain() { + let t = start(Actor::Verifier, false).await; + let instance_id = t.insert_instance(InstanceBridgeInStatus::Timeout).await; + let graph_id = t.insert_graph(instance_id, 1, GraphStatus::Obsoleted).await; + let taken = Unusable::ConnectorZeroTaken { spent_by: txid(7) }; + + // Posted on Goat after all: the projection gives way to the fact. + let posted = scan(10, GraphStatus::Obsoleted, GraphStatus::OperatorDataPushed, vec![], None); + t.submit(posted.event(graph_id).unwrap()).await; + t.wait_status(graph_id, GraphStatus::OperatorDataPushed).await; + // Another graph took Connector-0: unusable again. + let gone = scan( + 11, + GraphStatus::OperatorDataPushed, + GraphStatus::OperatorDataPushed, + vec![], + Some(taken), + ); + t.submit(gone.event(graph_id).unwrap()).await; + t.wait_status(graph_id, GraphStatus::Obsoleted).await; + // Its Pre-Kickoff lands: chain progress overrides the projection, and + // the verifier checks the previous slot. + let prekicked = scan( + 12, + GraphStatus::Obsoleted, + GraphStatus::PreKickoff, + vec![PegoutSignal::PreKickoffSent], + None, + ); + t.submit(prekicked.event(graph_id).unwrap()).await; + t.wait_status(graph_id, GraphStatus::PreKickoff).await; + t.wait_job(graph_id, JOB_ENFORCE_PREVIOUS_SLOT, EngineJobState::Done).await; + // Skipped: closed, out of the monitoring set. + let skipped = scan( + 13, + GraphStatus::PreKickoff, + GraphStatus::Skipped, + vec![PegoutSignal::PreKickoffSent], + None, + ); + t.submit(skipped.event(graph_id).unwrap()).await; + t.wait_status(graph_id, GraphStatus::Skipped).await; + t.wait_phase(graph_id, PHASE_CLOSED).await; + assert_eq!(t.called(JOB_ENFORCE_PREVIOUS_SLOT).len(), 1, "a signal runs its job once"); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_copy_stored_after_its_instance_failed_is_obsoleted_by_its_scan() { + let t = start(Actor::Committee, false).await; + // A pull stored the definition after the instance had already failed: + // the instance's own failure never saw it. + let instance_id = t.insert_instance(InstanceBridgeInStatus::Timeout).await; + let graph_id = t.insert_graph(instance_id, 2, GraphStatus::OperatorPresigned).await; + let event = + scan(40, GraphStatus::OperatorPresigned, GraphStatus::OperatorPresigned, Vec::new(), None) + .event(graph_id) + .unwrap(); + t.submit(event).await; + t.wait_status(graph_id, GraphStatus::Obsoleted).await; + t.wait_phase(graph_id, PHASE_OBSOLETED).await; + t.stop().await; +} diff --git a/node/src/aggregates/instance.rs b/node/src/aggregates/instance.rs new file mode 100644 index 000000000..dfa797894 --- /dev/null +++ b/node/src/aggregates/instance.rs @@ -0,0 +1,1663 @@ +//! Instance lifecycle decisions from messages, chain snapshots and job results. +//! +//! Apply updates the instance row and records its jobs, outgoing messages and +//! cleanup actions in one transaction. The engine row tracks status and phase. +//! Signing closes at the cancel height. After the abandon buffer, an unconfirmed +//! Peg-In-Confirm absent from the mempool can cause Timeout. + +use crate::aggregates::common::{ + EnvelopeProbe, InboundMessage, MembershipProbe, NextJob, PARKED_MESSAGE_TTL_SECS, + PlannedMessage, +}; +use crate::aggregates::graph::operator_start_event; +use crate::aggregates::jobs::Plan; +use crate::engine_bridge::record_event_in; +use crate::env::{ + get_committee_instance_key_delete_timelock_blocks, get_network, get_peer_id, + get_pegin_abandon_buffer_blocks, is_enable_committee_instance_key_delete, is_relayer, +}; +use crate::metrics_service::node_metrics_state; +use crate::middleware::get_topic_name; +use crate::utils::{ + find_pegin_instance_process_data, gen_instance_parameters_local, min_required_operator, + obsolete_instance_graphs_except, +}; +use anyhow::{Context, anyhow}; +use async_trait::async_trait; +use bitvm_lib::actors::Actor; +use bitvm_lib::timelocks::default_connector_z_timelock_blocks; +use bitvm_lib::transactions::base::BaseTransaction; +use client::btc_chain::BTCClient; +use engine::{ + AggregateHandler, AggregateRef, ApplyContext, ApplyError, ApplyOutcome, EngineInboxEvent, + EngineOutboxCloseTarget, EventSource, JOB_RESULT_KIND, JobPool, JobResultEvent, + JobResultOutcome, JobSpec, NewEvent, OutboxMessage, WaitingFor, +}; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use std::sync::Arc; +use store::localdb::{InstanceUpdate, LocalDB}; +use store::{Instance, InstanceBridgeInStatus, SerializableTxid}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +pub const INSTANCE_KIND: &str = "instance"; + +pub const EVENT_BTC_SNAPSHOT: &str = "btc_snapshot"; +pub const EVENT_ANSWER_WINDOW: &str = "answer_window"; +pub const EVENT_GRAPH_ENDORSED: &str = "graph_endorsed"; +pub const EVENT_L2_MINTED: &str = "l2_minted"; +/// A Goat `BridgeInRequest` this node indexed while its response window is open. +pub const EVENT_PEGIN_REQUEST: &str = "pegin_request"; + +pub const JOB_POST_PEGIN_DATA: &str = "post_pegin_data"; +pub const JOB_DELETE_INSTANCE_KEY: &str = "delete_instance_key"; +/// The committee's `answerPeginRequest` (Answer phase). +pub const JOB_ANSWER_PEGIN_REQUEST: &str = "answer_pegin_request"; +/// Peg-In-Prepare confirmed: store the parameters, operators open a setup. +pub const JOB_CONFIRM_INSTANCE: &str = "confirm_instance"; +/// The committee's PeginConfirm round start on entering Presigned. +pub const JOB_START_PEGIN_CONFIRM: &str = "start_pegin_confirm"; +pub const JOB_INGEST_PEGIN_CONFIRM_NONCE: &str = "ingest_pegin_confirm_nonce"; +pub const JOB_PEGIN_CONFIRM_VOTE: &str = "pegin_confirm_vote"; +pub const JOB_INGEST_PEGIN_CONFIRM_VOTE: &str = "ingest_pegin_confirm_vote"; +pub const JOB_PEGIN_CONFIRM_SIGN: &str = "pegin_confirm_sign"; +pub const JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG: &str = "ingest_pegin_confirm_partial_sig"; +/// The relayer's aggregation and broadcast of the Peg-In-Confirm. +pub const JOB_BROADCAST_PEGIN_CONFIRM: &str = "broadcast_pegin_confirm"; + +/// The jobs that return an [`InstancePlan`]. +pub const INSTANCE_JOBS: &[&str] = &[ + JOB_ANSWER_PEGIN_REQUEST, + JOB_CONFIRM_INSTANCE, + JOB_START_PEGIN_CONFIRM, + JOB_INGEST_PEGIN_CONFIRM_NONCE, + JOB_PEGIN_CONFIRM_VOTE, + JOB_INGEST_PEGIN_CONFIRM_VOTE, + JOB_PEGIN_CONFIRM_SIGN, + JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG, + JOB_BROADCAST_PEGIN_CONFIRM, +]; + +pub const PHASE_ANSWER: &str = "Answer"; +pub const PHASE_AWAIT_PREPARE: &str = "AwaitPrepare"; +pub const PHASE_PRESIGN: &str = "Presign"; +pub const PHASE_CONFIRM: &str = "Confirm"; +pub const PHASE_POST: &str = "Post"; +pub const PHASE_DONE: &str = "Done"; +pub const PHASE_FAILED: &str = "Failed"; +/// Phase of the key deletion jobs; they belong to no protocol phase. +pub const PHASE_KEY: &str = "Key"; + +pub fn instance_ref(instance_id: Uuid) -> AggregateRef { + AggregateRef::new(INSTANCE_KIND, instance_id.to_string()) +} + +/// Phase of the instance state machine for a status. +pub fn phase_of(status: InstanceBridgeInStatus) -> &'static str { + use InstanceBridgeInStatus::*; + match status { + UserInited => PHASE_ANSWER, + CommitteesAnswered => PHASE_AWAIT_PREPARE, + UserBroadcastPeginPrepare => PHASE_PRESIGN, + Presigned => PHASE_CONFIRM, + RelayerL1Broadcasted => PHASE_POST, + RelayerL2Minted => PHASE_DONE, + Timeout | UserCanceled | NoEnoughCommitteesAnswered | UserDiscarded => PHASE_FAILED, + } +} + +/// The job phase an action runs in; jobs are cancelled by phase. +pub fn job_phase(action: &str) -> &'static str { + match action { + JOB_ANSWER_PEGIN_REQUEST => PHASE_ANSWER, + JOB_CONFIRM_INSTANCE => PHASE_PRESIGN, + JOB_POST_PEGIN_DATA => PHASE_POST, + JOB_DELETE_INSTANCE_KEY => PHASE_KEY, + _ => PHASE_CONFIRM, + } +} + +/// The PeginConfirm round is over: nothing of the Confirm phase is sent or started any more. +pub fn confirm_round_over(status: InstanceBridgeInStatus) -> bool { + status.is_failed() + || matches!( + status, + InstanceBridgeInStatus::RelayerL1Broadcasted | InstanceBridgeInStatus::RelayerL2Minted + ) +} + +// ------------------------------------------------------------------- routing + +/// Message kinds the aggregate takes from the network, and which job ingests +/// them. Only committee nodes take part in the PeginConfirm round. +pub fn route_action(kind: &str, actor: &Actor) -> Option<&'static str> { + if *actor != Actor::Committee { + return None; + } + Some(match kind { + "PeginConfirmNonce" => JOB_INGEST_PEGIN_CONFIRM_NONCE, + "PeginConfirmNonceConsensus" => JOB_INGEST_PEGIN_CONFIRM_VOTE, + "PeginConfirmPartialSig" => JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG, + _ => return None, + }) +} + +/// Whether this role takes `kind` through the engine at all. +pub fn engine_handles(kind: &str, actor: &Actor) -> bool { + route_action(kind, actor).is_some() +} + +// ------------------------------------------------------------------ deadlines + +/// Block-height deadlines of a peg-in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PeginDeadlines { + /// `T0`: `connector_z` of the network. + pub t0_blocks: i64, + /// `B`: the abandon buffer after `T0`. + pub buffer_blocks: i64, + /// Success-side key deletion depth after Peg-In-Confirm confirmation. + pub key_delete_blocks: i64, +} + +impl PeginDeadlines { + pub fn from_env() -> Self { + Self { + t0_blocks: i64::from(default_connector_z_timelock_blocks(get_network())), + buffer_blocks: get_pegin_abandon_buffer_blocks(), + key_delete_blocks: get_committee_instance_key_delete_timelock_blocks(), + } + } + + /// `H_p + T0`: from this height the user may broadcast Peg-In-Cancel and + /// no node produces a new protocol signature for the instance. `None` + /// while the Prepare height is unknown. + pub fn cancel_from_height(&self, prepare_height: i64) -> Option { + (prepare_height > 0).then(|| prepare_height + self.t0_blocks) + } + + /// `H_p + T0 + B`: from this height the peg-in is abandoned unless the + /// Peg-In-Confirm is confirmed or visible in the mempool. + pub fn abandon_height(&self, prepare_height: i64) -> Option { + self.cancel_from_height(prepare_height).map(|height| height + self.buffer_blocks) + } + + /// The T0 gate: signing stays open while the Prepare height is unknown or + /// `h < H_p + T0`. + pub fn signing_open(&self, prepare_height: i64, height: i64) -> bool { + self.cancel_from_height(prepare_height).is_none_or(|t0| height < t0) + } +} + +// ---------------------------------------------------------------------- facts + +/// What this node's Bitcoin data source says about one transaction. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TxFact { + pub confirmed: bool, + pub block_height: Option, + /// Returned by the data source but unconfirmed. Per-node: another node + /// may not see it. + pub in_mempool: bool, +} + +impl TxFact { + pub const UNKNOWN: TxFact = TxFact { confirmed: false, block_height: None, in_mempool: false }; + + pub fn confirmed_at(height: i64) -> Self { + Self { confirmed: true, block_height: Some(height), in_mempool: false } + } + + pub fn mempool() -> Self { + Self { confirmed: false, block_height: None, in_mempool: true } + } + + /// Neither confirmed nor in this node's mempool. + pub fn invisible(&self) -> bool { + !self.confirmed && !self.in_mempool + } +} + +/// Bitcoin facts about an instance at tip height `height`. A `None` fact was +/// not queried for the status the scan saw; the apply never decides on it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BtcSnapshot { + pub height: i64, + pub status_at_scan: String, + pub prepare: Option, + pub confirm: Option, + pub cancel: Option, + /// Whether every input of Peg-In-Prepare is unspent or spent by Prepare itself. + pub inputs_available: Option, +} + +impl BtcSnapshot { + /// The event id carries the height and the status the scan saw, so a scan + /// after a status change at the same height is a new event. + pub fn event(&self, instance_id: Uuid) -> anyhow::Result { + Ok(NewEvent::new( + format!("instance:{instance_id}:btc:{}:{}", self.height, self.status_at_scan), + instance_ref(instance_id), + EVENT_BTC_SNAPSHOT, + EventSource::Height, + serde_json::to_vec(self)?, + )) + } +} + +/// Committee answers on Goat once the response window closed. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AnswerWindow { + pub goat_height: i64, + pub window_blocks: i64, + pub quorum: u64, + /// committee address -> instance pubkey, as stored on Goat. + pub answers: IndexMap>, +} + +impl AnswerWindow { + pub fn event(&self, instance_id: Uuid) -> anyhow::Result { + Ok(NewEvent::new( + format!("instance:{instance_id}:answers:{}", self.goat_height), + instance_ref(instance_id), + EVENT_ANSWER_WINDOW, + EventSource::Chain, + serde_json::to_vec(self)?, + )) + } +} + +/// The Goat `BridgeInRequest` the event watcher indexed. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PeginRequested { + pub tx_hash: String, +} + +/// One event per request transaction: a re-indexed request is a new event. +pub fn pegin_request_event( + instance_id: Uuid, + request: &PeginRequested, +) -> anyhow::Result { + Ok(NewEvent::new( + format!("instance:{instance_id}:pegin-request:{}", request.tx_hash), + instance_ref(instance_id), + EVENT_PEGIN_REQUEST, + EventSource::Chain, + serde_json::to_vec(request)?, + )) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphEndorsed { + pub graph_id: Uuid, +} + +/// A graph of the instance is finalized and endorsed locally. +pub fn graph_endorsed_event(instance_id: Uuid, graph_id: Uuid) -> anyhow::Result { + Ok(NewEvent::new( + format!("instance:{instance_id}:graph-endorsed:{graph_id}"), + instance_ref(instance_id), + EVENT_GRAPH_ENDORSED, + EventSource::Local, + serde_json::to_vec(&GraphEndorsed { graph_id })?, + )) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct L2Minted { + pub tx_hash: String, +} + +/// The Goat `BridgeIn` event. +pub fn l2_minted_event(instance_id: Uuid, tx_hash: &str) -> anyhow::Result { + Ok(NewEvent::new( + format!("instance:{instance_id}:l2-minted:{tx_hash}"), + instance_ref(instance_id), + EVENT_L2_MINTED, + EventSource::Chain, + serde_json::to_vec(&L2Minted { tx_hash: tx_hash.to_string() })?, + )) +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct KeyDeleteParams { + pub reason: String, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct KeyDeleteResult { + pub deleted: bool, + pub detail: String, +} + +// ---------------------------------------------------------------------- plans + +/// The operator opens a setup session for its graph of the instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StartGraph { + /// The id of the session, or of the existing session when one is open. + pub graph_id: Uuid, + pub operator_pubkey: String, +} + +/// What an instance job decided; the apply commits it with the row. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstancePlan { + pub messages: Vec, + pub next: Vec, + pub start_graph: Option, + pub wake_refs: Vec, + pub note: Option, +} + +impl Plan for InstancePlan { + fn noted(note: impl Into) -> Self { + Self { note: Some(note.into()), ..Default::default() } + } +} + +impl InstancePlan { + pub fn encode(&self) -> anyhow::Result> { + Ok(bincode::serialize(self)?) + } + + pub fn decode(bytes: &[u8]) -> anyhow::Result { + Ok(bincode::deserialize(bytes)?) + } + + pub fn send(&mut self, message: PlannedMessage) -> &mut Self { + self.messages.push(message); + self + } + + pub fn then(&mut self, action: &str, params: Vec, delay_secs: Option) -> &mut Self { + self.next.push(NextJob { action: action.to_string(), params, delay_secs }); + self + } +} + +// ------------------------------------------------------------------ decisions + +/// Conditions that need a human: the state machine does not move on them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InstanceAlert { + /// Peg-In-Confirm confirmed after the instance was abandoned. + PeginConfirmAfterTimeout, + /// Goat minted for an instance this node holds in a failure state. + L2MintedAfterFailure, + /// The gateway rejected `postPeginData` permanently. + PostPeginDataRejected, + /// The committee key envelope could not be deleted. + KeyDeleteFailed, + /// A step of the committee's PeginConfirm round failed for good. + PeginConfirmRoundFailed, + /// The committee could not answer the peg-in request. + AnswerPeginRequestFailed, + /// Goat refused the data of a confirmed peg-in for good: no instance + /// parameters stored, no graph started by this operator. + ConfirmInstanceRejected, +} + +impl InstanceAlert { + pub const fn reason(self) -> &'static str { + match self { + InstanceAlert::PeginConfirmAfterTimeout => "pegin_confirm_after_timeout", + InstanceAlert::L2MintedAfterFailure => "l2_minted_after_failure", + InstanceAlert::PostPeginDataRejected => "post_pegin_data_rejected", + InstanceAlert::KeyDeleteFailed => "key_delete_failed", + InstanceAlert::PeginConfirmRoundFailed => "pegin_confirm_round_failed", + InstanceAlert::AnswerPeginRequestFailed => "answer_pegin_request_failed", + InstanceAlert::ConfirmInstanceRejected => "confirm_instance_rejected", + } + } +} + +/// What one event changes. Pure: computed from the current row and the facts. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Transition { + pub next: Option, + /// Peg-In-Prepare confirmation height, when the snapshot established it. + pub prepare_height: Option, + pub committees_answers: Option>>, + pub post_pegin_txhash: Option, + /// Success-side key deletion is due (Peg-In-Confirm confirmed + depth). + pub delete_key: bool, + pub alerts: Vec, +} + +impl Transition { + fn to(status: InstanceBridgeInStatus) -> Self { + Self { next: Some(status), ..Default::default() } + } + + fn alert(alert: InstanceAlert) -> Self { + Self { alerts: vec![alert], ..Default::default() } + } +} + +/// Decide from a Bitcoin snapshot. A scan may cross several states: Prepare confirmation, then +/// Confirm or Cancel confirmation, are evaluated in order on the same facts. +pub fn decide_btc_snapshot( + status: InstanceBridgeInStatus, + prepare_height: i64, + key_deleted: bool, + snapshot: &BtcSnapshot, + deadlines: &PeginDeadlines, +) -> Transition { + use InstanceBridgeInStatus::*; + let confirm = snapshot.confirm; + let cancel = snapshot.cancel; + let prepare = snapshot.prepare; + let confirm_confirmed = confirm.is_some_and(|f| f.confirmed); + let cancel_confirmed = cancel.is_some_and(|f| f.confirmed); + + if status.is_failed() { + // Failure states are terminal except Timeout -> UserCanceled. A late Peg-In-Confirm + // is reported, never acted on. + if status == Timeout { + if cancel_confirmed { + return Transition::to(UserCanceled); + } + if confirm_confirmed { + return Transition::alert(InstanceAlert::PeginConfirmAfterTimeout); + } + } + return Transition::default(); + } + + let mut transition = Transition::default(); + let mut current = status; + let mut prepare_height = prepare_height; + + // 1. Answer / AwaitPrepare: Prepare confirmation or a conflicting spend. + // A conflicting spend is only judged once the Prepare txid is known + // (CommitteesAnswered): before that the user's own Prepare, seen by a + // node whose answer-window scan lags, would pass for a double spend. + if matches!(current, UserInited | CommitteesAnswered) { + if let Some(prepare) = prepare.filter(|f| f.confirmed) { + current = UserBroadcastPeginPrepare; + prepare_height = prepare.block_height.unwrap_or(prepare_height); + transition.prepare_height = Some(prepare_height); + } else if current == CommitteesAnswered && snapshot.inputs_available == Some(false) { + transition.next = Some(UserDiscarded); + return transition; + } + } + + // 2. Presign / Confirm: chain facts first, the deadline last. + if matches!(current, UserBroadcastPeginPrepare | Presigned) { + if confirm_confirmed { + current = RelayerL1Broadcasted; + } else if cancel_confirmed { + current = UserCanceled; + } else if current == UserBroadcastPeginPrepare + && prepare.is_some_and(|f| !f.confirmed) + && snapshot.inputs_available == Some(false) + { + // Prepare was reorged out and an input is spent elsewhere. + current = UserDiscarded; + } else if let Some(confirm) = confirm + && deadlines.abandon_height(prepare_height).is_some_and(|h| snapshot.height >= h) + && !confirm.in_mempool + { + current = Timeout; + } + } + + // 3. Post / Done: success-side key deletion depth. + if matches!(current, RelayerL1Broadcasted | RelayerL2Minted) + && !key_deleted + && let Some(confirm) = confirm.filter(|f| f.confirmed) + && let Some(confirmed_at) = confirm.block_height + && confirmed_at + deadlines.key_delete_blocks <= snapshot.height + { + transition.delete_key = true; + } + + if current != status { + transition.next = Some(current); + } + transition +} + +/// Decide the Answer phase once the response window closed. +pub fn decide_answer_window( + status: InstanceBridgeInStatus, + request_height: i64, + known_answers: &IndexMap>, + window: &AnswerWindow, +) -> Transition { + if status != InstanceBridgeInStatus::UserInited + || window.goat_height <= request_height + window.window_blocks + { + return Transition::default(); + } + let mut answers = known_answers.clone(); + for (address, pubkey) in &window.answers { + answers.insert(address.clone(), pubkey.clone()); + } + let next = if answers.len() as u64 >= window.quorum { + InstanceBridgeInStatus::CommitteesAnswered + } else { + InstanceBridgeInStatus::NoEnoughCommitteesAnswered + }; + Transition { next: Some(next), committees_answers: Some(answers), ..Default::default() } +} + +/// The Presigned threshold: finalized and endorsed graphs >= `min_required_operator()`. +pub fn decide_graph_endorsed(status: InstanceBridgeInStatus, endorsed_graphs: usize) -> Transition { + if status == InstanceBridgeInStatus::UserBroadcastPeginPrepare + && endorsed_graphs >= min_required_operator() + { + Transition::to(InstanceBridgeInStatus::Presigned) + } else { + Transition::default() + } +} + +/// The Goat `BridgeIn` event: success from any live state; an alert for failed instances. +pub fn decide_l2_minted(status: InstanceBridgeInStatus, tx_hash: &str) -> Transition { + if status.is_failed() { + return Transition::alert(InstanceAlert::L2MintedAfterFailure); + } + Transition { + next: (status != InstanceBridgeInStatus::RelayerL2Minted) + .then_some(InstanceBridgeInStatus::RelayerL2Minted), + post_pegin_txhash: Some(tx_hash.to_string()), + ..Default::default() + } +} + +/// The relayer re-broadcasts a Peg-In-Confirm that is neither confirmed nor in its mempool +/// while the instance stays Presigned. +pub fn rebroadcast_due( + status: InstanceBridgeInStatus, + transition: &Transition, + snapshot: &BtcSnapshot, +) -> bool { + status == InstanceBridgeInStatus::Presigned + && transition.next.is_none() + && snapshot.confirm.is_some_and(|confirm| confirm.invisible()) +} + +// --------------------------------------------------------------- signing gate + +/// Signing and broadcast eligibility for an instance. +/// Closed for failed instances or when the tip reaches the Prepare height plus +/// its cancellation delay. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SigningGate { + Open, + Closed(String), +} + +impl SigningGate { + pub fn is_open(&self) -> bool { + matches!(self, SigningGate::Open) + } +} + +pub fn signing_gate_for( + status: InstanceBridgeInStatus, + prepare_height: i64, + height: i64, + deadlines: &PeginDeadlines, +) -> SigningGate { + if status.is_failed() { + return SigningGate::Closed(format!("instance is {status}")); + } + match deadlines.cancel_from_height(prepare_height) { + Some(t0) if height >= t0 => { + SigningGate::Closed(format!("height {height} reached T0 at {t0}")) + } + _ => SigningGate::Open, + } +} + +/// Evaluate the gate against the instance row and the current tip height. +pub async fn signing_gate( + local_db: &LocalDB, + btc_client: &BTCClient, + instance_id: Uuid, +) -> anyhow::Result { + let instance = local_db.acquire().await?.find_instance(&instance_id).await?; + let Some(instance) = instance else { + return Ok(SigningGate::Closed("instance not found".to_string())); + }; + let Ok(status) = InstanceBridgeInStatus::from_str(&instance.status) else { + return Ok(SigningGate::Closed(format!("unknown instance status {}", instance.status))); + }; + if status.is_failed() { + return Ok(SigningGate::Closed(format!("instance is {status}"))); + } + if instance.btc_height <= 0 { + return Ok(SigningGate::Open); + } + let height = i64::from(btc_client.get_height().await?); + Ok(signing_gate_for(status, instance.btc_height, height, &PeginDeadlines::from_env())) +} + +// -------------------------------------------------------------------- handler + +pub struct InstanceAggregate { + actor: Actor, + deadlines: PeginDeadlines, + relayer: bool, + key_delete_enabled: bool, + local_peer: String, + membership: Arc, +} + +impl InstanceAggregate { + pub fn from_env(actor: Actor) -> Self { + Self::new( + actor, + PeginDeadlines::from_env(), + is_relayer(), + is_enable_committee_instance_key_delete(), + get_peer_id(), + ) + } + + pub fn new( + actor: Actor, + deadlines: PeginDeadlines, + relayer: bool, + key_delete_enabled: bool, + local_peer: String, + ) -> Self { + Self { + actor, + deadlines, + relayer, + key_delete_enabled, + local_peer, + membership: Arc::new(EnvelopeProbe), + } + } + + pub fn with_membership(mut self, membership: Arc) -> Self { + self.membership = membership; + self + } + + fn job(&self, ctx: &mut ApplyContext<'_>, action: &str, params: Vec) { + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + job_phase(action), + action, + JobPool::Network, + params, + )); + } + + async fn commit( + &self, + ctx: &mut ApplyContext<'_>, + instance: &Instance, + status: InstanceBridgeInStatus, + transition: Transition, + ) -> anyhow::Result<()> { + let instance_id = instance.instance_id; + let mut final_status = status; + if let Some(next) = transition.next.filter(|next| *next != status) { + let mut update = InstanceUpdate::new_with_instance_id(instance_id) + .with_status(next.to_string()) + .with_only_if_status_in(vec![status.to_string()]); + if let Some(height) = transition.prepare_height { + update = update.with_btc_height(height); + } + if let Some(answers) = transition.committees_answers.clone() { + update = update.with_committees_answers(answers.clone()); + if next == InstanceBridgeInStatus::CommitteesAnswered { + let mut seeded = instance.clone(); + seeded.committees_answers = answers; + match pegin_txids(&seeded) { + Ok((deposit, confirm, cancel)) => { + update = update + .with_btc_txid(deposit) + .with_pegin_confirm_txid(confirm) + .with_pegin_cancel_txid(cancel); + } + Err(error) => warn!( + event = "instance_transition", + instance_id = %instance_id, + error = %error, + "could not derive the peg-in txids from the committee answers" + ), + } + } + } + if let Some(tx_hash) = transition.post_pegin_txhash.clone() { + update = update.with_post_pegin(tx_hash); + } + if !ctx.storage.update_instance(&update).await? { + anyhow::bail!("instance {instance_id} left status {status} during the apply"); + } + info!( + event = "instance_transition", + instance_id = %instance_id, + from = %status, + to = %next, + height = ?transition.prepare_height, + "instance status updated" + ); + final_status = next; + self.on_entered(ctx, instance, status, next).await?; + } else if let Some(tx_hash) = transition.post_pegin_txhash.clone() { + ctx.storage + .update_instance( + &InstanceUpdate::new_with_instance_id(instance_id).with_post_pegin(tx_hash), + ) + .await?; + } + if transition.prepare_height.is_some() { + // Peg-In-Prepare confirmed: every role stores the instance parameters, operators + // start graphs. + self.job(ctx, JOB_CONFIRM_INSTANCE, Vec::new()); + } + if transition.delete_key { + self.request_key_delete(ctx, instance, "success"); + } + for alert in transition.alerts { + raise_alert(instance_id, alert, ""); + } + ctx.effects + .set_status(final_status.to_string()) + .set_phase(phase_of(final_status).to_string()); + Ok(()) + } + + async fn on_entered( + &self, + ctx: &mut ApplyContext<'_>, + instance: &Instance, + previous: InstanceBridgeInStatus, + next: InstanceBridgeInStatus, + ) -> anyhow::Result<()> { + use InstanceBridgeInStatus::*; + match next { + CommitteesAnswered => { + ctx.effects.cancel_phase_jobs(PHASE_ANSWER); + } + Presigned => { + // The committee's PeginConfirm round starts from the stored values; the outbox + // re-publishes them until the round closes. + if self.actor == Actor::Committee { + self.job(ctx, JOB_START_PEGIN_CONFIRM, Vec::new()); + } + } + RelayerL1Broadcasted => { + self.close_confirm_round(ctx, instance.instance_id, "Peg-In-Confirm confirmed"); + if self.relayer { + self.job(ctx, JOB_POST_PEGIN_DATA, Vec::new()); + } + } + RelayerL2Minted => { + self.close_confirm_round(ctx, instance.instance_id, "minted on Goat"); + ctx.effects.cancel_phase_jobs(PHASE_CONFIRM); + ctx.effects.cancel_phase_jobs(PHASE_POST); + } + failed if failed.is_failed() => { + self.enter_failure(ctx, instance, previous, failed).await?; + } + _ => {} + } + Ok(()) + } + + /// Close the instance's Confirm-phase outbox deliveries. + fn close_confirm_round(&self, ctx: &mut ApplyContext<'_>, instance_id: Uuid, reason: &str) { + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Phase { + aggregate_kind: INSTANCE_KIND.to_string(), + aggregate_id: instance_id.to_string(), + phase_ref: PHASE_CONFIRM.to_string(), + }, + reason, + ); + } + + /// Close instance deliveries and retire its graphs on failure. + /// Delete the committee key unless the status is Timeout. A later UserCanceled + /// transition repeats cleanup and requests key deletion. + async fn enter_failure( + &self, + ctx: &mut ApplyContext<'_>, + instance: &Instance, + previous: InstanceBridgeInStatus, + failed: InstanceBridgeInStatus, + ) -> anyhow::Result<()> { + let instance_id = instance.instance_id; + let obsoleted = obsolete_instance_graphs_except(ctx.storage, instance_id, None).await?; + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Aggregate { + aggregate_kind: INSTANCE_KIND.to_string(), + aggregate_id: instance_id.to_string(), + }, + format!("instance {failed}"), + ); + for phase in [PHASE_ANSWER, PHASE_PRESIGN, PHASE_CONFIRM, PHASE_POST] { + ctx.effects.cancel_phase_jobs(phase); + } + // Close graph deliveries and cancel setup/post jobs. Keep kickoff and settlement + // jobs so chain monitoring and challenges can continue. + for graph in ctx.storage.get_graphs_by_instance_id(&instance_id).await? { + let graph_ref = crate::aggregates::graph::graph_ref(graph.graph_id); + ctx.effects.close_outbox( + EngineOutboxCloseTarget::Aggregate { + aggregate_kind: graph_ref.kind.clone(), + aggregate_id: graph_ref.id.clone(), + }, + format!("instance {failed}"), + ); + for phase in [crate::aggregates::graph::JOB_PHASE, crate::aggregates::graph::PHASE_POST] + { + ctx.effects.cancel_phase_jobs_for_aggregate(&graph_ref, phase); + } + ctx.effects.wake(crate::aggregates::graph::slot_ref( + &graph.operator_pubkey, + graph.kickoff_index as u64, + )); + } + // A setup session that never stored a definition (the operator's own + // graph in Setup) is released with the slot it was taking. + for session in ctx.storage.list_pending_graph_inits_by_instance(&instance_id).await? { + let next_slot = crate::aggregates::graph::release_setup_session( + ctx, + &session, + &format!("instance {failed}"), + ) + .await?; + info!( + event = "instance_failed", + instance_id = %instance_id, + graph_id = %session.graph_id, + next_slot, + "dropped the setup session of a failed instance" + ); + } + warn!( + event = "instance_failed", + instance_id = %instance_id, + from = %previous, + to = %failed, + obsoleted_graphs = ?obsoleted, + "instance entered a failure state" + ); + if failed.deletes_key_on_entry() { + self.request_key_delete(ctx, instance, "failure"); + } else { + info!( + event = "instance_key_delete", + instance_id = %instance_id, + outcome = "kept", + reason = %failed, + "committee instance key kept for manual handling" + ); + } + Ok(()) + } + + /// Queue the deletion of the committee instance key. Only committee nodes + /// hold an envelope; with deletion disabled the intent is only recorded. + fn request_key_delete(&self, ctx: &mut ApplyContext<'_>, instance: &Instance, reason: &str) { + if self.actor != Actor::Committee || instance.key_deleted { + return; + } + if !self.key_delete_enabled { + info!( + event = "instance_key_delete", + instance_id = %instance.instance_id, + outcome = "disabled", + reason, + "committee instance key is due for deletion; deletion is disabled" + ); + return; + } + let params = serde_json::to_vec(&KeyDeleteParams { reason: reason.to_string() }) + .expect("KeyDeleteParams serializes"); + ctx.effects.enqueue_job(JobSpec::new( + &ctx.aggregate, + PHASE_KEY, + JOB_DELETE_INSTANCE_KEY, + JobPool::Network, + params, + )); + } + + /// A `pegin_request` event: the row was stored by the scan; a committee + /// node answers on Goat while the request is still in its Answer phase. + fn apply_pegin_request( + &self, + ctx: &mut ApplyContext<'_>, + status: InstanceBridgeInStatus, + payload: &[u8], + ) -> ApplyOutcome { + if status != InstanceBridgeInStatus::UserInited { + return ApplyOutcome::Applied; + } + if self.actor == Actor::Committee { + self.job(ctx, JOB_ANSWER_PEGIN_REQUEST, payload.to_vec()); + } + ApplyOutcome::Applied + } + + /// A committee member's PeginConfirm round message: ignored from this + /// node itself, for instances this node holds no key of, and once the + /// round is over; otherwise handed to its ingest job. A partial + /// signature is still taken after the Peg-In-Confirm confirmed: it + /// carries the endorsement the relayer's post needs. + fn apply_round_message( + &self, + ctx: &mut ApplyContext<'_>, + instance_id: Uuid, + status: InstanceBridgeInStatus, + kind: &str, + payload: &[u8], + ) -> ApplyOutcome { + let inbound = match InboundMessage::decode(payload) { + Ok(inbound) => inbound, + Err(error) => { + return ApplyOutcome::Rejected(format!("malformed {kind} event: {error}")); + } + }; + if inbound.author == self.local_peer { + return ApplyOutcome::Applied; + } + let Some(action) = route_action(kind, &self.actor) else { + return ApplyOutcome::Rejected(format!("{kind} is not handled by {}", self.actor)); + }; + // A relayer collects every round whether or not it signs in it: it + // aggregates and broadcasts for committees it is not a member of. + if !self.relayer && !self.membership.holds_instance_key(instance_id) { + debug!(event = "instance_apply", instance_id = %instance_id, kind, "not in this instance's committee; message ignored"); + return ApplyOutcome::Applied; + } + // The round's parameters derive from the committee answers; a message + // that arrives before the window closed here (a node catching up) + // waits for them instead of running a job that cannot aggregate. + if status == InstanceBridgeInStatus::UserInited { + return ApplyOutcome::Waiting(WaitingFor::on( + &ctx.aggregate, + "committee answers not recorded yet", + )); + } + let ingest_closed = status.is_failed() + || status == InstanceBridgeInStatus::RelayerL2Minted + || (status == InstanceBridgeInStatus::RelayerL1Broadcasted + && kind != "PeginConfirmPartialSig"); + if ingest_closed { + debug!(event = "instance_apply", instance_id = %instance_id, kind, status = %status, "PeginConfirm round is over; message ignored"); + return ApplyOutcome::Applied; + } + ctx.effects.enqueue_job( + JobSpec::new(&ctx.aggregate, PHASE_CONFIRM, action, JobPool::Network, payload.to_vec()) + .with_deadline(ctx.now + PARKED_MESSAGE_TTL_SECS), + ); + ApplyOutcome::Applied + } + + async fn apply_plan( + &self, + ctx: &mut ApplyContext<'_>, + instance: &Instance, + status: InstanceBridgeInStatus, + plan: InstancePlan, + ) -> anyhow::Result<()> { + let instance_id = instance.instance_id; + let round_over = confirm_round_over(status); + for message in plan.messages { + if round_over && message.phase_ref == PHASE_CONFIRM { + continue; + } + ctx.effects.send(OutboxMessage { + outbox_id: message.outbox_id, + phase_ref: message.phase_ref, + kind: message.kind, + topic: get_topic_name(&message.actor), + payload: message.payload, + ack_set: message.ack_set, + }); + } + for NextJob { action, params, delay_secs } in plan.next { + let phase = job_phase(&action); + if round_over && phase == PHASE_CONFIRM { + continue; + } + let mut job = JobSpec::new(&ctx.aggregate, phase, &action, JobPool::Network, params); + if let Some(delay) = delay_secs { + job = job.with_run_after(ctx.now + delay.max(0)); + } + ctx.effects.enqueue_job(job); + } + if let Some(start) = plan.start_graph + && !status.is_failed() + { + // One session per (instance, operator): a repeat of the job + // resumes the open session instead of opening a second one. + let graph_id = match ctx + .storage + .find_pending_graph_init_by_instance_and_operator_pubkey( + &instance_id, + &start.operator_pubkey, + ) + .await? + { + Some(session) => session.graph_id, + None => { + ctx.storage + .upsert_pending_graph_init( + &instance_id, + &start.operator_pubkey, + &start.graph_id, + ) + .await?; + start.graph_id + } + }; + record_event_in( + ctx.storage, + operator_start_event(instance_id, graph_id, &start.operator_pubkey)?, + ) + .await?; + info!(event = "graph_start", instance_id = %instance_id, graph_id = %graph_id, "operator setup session opened"); + } + for wake_ref in plan.wake_refs { + ctx.effects.wake(wake_ref); + } + Ok(()) + } + + async fn apply_job_result( + &self, + ctx: &mut ApplyContext<'_>, + instance: &Instance, + status: InstanceBridgeInStatus, + result: JobResultEvent, + ) -> Result { + let instance_id = instance.instance_id; + match result.action.as_str() { + JOB_DELETE_INSTANCE_KEY => match &result.outcome { + JobResultOutcome::Done(bytes) => { + let outcome: KeyDeleteResult = + serde_json::from_slice(bytes).unwrap_or_default(); + if outcome.deleted { + ctx.storage + .update_instance( + &InstanceUpdate::new_with_instance_id(instance_id) + .with_key_deleted(true), + ) + .await?; + info!( + event = "instance_key_delete", + instance_id = %instance_id, + outcome = "deleted", + stale = result.stale, + detail = %outcome.detail, + "committee instance key envelope deleted" + ); + } else { + info!( + event = "instance_key_delete", + instance_id = %instance_id, + outcome = "recorded", + detail = %outcome.detail, + "committee instance key deletion recorded without deleting" + ); + } + } + JobResultOutcome::Failed(error) => { + raise_alert(instance_id, InstanceAlert::KeyDeleteFailed, error); + } + JobResultOutcome::Cancelled => { + debug!(event = "instance_key_delete", instance_id = %instance_id, outcome = "cancelled", "key deletion job cancelled"); + } + JobResultOutcome::Poisoned(error) => { + warn!(event = "instance_key_delete", instance_id = %instance_id, outcome = "poisoned", error = %error, "key deletion job panicked"); + } + }, + JOB_POST_PEGIN_DATA => match &result.outcome { + JobResultOutcome::Done(_) => { + info!(event = "instance_post", instance_id = %instance_id, outcome = "posted", stale = result.stale, "pegin data posted; waiting for the BridgeIn event"); + } + JobResultOutcome::Failed(error) => { + raise_alert(instance_id, InstanceAlert::PostPeginDataRejected, error); + } + JobResultOutcome::Cancelled => { + debug!(event = "instance_post", instance_id = %instance_id, outcome = "cancelled", "post job cancelled"); + } + JobResultOutcome::Poisoned(error) => { + warn!(event = "instance_post", instance_id = %instance_id, outcome = "poisoned", error = %error, "post job panicked"); + } + }, + action if INSTANCE_JOBS.contains(&action) => match &result.outcome { + JobResultOutcome::Done(bytes) => { + let plan = match InstancePlan::decode(bytes) { + Ok(plan) => plan, + Err(error) => { + return Ok(ApplyOutcome::Rejected(format!( + "malformed plan from {action}: {error}" + ))); + } + }; + if let Some(note) = &plan.note { + info!(event = "instance_job", instance_id = %instance_id, action, stale = result.stale, note = %note, "job note"); + } + self.apply_plan(ctx, instance, status, plan).await?; + } + JobResultOutcome::Failed(error) => { + let alert = match action { + JOB_ANSWER_PEGIN_REQUEST => Some(InstanceAlert::AnswerPeginRequestFailed), + JOB_CONFIRM_INSTANCE => Some(InstanceAlert::ConfirmInstanceRejected), + JOB_START_PEGIN_CONFIRM + | JOB_PEGIN_CONFIRM_VOTE + | JOB_PEGIN_CONFIRM_SIGN + | JOB_BROADCAST_PEGIN_CONFIRM => { + Some(InstanceAlert::PeginConfirmRoundFailed) + } + _ => None, + }; + match alert { + Some(alert) => raise_alert(instance_id, alert, error), + None => { + warn!(event = "instance_job", instance_id = %instance_id, action, error = %error, "job failed") + } + } + } + JobResultOutcome::Cancelled => { + debug!(event = "instance_job", instance_id = %instance_id, action, "job cancelled"); + } + JobResultOutcome::Poisoned(error) => { + warn!(event = "instance_job", instance_id = %instance_id, action, error = %error, "job panicked"); + } + }, + other => { + return Ok(ApplyOutcome::Rejected(format!("unknown instance job action {other}"))); + } + } + ctx.storage.clear_engine_job_result(&result.job_id, ctx.now).await?; + ctx.effects.set_status(status.to_string()).set_phase(phase_of(status).to_string()); + Ok(ApplyOutcome::Applied) + } + + /// Every partial signature of the committee is stored locally: the + /// relayer can aggregate. + async fn all_partial_sigs_stored( + &self, + ctx: &mut ApplyContext<'_>, + instance: &Instance, + ) -> anyhow::Result { + let committee = instance.committees_answers.len(); + if committee == 0 { + return Ok(false); + } + let process_data = + find_pegin_instance_process_data(ctx.storage, instance.instance_id).await?; + Ok(process_data.values().filter(|item| item.partial_sign.is_some()).count() >= committee) + } +} + +fn decode Deserialize<'de>>(kind: &str, payload: &[u8]) -> Result { + serde_json::from_slice(payload) + .map_err(|error| ApplyOutcome::Rejected(format!("malformed {kind} payload: {error}"))) +} + +#[async_trait] +impl AggregateHandler for InstanceAggregate { + async fn apply( + &self, + ctx: &mut ApplyContext<'_>, + event: &EngineInboxEvent, + ) -> Result { + let instance_id = Uuid::from_str(&ctx.aggregate.id).map_err(|error| { + ApplyError::invariant(anyhow!("instance aggregate id {}: {error}", ctx.aggregate.id)) + })?; + let Some(instance) = ctx.storage.find_instance(&instance_id).await? else { + return Ok(ApplyOutcome::Waiting(WaitingFor::on( + &ctx.aggregate, + "instance row not found", + ))); + }; + let status = InstanceBridgeInStatus::from_str(&instance.status).map_err(|_| { + ApplyError::invariant(anyhow!( + "instance {instance_id} has unknown status {}", + instance.status + )) + })?; + let payload = event.payload.as_deref().unwrap_or_default(); + let transition = match event.kind.as_str() { + EVENT_BTC_SNAPSHOT => { + let snapshot: BtcSnapshot = match decode(EVENT_BTC_SNAPSHOT, payload) { + Ok(snapshot) => snapshot, + Err(rejected) => return Ok(rejected), + }; + let transition = decide_btc_snapshot( + status, + instance.btc_height, + instance.key_deleted, + &snapshot, + &self.deadlines, + ); + // The relayer re-sends a Peg-In-Confirm that fell out of its + // mempool, once per block, until it is seen or the instance + // is abandoned. + if self.relayer + && rebroadcast_due(status, &transition, &snapshot) + && self.all_partial_sigs_stored(ctx, &instance).await? + { + self.job( + ctx, + JOB_BROADCAST_PEGIN_CONFIRM, + snapshot.height.to_le_bytes().to_vec(), + ); + } + transition + } + EVENT_ANSWER_WINDOW => { + let window: AnswerWindow = match decode(EVENT_ANSWER_WINDOW, payload) { + Ok(window) => window, + Err(rejected) => return Ok(rejected), + }; + decide_answer_window( + status, + instance.goat_tx_height, + &instance.committees_answers, + &window, + ) + } + EVENT_GRAPH_ENDORSED => { + if matches!( + status, + InstanceBridgeInStatus::UserInited | InstanceBridgeInStatus::CommitteesAnswered + ) { + // Keep the endorsement event pending until Prepare is confirmed, so an early + // finalized graph still counts toward the Presigned threshold. + return Ok(ApplyOutcome::Waiting(WaitingFor::on( + &ctx.aggregate, + "instance has not reached the Presign phase", + ))); + } + let endorsed = ctx + .storage + .get_pegin_graph_endorsed_len_by_instance_id(&instance_id, true) + .await + .context("count endorsed graphs")?; + decide_graph_endorsed(status, usize::try_from(endorsed).unwrap_or(0)) + } + EVENT_L2_MINTED => { + let minted: L2Minted = match decode(EVENT_L2_MINTED, payload) { + Ok(minted) => minted, + Err(rejected) => return Ok(rejected), + }; + decide_l2_minted(status, &minted.tx_hash) + } + EVENT_PEGIN_REQUEST => { + if let Err(rejected) = decode::(EVENT_PEGIN_REQUEST, payload) { + return Ok(rejected); + } + let outcome = self.apply_pegin_request(ctx, status, payload); + ctx.effects.set_status(status.to_string()).set_phase(phase_of(status).to_string()); + return Ok(outcome); + } + JOB_RESULT_KIND => { + let result = JobResultEvent::decode(payload).map_err(ApplyError::invariant)?; + return self.apply_job_result(ctx, &instance, status, result).await; + } + kind if engine_handles(kind, &Actor::Committee) => { + let outcome = self.apply_round_message(ctx, instance_id, status, kind, payload); + ctx.effects.set_status(status.to_string()).set_phase(phase_of(status).to_string()); + return Ok(outcome); + } + other => { + return Ok(ApplyOutcome::Rejected(format!("unknown instance event kind {other}"))); + } + }; + self.commit(ctx, &instance, status, transition).await?; + Ok(ApplyOutcome::Applied) + } +} + +fn raise_alert(instance_id: Uuid, alert: InstanceAlert, detail: &str) { + warn!( + event = "instance_alert", + instance_id = %instance_id, + reason = alert.reason(), + detail, + "instance needs manual handling" + ); + if let Some(metrics) = node_metrics_state() { + metrics.record_instance_alert(alert.reason()); + } +} + +/// The three peg-in txids once the committee answers are known. +pub fn pegin_txids( + instance: &Instance, +) -> anyhow::Result<(SerializableTxid, SerializableTxid, SerializableTxid)> { + let (deposit, confirm, cancel) = gen_instance_parameters_local(instance)?.build_pegin_tx()?; + Ok(( + deposit.tx().compute_txid().into(), + confirm.finalize().compute_txid().into(), + cancel.finalize().compute_txid().into(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use InstanceBridgeInStatus::*; + + const DEADLINES: PeginDeadlines = + PeginDeadlines { t0_blocks: 60, buffer_blocks: 6, key_delete_blocks: 32 }; + const PREPARE: i64 = 1_000; + + fn snapshot(height: i64) -> BtcSnapshot { + BtcSnapshot { height, status_at_scan: String::new(), ..Default::default() } + } + + fn decide(status: InstanceBridgeInStatus, snapshot: &BtcSnapshot) -> Transition { + decide_btc_snapshot(status, PREPARE, false, snapshot, &DEADLINES) + } + + #[test] + fn deadlines_follow_the_prepare_height() { + assert_eq!(DEADLINES.cancel_from_height(0), None); + assert_eq!(DEADLINES.abandon_height(0), None); + assert_eq!(DEADLINES.cancel_from_height(PREPARE), Some(1_060)); + assert_eq!(DEADLINES.abandon_height(PREPARE), Some(1_066)); + assert!(DEADLINES.signing_open(0, 5_000)); + assert!(DEADLINES.signing_open(PREPARE, 1_059)); + assert!(!DEADLINES.signing_open(PREPARE, 1_060)); + } + + #[test] + fn inputs_spent_discards_before_prepare() { + let mut snap = snapshot(10); + snap.inputs_available = Some(false); + // Before the Prepare txid is known the spender may be the user's own + // Prepare: no decision at UserInited. + assert_eq!(decide(UserInited, &snap).next, None); + assert_eq!(decide(CommitteesAnswered, &snap).next, Some(UserDiscarded)); + snap.inputs_available = Some(true); + assert_eq!(decide(UserInited, &snap).next, None); + // Not queried: no decision. + snap.inputs_available = None; + assert_eq!(decide(CommitteesAnswered, &snap).next, None); + } + + #[test] + fn prepare_confirmation_records_the_height_and_may_cross_states() { + let mut snap = snapshot(500); + snap.prepare = Some(TxFact::confirmed_at(480)); + let t = decide(CommitteesAnswered, &snap); + assert_eq!(t.next, Some(UserBroadcastPeginPrepare)); + assert_eq!(t.prepare_height, Some(480)); + // Prepare and Confirm seen in one scan. + snap.confirm = Some(TxFact::confirmed_at(490)); + let t = decide(CommitteesAnswered, &snap); + assert_eq!(t.next, Some(RelayerL1Broadcasted)); + assert_eq!(t.prepare_height, Some(480)); + // Prepare and Cancel seen in one scan. + snap.confirm = Some(TxFact::UNKNOWN); + snap.cancel = Some(TxFact::confirmed_at(545)); + assert_eq!(decide(CommitteesAnswered, &snap).next, Some(UserCanceled)); + } + + #[test] + fn reorged_prepare_with_spent_inputs_discards() { + let mut snap = snapshot(1_010); + snap.prepare = Some(TxFact::UNKNOWN); + snap.inputs_available = Some(false); + snap.confirm = Some(TxFact::UNKNOWN); + snap.cancel = Some(TxFact::UNKNOWN); + assert_eq!(decide(UserBroadcastPeginPrepare, &snap).next, Some(UserDiscarded)); + // A Presigned instance has no Prepare fact; nothing happens. + assert_eq!(decide(Presigned, &snap).next, None); + } + + #[test] + fn confirm_confirmed_in_the_buffer_still_broadcasts() { + for height in [1_050, 1_060, 1_065, 1_066, 2_000] { + let mut snap = snapshot(height); + snap.confirm = Some(TxFact::confirmed_at(height - 1)); + snap.cancel = Some(TxFact::UNKNOWN); + assert_eq!(decide(Presigned, &snap).next, Some(RelayerL1Broadcasted), "{height}"); + assert_eq!( + decide(UserBroadcastPeginPrepare, &snap).next, + Some(RelayerL1Broadcasted), + "{height}" + ); + } + } + + #[test] + fn cancel_confirmed_fails_from_presign_confirm_and_timeout() { + let mut snap = snapshot(1_100); + snap.confirm = Some(TxFact::UNKNOWN); + snap.cancel = Some(TxFact::confirmed_at(1_099)); + assert_eq!(decide(UserBroadcastPeginPrepare, &snap).next, Some(UserCanceled)); + assert_eq!(decide(Presigned, &snap).next, Some(UserCanceled)); + assert_eq!(decide(Timeout, &snap).next, Some(UserCanceled)); + } + + #[test] + fn timeout_only_past_the_abandon_height_without_mempool_visibility() { + let mut snap = snapshot(1_065); + snap.confirm = Some(TxFact::UNKNOWN); + snap.cancel = Some(TxFact::UNKNOWN); + // T0 reached, buffer not over: keep following the chain. + assert_eq!(decide(Presigned, &snap).next, None); + snap.height = 1_066; + assert_eq!(decide(Presigned, &snap).next, Some(Timeout)); + assert_eq!(decide(UserBroadcastPeginPrepare, &snap).next, Some(Timeout)); + // Visible in the mempool: keep waiting, no upper bound. + snap.confirm = Some(TxFact::mempool()); + snap.height = 9_999; + assert_eq!(decide(Presigned, &snap).next, None); + // Confirm fact missing: never decide Timeout on it. + snap.confirm = None; + assert_eq!(decide(Presigned, &snap).next, None); + // Unknown Prepare height: no deadline at all. + snap.confirm = Some(TxFact::UNKNOWN); + assert_eq!(decide_btc_snapshot(Presigned, 0, false, &snap, &DEADLINES).next, None); + } + + #[test] + fn failure_states_are_absorbing_and_late_confirm_only_alerts() { + let mut snap = snapshot(1_200); + snap.confirm = Some(TxFact::confirmed_at(1_150)); + snap.cancel = Some(TxFact::UNKNOWN); + let t = decide(Timeout, &snap); + assert_eq!(t.next, None); + assert_eq!(t.alerts, vec![InstanceAlert::PeginConfirmAfterTimeout]); + for failed in [UserCanceled, UserDiscarded, NoEnoughCommitteesAnswered] { + let t = decide(failed, &snap); + assert_eq!(t.next, None, "{failed}"); + assert!(t.alerts.is_empty(), "{failed}"); + } + snap.cancel = Some(TxFact::confirmed_at(1_190)); + assert_eq!(decide(UserCanceled, &snap).next, None); + } + + #[test] + fn success_side_key_deletion_waits_for_the_depth() { + let mut snap = snapshot(1_131); + snap.confirm = Some(TxFact::confirmed_at(1_100)); + assert!(!decide(RelayerL1Broadcasted, &snap).delete_key); + snap.height = 1_132; + assert!(decide(RelayerL1Broadcasted, &snap).delete_key); + assert!(decide(RelayerL2Minted, &snap).delete_key); + assert!(!decide_btc_snapshot(RelayerL2Minted, PREPARE, true, &snap, &DEADLINES).delete_key); + assert_eq!(decide(RelayerL2Minted, &snap).next, None); + } + + #[test] + fn answer_window_decides_quorum_once_closed() { + let mut answers = IndexMap::new(); + answers.insert("0xa".to_string(), vec![1u8]); + let window = AnswerWindow { goat_height: 200, window_blocks: 50, quorum: 2, answers }; + // Window not over. + assert_eq!(decide_answer_window(UserInited, 150, &IndexMap::new(), &window).next, None); + assert_eq!( + decide_answer_window(UserInited, 149, &IndexMap::new(), &window).next, + Some(NoEnoughCommitteesAnswered) + ); + let mut known = IndexMap::new(); + known.insert("0xb".to_string(), vec![2u8]); + let t = decide_answer_window(UserInited, 100, &known, &window); + assert_eq!(t.next, Some(CommitteesAnswered)); + assert_eq!(t.committees_answers.as_ref().map(IndexMap::len), Some(2)); + // Only the Answer phase decides on it. + assert_eq!(decide_answer_window(CommitteesAnswered, 100, &known, &window).next, None); + assert_eq!(decide_answer_window(UserDiscarded, 100, &known, &window).next, None); + } + + #[test] + fn presigned_threshold_only_from_presign_phase() { + assert_eq!(decide_graph_endorsed(UserBroadcastPeginPrepare, 0).next, None); + assert_eq!(decide_graph_endorsed(UserBroadcastPeginPrepare, 1).next, Some(Presigned)); + assert_eq!(decide_graph_endorsed(Presigned, 3).next, None); + assert_eq!(decide_graph_endorsed(Timeout, 3).next, None); + assert_eq!(decide_graph_endorsed(CommitteesAnswered, 3).next, None); + } + + #[test] + fn l2_mint_succeeds_from_live_states_and_alerts_on_failure() { + for live in [ + UserInited, + CommitteesAnswered, + UserBroadcastPeginPrepare, + Presigned, + RelayerL1Broadcasted, + ] { + let t = decide_l2_minted(live, "0xmint"); + assert_eq!(t.next, Some(RelayerL2Minted), "{live}"); + assert_eq!(t.post_pegin_txhash.as_deref(), Some("0xmint")); + } + assert_eq!(decide_l2_minted(RelayerL2Minted, "0xmint").next, None); + for failed in InstanceBridgeInStatus::FAILED { + let t = decide_l2_minted(failed, "0xmint"); + assert_eq!(t.next, None, "{failed}"); + assert_eq!(t.alerts, vec![InstanceAlert::L2MintedAfterFailure]); + } + } + + #[test] + fn signing_gate_closes_at_t0_and_on_failure() { + assert!(signing_gate_for(Presigned, PREPARE, 1_059, &DEADLINES).is_open()); + assert!(!signing_gate_for(Presigned, PREPARE, 1_060, &DEADLINES).is_open()); + assert!(signing_gate_for(UserBroadcastPeginPrepare, 0, 9_999, &DEADLINES).is_open()); + for failed in InstanceBridgeInStatus::FAILED { + assert!(!signing_gate_for(failed, PREPARE, 10, &DEADLINES).is_open()); + } + } + + #[test] + fn phases_follow_the_phase_table() { + assert_eq!(phase_of(UserInited), PHASE_ANSWER); + assert_eq!(phase_of(CommitteesAnswered), PHASE_AWAIT_PREPARE); + assert_eq!(phase_of(UserBroadcastPeginPrepare), PHASE_PRESIGN); + assert_eq!(phase_of(Presigned), PHASE_CONFIRM); + assert_eq!(phase_of(RelayerL1Broadcasted), PHASE_POST); + assert_eq!(phase_of(RelayerL2Minted), PHASE_DONE); + for failed in InstanceBridgeInStatus::FAILED { + assert_eq!(phase_of(failed), PHASE_FAILED); + } + } + + #[test] + fn snapshot_event_ids_change_with_height_and_status() { + let id = Uuid::new_v4(); + let mut snap = snapshot(10); + snap.status_at_scan = "Presigned".into(); + let a = snap.event(id).unwrap().event_id; + snap.status_at_scan = "RelayerL1Broadcasted".into(); + let b = snap.event(id).unwrap().event_id; + snap.height = 11; + let c = snap.event(id).unwrap().event_id; + assert!(a != b && b != c && a != c); + assert_eq!(graph_endorsed_event(id, id).unwrap().aggregate, instance_ref(id)); + let request = PeginRequested { tx_hash: "0xreq".into() }; + let event = pegin_request_event(id, &request).unwrap(); + assert_eq!(event.kind, EVENT_PEGIN_REQUEST); + assert_eq!(event.event_id, format!("instance:{id}:pegin-request:0xreq")); + } + + #[test] + fn round_messages_are_routed_to_committee_ingest_jobs_only() { + assert_eq!( + route_action("PeginConfirmNonce", &Actor::Committee), + Some(JOB_INGEST_PEGIN_CONFIRM_NONCE) + ); + assert_eq!( + route_action("PeginConfirmNonceConsensus", &Actor::Committee), + Some(JOB_INGEST_PEGIN_CONFIRM_VOTE) + ); + assert_eq!( + route_action("PeginConfirmPartialSig", &Actor::Committee), + Some(JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG) + ); + for actor in [Actor::Operator, Actor::Verifier, Actor::Watchtower, Actor::All] { + assert!(!engine_handles("PeginConfirmNonce", &actor), "{actor}"); + } + assert!(!engine_handles("NonceGeneration", &Actor::Committee)); + for action in INSTANCE_JOBS { + let phase = job_phase(action); + let expected = match *action { + JOB_ANSWER_PEGIN_REQUEST => PHASE_ANSWER, + JOB_CONFIRM_INSTANCE => PHASE_PRESIGN, + _ => PHASE_CONFIRM, + }; + assert_eq!(phase, expected, "{action}"); + } + assert_eq!(job_phase(JOB_POST_PEGIN_DATA), PHASE_POST); + assert_eq!(job_phase(JOB_DELETE_INSTANCE_KEY), PHASE_KEY); + } + + #[test] + fn plans_round_trip_and_the_round_ends_with_the_confirm() { + let mut plan = InstancePlan::noted("x"); + plan.send(PlannedMessage { + outbox_id: "instance:i:PeginConfirmNonce".into(), + phase_ref: PHASE_CONFIRM.into(), + kind: "PeginConfirmNonce".into(), + actor: Actor::Committee.to_string(), + payload: vec![1, 2, 3], + ack_set: None, + }) + .then(JOB_PEGIN_CONFIRM_VOTE, Vec::new(), None); + plan.start_graph = Some(StartGraph { graph_id: Uuid::nil(), operator_pubkey: "op".into() }); + plan.wake_refs.push("graph:x".into()); + assert_eq!(InstancePlan::decode(&plan.encode().unwrap()).unwrap(), plan); + for status in [RelayerL1Broadcasted, RelayerL2Minted] { + assert!(confirm_round_over(status), "{status}"); + } + for status in InstanceBridgeInStatus::FAILED { + assert!(confirm_round_over(status), "{status}"); + } + for status in [UserInited, CommitteesAnswered, UserBroadcastPeginPrepare, Presigned] { + assert!(!confirm_round_over(status), "{status}"); + } + } + + #[test] + fn rebroadcast_is_due_only_for_an_invisible_confirm_while_presigned() { + let mut snap = snapshot(1_050); + snap.confirm = Some(TxFact::UNKNOWN); + snap.cancel = Some(TxFact::UNKNOWN); + let stays = decide(Presigned, &snap); + assert!(rebroadcast_due(Presigned, &stays, &snap)); + assert!(!rebroadcast_due(UserBroadcastPeginPrepare, &stays, &snap)); + snap.confirm = Some(TxFact::mempool()); + assert!(!rebroadcast_due(Presigned, &decide(Presigned, &snap), &snap)); + snap.confirm = Some(TxFact::confirmed_at(1_049)); + let moves = decide(Presigned, &snap); + assert_eq!(moves.next, Some(RelayerL1Broadcasted)); + assert!(!rebroadcast_due(Presigned, &moves, &snap)); + snap.confirm = None; + assert!(!rebroadcast_due(Presigned, &decide(Presigned, &snap), &snap)); + } +} diff --git a/node/src/aggregates/instance_jobs.rs b/node/src/aggregates/instance_jobs.rs new file mode 100644 index 000000000..5771cdf45 --- /dev/null +++ b/node/src/aggregates/instance_jobs.rs @@ -0,0 +1,1124 @@ +//! Jobs for committee answers, instance parameters and the PeginConfirm round. +//! +//! Jobs query external state, persist protocol material and return an +//! [`InstancePlan`] for outgoing messages and follow-up work. Goat posting and +//! committee key deletion use their own result handlers. + +use super::common::PlannedMessage; +use super::graph::graph_ref; +use super::instance::{ + INSTANCE_JOBS, INSTANCE_KIND, InstancePlan, JOB_ANSWER_PEGIN_REQUEST, + JOB_BROADCAST_PEGIN_CONFIRM, JOB_CONFIRM_INSTANCE, JOB_DELETE_INSTANCE_KEY, + JOB_INGEST_PEGIN_CONFIRM_NONCE, JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG, + JOB_INGEST_PEGIN_CONFIRM_VOTE, JOB_PEGIN_CONFIRM_SIGN, JOB_PEGIN_CONFIRM_VOTE, + JOB_POST_PEGIN_DATA, JOB_START_PEGIN_CONFIRM, KeyDeleteResult, PHASE_CONFIRM, StartGraph, + confirm_round_over, +}; +use super::jobs::{ + JobDeps, Plan, aggregate_uuid, committee_pubkeys, decode_inbound, ensure_committee_sender, + gate_closed, instance_status, local_member, local_operator, plan_outcome, signing_nonce_id, +}; +use crate::clock::current_time_secs; +use crate::env::{ + get_bitvm_key, get_operator_graph_start_cutoff_blocks, is_enable_committee_instance_key_delete, +}; +use crate::error::{FailureClass, SpecialError, classify_failure}; +use crate::messages::{ + GOATMessage, GOATMessageContent, PeginConfirmNonce, PeginConfirmNonceConsensus, + PeginConfirmPartialSig, +}; +use crate::utils::{ + avg_block_time_secs, broadcast_tx, committee_instance_keys_envelope_path, endorse_pegin, + get_committee_agg_nonce_consensus_for_instance, get_committee_endorse_sigs_for_pegin, + get_committee_partial_sig_for_instance, get_committee_partial_sigs_for_instance, + get_committee_pub_nonce_for_instance, get_committee_pub_nonces_for_instance, + get_graph_by_instance_id_and_operator_pubkey, get_instance_parameters, + has_required_presigned_graphs, is_io_not_found_error, order_committee_values, + read_instance_info_from_goat, read_pegin_request, + store_committee_agg_nonce_consensus_for_instance, store_committee_endorse_sig_for_pegin, + store_committee_partial_sig_for_instance, store_committee_pub_nonce_for_instance, + store_instance_parameters, tx_visible, verify_pegin_endorsement, +}; +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use bitcoin::key::Keypair; +use bitcoin::{PublicKey, XOnlyPublicKey}; +use bitvm_lib::actors::Actor; +use bitvm_lib::committee::{ + nonce_aggregation, pegin_confirm_nonce_consensus_hash, verify_pegin_confirm_partial_sig, +}; +use bitvm_lib::keys::CommitteeMasterKey; +use bitvm_lib::types::BitvmGcInstanceParameters; +use client::goat_chain::PeginStatus; +use engine::{EngineBuilder, JobContext, JobOutcome, JobRunner}; +use goat::transactions::pre_signed::PreSignedTransaction; +use goat::transactions::pre_signed_musig2::verify_public_nonce; +use secp256k1::{Message as SecpMessage, SECP256K1}; +use std::str::FromStr; +use std::sync::Arc; +use store::localdb::LocalDB; +use store::{InstanceBridgeInStatus, SigningAuthorization}; +use tracing::{info, warn}; +use uuid::Uuid; + +pub fn register_instance_jobs(builder: EngineBuilder, deps: Arc) -> EngineBuilder { + let runner: Arc = Arc::new(InstanceJobRunner { deps: deps.clone() }); + let mut builder = builder; + for action in INSTANCE_JOBS { + builder = builder.job_runner(INSTANCE_KIND, *action, runner.clone()); + } + builder + .job_runner( + INSTANCE_KIND, + JOB_POST_PEGIN_DATA, + Arc::new(PostPeginDataRunner { deps: deps.clone() }), + ) + .job_runner(INSTANCE_KIND, JOB_DELETE_INSTANCE_KEY, Arc::new(DeleteInstanceKeyRunner)) +} + +type Step = super::jobs::Step; + +/// Rejections that need a human: the node's own signing steps, the +/// relayer's broadcast and the reading of a confirmed peg-in on any, the +/// committee's answer only when Goat refused it (a request the user got +/// wrong, or one whose window is over, is noted). +fn fails_on_rejection(action: &str, error: &anyhow::Error) -> bool { + match action { + JOB_CONFIRM_INSTANCE + | JOB_START_PEGIN_CONFIRM + | JOB_PEGIN_CONFIRM_VOTE + | JOB_PEGIN_CONFIRM_SIGN + | JOB_BROADCAST_PEGIN_CONFIRM => true, + JOB_ANSWER_PEGIN_REQUEST => crate::error::is_contract_revert(error), + _ => false, + } +} + +struct InstanceJobRunner { + deps: Arc, +} + +#[async_trait] +impl JobRunner for InstanceJobRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let instance_id = match aggregate_uuid(&ctx, "an instance") { + Ok(instance_id) => instance_id, + Err(outcome) => return outcome, + }; + let params = ctx.job.params.clone().unwrap_or_default(); + let action = ctx.job.action.clone(); + let attempts = ctx.job.attempts; + match run_step(&self.deps, &action, instance_id, ¶ms).await { + Ok(step) => step.into_outcome(), + Err(error) => match classify_failure(&error) { + FailureClass::Permanent(reason) if fails_on_rejection(&action, &error) => { + JobOutcome::Failed(reason) + } + FailureClass::Permanent(reason) => { + warn!(event = "instance_job", instance_id = %instance_id, action = %action, reason = %reason, "rejected"); + plan_outcome(&InstancePlan::noted(format!("rejected: {reason}"))) + } + FailureClass::Transient => { + JobOutcome::Transient { error: format!("{error:#}"), retry_after_secs: None } + } + // Round steps are bounded by the instance's own deadlines, so an + // unrecognised failure keeps retrying; it is reported every so often. + FailureClass::Unknown => { + if attempts >= UNKNOWN_FAILURE_ALERT_EVERY + && attempts % UNKNOWN_FAILURE_ALERT_EVERY == 0 + { + warn!(event = "instance_job", instance_id = %instance_id, action = %action, attempts, error = %format!("{error:#}"), "step keeps failing for an unknown reason"); + self.deps.metrics_state.record_instance_alert("instance_job_stuck"); + } + JobOutcome::Transient { error: format!("{error:#}"), retry_after_secs: None } + } + }, + } + } +} + +const UNKNOWN_FAILURE_ALERT_EVERY: i64 = 20; + +async fn run_step(deps: &JobDeps, action: &str, instance_id: Uuid, params: &[u8]) -> Result { + match action { + JOB_ANSWER_PEGIN_REQUEST => answer_pegin_request(deps, instance_id).await, + JOB_CONFIRM_INSTANCE => confirm_instance(deps, instance_id).await, + JOB_START_PEGIN_CONFIRM => start_pegin_confirm(deps, instance_id).await, + JOB_INGEST_PEGIN_CONFIRM_NONCE => { + ingest_pegin_confirm_nonce(deps, instance_id, params).await + } + JOB_PEGIN_CONFIRM_VOTE => pegin_confirm_vote(deps, instance_id).await, + JOB_INGEST_PEGIN_CONFIRM_VOTE => ingest_pegin_confirm_vote(deps, instance_id, params).await, + JOB_PEGIN_CONFIRM_SIGN => pegin_confirm_sign(deps, instance_id).await, + JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG => { + ingest_pegin_confirm_partial_sig(deps, instance_id, params).await + } + JOB_BROADCAST_PEGIN_CONFIRM => broadcast_pegin_confirm(deps, instance_id, params).await, + other => bail!("unknown instance job {other}"), + } +} + +// ------------------------------------------------------------------ helpers + +fn invalid(message: impl Into) -> Result { + Err(SpecialError::InvalidPeginData(message.into()).into()) +} + +/// A PeginConfirm round message for the committee topic: one outbox row per +/// kind and instance, re-published until the Confirm phase ends. +async fn round_message(instance_id: Uuid, content: GOATMessageContent) -> Result { + let kind = content.event_type(); + let payload = GOATMessage::new(Actor::Committee, content).serialize_message().await?; + Ok(PlannedMessage { + outbox_id: format!("instance:{instance_id}:{kind}"), + phase_ref: PHASE_CONFIRM.to_string(), + kind: kind.to_string(), + actor: Actor::Committee.to_string(), + payload, + ack_set: None, + }) +} + +/// Load the committee instance key, creating the envelope on first use. A deleted key is never +/// regenerated. +async fn load_or_create_committee_instance_keypair( + local_db: &LocalDB, + committee_master_key: &CommitteeMasterKey, + instance_id: Uuid, +) -> Result { + let envelope_path = committee_instance_keys_envelope_path(instance_id); + match committee_master_key.load_instance_keypair(instance_id, &envelope_path) { + Ok(keypair) => Ok(keypair), + Err(load_err) => { + if !is_io_not_found_error(&load_err) { + return Err(load_err).with_context(|| { + format!( + "load committee instance keypair failed for {} at {} (refuse auto-overwrite for non-missing envelope)", + instance_id, + envelope_path.display() + ) + }); + } + if local_db + .acquire() + .await? + .find_instance(&instance_id) + .await? + .is_some_and(|instance| instance.key_deleted) + { + bail!( + "committee instance key for {instance_id} was deleted; refusing to regenerate it" + ); + } + info!( + "committee instance key not found for {} at {}: {}, creating new envelope", + instance_id, + envelope_path.display(), + load_err + ); + committee_master_key + .create_instance_keypair_envelope(instance_id, &envelope_path) + .with_context(|| { + format!( + "create committee instance key envelope failed for {} at {}", + instance_id, + envelope_path.display() + ) + })?; + committee_master_key.load_instance_keypair(instance_id, &envelope_path).with_context( + || { + format!( + "reload committee instance key envelope failed for {} at {}", + instance_id, + envelope_path.display() + ) + }, + ) + } + } +} + +// ------------------------------------------------------------------- answer + +/// Answer a pending request with this committee node's instance key. +/// Skip requests already answered, no longer pending, or with spent inputs. +async fn answer_pegin_request(deps: &JobDeps, instance_id: Uuid) -> Result { + if deps.actor != Actor::Committee { + return Step::ignore("only committee nodes answer"); + } + match instance_status(deps, instance_id).await? { + Some(InstanceBridgeInStatus::UserInited) => {} + other => return Step::ignore(format!("instance is {other:?}; answer window closed")), + } + let (_, _, answered_by) = + read_pegin_request(&deps.btc_client, &deps.goat_client, instance_id).await?; + // Gateway.answerPeginRequest accepts a repeat answer (it overwrites the + // stored pubkey and emits CommitteeResponse again), so every re-run must + // check what the contract already holds for this signer. + if answered_by.contains(&deps.goat_client.get_default_signer_address()) { + return Step::ignore("this committee already answered"); + } + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + let instance_keypair = load_or_create_committee_instance_keypair( + &deps.local_db, + &committee_master_key, + instance_id, + ) + .await?; + let pubkey_for_instance: PublicKey = instance_keypair.public_key().into(); + deps.goat_client.gateway_answer_pegin_request(&instance_id, &pubkey_for_instance).await?; + info!(event = "pegin_request", instance_id = %instance_id, outcome = "answered", "answerPeginRequest sent"); + Step::done(InstancePlan::noted("answered")) +} + +// ------------------------------------------------------------------ confirm + +/// Local operator policy: no new graph for an instance past `Prepare height + +/// OPERATOR_GRAPH_START_CUTOFF_BLOCKS`. +async fn graph_start_blocked(deps: &JobDeps, instance_id: Uuid) -> Result> { + let instance = deps.local_db.acquire().await?.find_instance(&instance_id).await?; + let Some(instance) = instance else { + return Ok(None); + }; + if let Ok(status) = InstanceBridgeInStatus::from_str(&instance.status) + && status.is_failed() + { + return Ok(Some(format!("instance is {status}"))); + } + if instance.btc_height > 0 { + let cutoff = instance.btc_height + get_operator_graph_start_cutoff_blocks(); + let height = i64::from(deps.btc_client.get_height().await?); + if height > cutoff { + return Ok(Some(format!("height {height} is past the graph start cutoff {cutoff}"))); + } + } + Ok(None) +} + +/// Read and store instance parameters after Prepare confirmation. +/// Operators also prepare a setup session and return its start event. +async fn confirm_instance(deps: &JobDeps, instance_id: Uuid) -> Result { + match instance_status(deps, instance_id).await? { + Some(status) if status.is_failed() => { + return Step::ignore(format!("instance is {status}")); + } + Some(_) => {} + None => return Step::ignore("instance row missing"), + } + let instance_params = read_instance_info_from_goat(&deps.goat_client, instance_id).await?; + let pegin_deposit_txid = instance_params.build_pegin_tx()?.0.tx().compute_txid(); + if !tx_visible(&deps.btc_client, &pegin_deposit_txid).await? { + // The scan saw the Prepare confirmed; the backend is lagging. + return Step::retry( + format!("pegin deposit tx {pegin_deposit_txid} not found on chain yet"), + 30, + ); + } + store_instance_parameters(&deps.local_db, &instance_params).await?; + if deps.actor != Actor::Operator { + return Step::ignore("instance parameters stored"); + } + let (_, operator_pubkey) = local_operator()?; + let mut plan = InstancePlan::default(); + if let Some(graph) = + get_graph_by_instance_id_and_operator_pubkey(&deps.local_db, instance_id, &operator_pubkey) + .await? + { + // Built already: its deliveries live in the Graph aggregate's outbox. + let graph_id = graph.parameters.graph_id; + plan.wake_refs.push(graph_ref(graph_id).wake_ref()); + plan.note = Some(format!("graph {graph_id} exists")); + return Step::done(plan); + } + let session = deps + .local_db + .acquire() + .await? + .find_pending_graph_init_by_instance_and_operator_pubkey( + &instance_id, + &operator_pubkey.to_string(), + ) + .await?; + let graph_id = match session { + Some(session) => session.graph_id, + None => { + if let Some(reason) = graph_start_blocked(deps, instance_id).await? { + info!(event = "graph_start_cutoff", instance_id = %instance_id, reason = %reason, "not starting a new graph for this instance"); + return Step::ignore(reason); + } + Uuid::new_v4() + } + }; + plan.start_graph = Some(StartGraph { graph_id, operator_pubkey: operator_pubkey.to_string() }); + Step::done(plan) +} + +// ------------------------------------------------------ pegin confirm round + +/// The nonce transcript every member must agree on before signing. +struct NonceConsensus { + parameters: BitvmGcInstanceParameters, + committee: Vec, + agg_nonce: musig2::AggNonce, + consensus_hash: [u8; 32], +} + +async fn nonce_consensus(deps: &JobDeps, instance_id: Uuid) -> Result> { + let parameters = get_instance_parameters(&deps.local_db, instance_id) + .await? + .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; + let committee = committee_pubkeys(deps, instance_id).await?; + let pub_nonces = get_committee_pub_nonces_for_instance(&deps.local_db, instance_id).await?; + if pub_nonces.len() < committee.len() { + return Ok(None); + } + let ordered = order_committee_values(&committee, pub_nonces, "pegin committee pub nonces")?; + let agg_nonce = nonce_aggregation(&ordered); + let consensus_hash = pegin_confirm_nonce_consensus_hash(¶meters, &committee, &ordered)?; + Ok(Some(NonceConsensus { parameters, committee, agg_nonce, consensus_hash })) +} + +/// Every member signed the same ordered nonce set (the consensus barrier). +async fn votes_complete( + deps: &JobDeps, + instance_id: Uuid, + consensus: &NonceConsensus, +) -> Result { + let votes = get_committee_agg_nonce_consensus_for_instance(&deps.local_db, instance_id).await?; + if votes.len() < consensus.committee.len() { + return Ok(false); + } + let ordered = order_committee_values( + &consensus.committee, + votes.into_iter().map(|(pubkey, hash, signature)| (pubkey, (hash, signature))).collect(), + "PeginConfirm agg nonce consensus", + )?; + for (committee_pubkey, (received_hash, signature)) in consensus.committee.iter().zip(ordered) { + if received_hash != consensus.consensus_hash { + return invalid(format!( + "committee PeginConfirm nonce consensus differs for instance {instance_id} and committee {committee_pubkey}" + )); + } + SECP256K1 + .verify_schnorr( + &signature, + &SecpMessage::from_digest(consensus.consensus_hash), + &XOnlyPublicKey::from(*committee_pubkey), + ) + .map_err(|error| { + SpecialError::InvalidPeginData(format!( + "invalid PeginConfirm nonce consensus signature for instance {instance_id} and committee {committee_pubkey}: {error}" + )) + })?; + } + Ok(true) +} + +async fn nonces_complete(deps: &JobDeps, instance_id: Uuid) -> Result { + let committee = committee_pubkeys(deps, instance_id).await?.len(); + let nonces = get_committee_pub_nonces_for_instance(&deps.local_db, instance_id).await?.len(); + Ok(committee > 0 && nonces >= committee) +} + +async fn partial_sigs_complete(deps: &JobDeps, instance_id: Uuid) -> Result { + let committee = committee_pubkeys(deps, instance_id).await?.len(); + let sigs = get_committee_partial_sigs_for_instance(&deps.local_db, instance_id).await?.len(); + Ok(committee > 0 && sigs >= committee) +} + +/// The relayer aggregates and broadcasts once the last partial signature is in. +fn then_broadcast(deps: &JobDeps, plan: &mut InstancePlan) { + if deps.relayer { + plan.then(JOB_BROADCAST_PEGIN_CONFIRM, Vec::new(), None); + } +} + +/// Start or resume PeginConfirm after enough graphs are endorsed. +/// Persist the deterministic nonce before scheduling its delivery. +async fn start_pegin_confirm(deps: &JobDeps, instance_id: Uuid) -> Result { + if deps.actor != Actor::Committee { + return Step::ignore("not a committee node"); + } + if let Some(status) = instance_status(deps, instance_id).await? + && confirm_round_over(status) + { + return Step::ignore(format!("instance is {status}; round over")); + } + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + if !has_required_presigned_graphs(&deps.local_db, instance_id).await? { + return Step::ignore("not enough finalized graphs for the PeginConfirm round"); + } + let parameters = get_instance_parameters(&deps.local_db, instance_id) + .await? + .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + let local_pubkey: PublicKey = instance_keypair.public_key().into(); + let (_, derived_nonce, nonce_sig) = + committee_master_key.nonce_for_instance_job_with_keypair(¶meters, instance_keypair)?; + match get_committee_pub_nonce_for_instance(&deps.local_db, instance_id, &local_pubkey).await? { + Some(stored) => { + if stored != derived_nonce { + return invalid(format!( + "stored PeginConfirm nonce differs from the deterministic nonce for {instance_id}" + )); + } + } + None => { + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + if get_committee_partial_sig_for_instance(&deps.local_db, instance_id, &local_pubkey) + .await? + .is_some() + { + return Step::ignore("local PeginConfirm partial signature already exists"); + } + store_committee_pub_nonce_for_instance( + &deps.local_db, + instance_id, + local_pubkey, + derived_nonce.clone(), + ) + .await?; + } + } + let mut plan = InstancePlan::default(); + plan.send( + round_message( + instance_id, + GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { + instance_id, + committee_pubkey: local_pubkey, + pub_nonce: derived_nonce, + nonce_sig, + }), + ) + .await?, + ); + if nonces_complete(deps, instance_id).await? { + plan.then(JOB_PEGIN_CONFIRM_VOTE, Vec::new(), None); + } + Step::done(plan) +} + +async fn ingest_pegin_confirm_nonce( + deps: &JobDeps, + instance_id: Uuid, + params: &[u8], +) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { + instance_id: message_instance_id, + committee_pubkey, + pub_nonce, + nonce_sig, + }) = message.content + else { + return invalid("not a PeginConfirmNonce message"); + }; + if message_instance_id != instance_id || inbound.instance_id != instance_id { + return invalid("PeginConfirmNonce identity mismatch"); + } + ensure_committee_sender(deps, &inbound, &committee_pubkey).await?; + if !verify_public_nonce(&nonce_sig, &pub_nonce, &XOnlyPublicKey::from(committee_pubkey)) { + return invalid(format!("invalid pub_nonce or nonce_sig from {committee_pubkey}")); + } + store_committee_pub_nonce_for_instance( + &deps.local_db, + instance_id, + committee_pubkey, + pub_nonce, + ) + .await?; + let mut plan = InstancePlan::default(); + if nonces_complete(deps, instance_id).await? { + plan.then(JOB_PEGIN_CONFIRM_VOTE, Vec::new(), None); + } + Step::done(plan) +} + +/// Sign the agreed nonce transcript once every nonce is in. +async fn pegin_confirm_vote(deps: &JobDeps, instance_id: Uuid) -> Result { + if let Some(status) = instance_status(deps, instance_id).await? + && confirm_round_over(status) + { + return Step::ignore(format!("instance is {status}; round over")); + } + let Some(consensus) = nonce_consensus(deps, instance_id).await? else { + return Step::ignore("waiting for committee public nonces"); + }; + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + let local_pubkey: PublicKey = instance_keypair.public_key().into(); + let votes = get_committee_agg_nonce_consensus_for_instance(&deps.local_db, instance_id).await?; + let local_vote = votes.iter().find(|(pubkey, _, _)| *pubkey == local_pubkey); + if let Some((_, stored_hash, _)) = local_vote + && *stored_hash != consensus.consensus_hash + { + return invalid(format!("local PeginConfirm nonce consensus differs for {instance_id}")); + } + let signature = match local_vote { + Some((_, _, signature)) => *signature, + None => { + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + let signature = SECP256K1.sign_schnorr( + &SecpMessage::from_digest(consensus.consensus_hash), + &instance_keypair, + ); + store_committee_agg_nonce_consensus_for_instance( + &deps.local_db, + instance_id, + local_pubkey, + consensus.consensus_hash, + signature, + ) + .await?; + signature + } + }; + let mut plan = InstancePlan::default(); + plan.send( + round_message( + instance_id, + GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { + instance_id, + committee_pubkey: local_pubkey, + consensus_hash: consensus.consensus_hash, + signature, + }), + ) + .await?, + ); + if votes_complete(deps, instance_id, &consensus).await? { + plan.then(JOB_PEGIN_CONFIRM_SIGN, Vec::new(), None); + } + Step::done(plan) +} + +async fn ingest_pegin_confirm_vote( + deps: &JobDeps, + instance_id: Uuid, + params: &[u8], +) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { + instance_id: message_instance_id, + committee_pubkey, + consensus_hash: received_hash, + signature, + }) = message.content + else { + return invalid("not a PeginConfirmNonceConsensus message"); + }; + if message_instance_id != instance_id || inbound.instance_id != instance_id { + return invalid("PeginConfirmNonceConsensus identity mismatch"); + } + ensure_committee_sender(deps, &inbound, &committee_pubkey).await?; + let Some(consensus) = nonce_consensus(deps, instance_id).await? else { + return Step::retry( + "waiting for committee public nonces before validating PeginConfirm nonce consensus", + 30, + ); + }; + if received_hash != consensus.consensus_hash { + return invalid(format!("consensus hash mismatch from {committee_pubkey}")); + } + if SECP256K1 + .verify_schnorr( + &signature, + &SecpMessage::from_digest(consensus.consensus_hash), + &XOnlyPublicKey::from(committee_pubkey), + ) + .is_err() + { + return invalid(format!("invalid consensus signature from {committee_pubkey}")); + } + store_committee_agg_nonce_consensus_for_instance( + &deps.local_db, + instance_id, + committee_pubkey, + received_hash, + signature, + ) + .await?; + let mut plan = InstancePlan::default(); + if votes_complete(deps, instance_id, &consensus).await? { + plan.then(JOB_PEGIN_CONFIRM_SIGN, Vec::new(), None); + } + Step::done(plan) +} + +/// The committee's partial signature over the Peg-In-Confirm, with the EVM +/// endorsement the relayer's `postPeginData` needs. Authorization is written +/// before signing: the deterministic nonce signs exactly one transcript. +async fn pegin_confirm_sign(deps: &JobDeps, instance_id: Uuid) -> Result { + if let Some(status) = instance_status(deps, instance_id).await? + && confirm_round_over(status) + { + return Step::ignore(format!("instance is {status}; round over")); + } + let Some(consensus) = nonce_consensus(deps, instance_id).await? else { + return Step::ignore("waiting for committee public nonces"); + }; + if !votes_complete(deps, instance_id, &consensus).await? { + return Step::ignore("waiting for agg nonce consensus"); + } + let Some(instance_keypair) = local_member(deps, instance_id).await? else { + return Step::ignore("not in this instance's committee"); + }; + let local_pubkey: PublicKey = instance_keypair.public_key().into(); + let pegin_txid = consensus.parameters.build_pegin_tx()?.1.tx().compute_txid(); + let (partial_sig, endorse_sig) = match get_committee_partial_sig_for_instance( + &deps.local_db, + instance_id, + &local_pubkey, + ) + .await? + { + Some(stored) => { + // Reuse the stored partial signature and generate a missing EVM endorsement. + let stored_endorse_sig = + get_committee_endorse_sigs_for_pegin(&deps.local_db, instance_id) + .await? + .into_iter() + .find(|(pubkey, _)| *pubkey == local_pubkey) + .map(|(_, endorse_sig)| endorse_sig); + let endorse_sig = match stored_endorse_sig { + Some(endorse_sig) => endorse_sig, + None => { + let endorse_sig = endorse_pegin(&deps.goat_client, instance_id, &pegin_txid) + .await? + .as_bytes() + .to_vec(); + store_committee_endorse_sig_for_pegin( + &deps.local_db, + instance_id, + local_pubkey, + endorse_sig.clone(), + ) + .await?; + endorse_sig + } + }; + (stored, endorse_sig) + } + None => { + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + let (sec_nonce, pub_nonce, _) = committee_master_key + .nonce_for_instance_job_with_keypair(&consensus.parameters, instance_keypair)?; + let nonce_id = signing_nonce_id("pegin-confirm", &pub_nonce.serialize()); + let transcript = hex::encode(consensus.consensus_hash); + let authorization = deps + .local_db + .acquire() + .await? + .authorize_signing( + "instance", + &instance_id.to_string(), + &nonce_id, + &transcript, + current_time_secs(), + ) + .await?; + if let SigningAuthorization::Conflict { existing_digest } = authorization { + return invalid(format!( + "PeginConfirm nonce for {instance_id} is already authorized for transcript {existing_digest}; refusing {transcript}" + )); + } + let mut pegin_confirm = consensus.parameters.build_pegin_tx()?.1; + let committee_context = consensus.parameters.get_committee_context(instance_keypair)?; + let partial_sig = pegin_confirm + .sign_input_0_musig2(&committee_context, &sec_nonce, &consensus.agg_nonce) + .map_err(|error| { + anyhow!("Failed to sign PeginConfirm for {instance_id}: {error}") + })?; + let endorse_sig = endorse_pegin(&deps.goat_client, instance_id, &pegin_txid) + .await? + .as_bytes() + .to_vec(); + store_committee_partial_sig_for_instance( + &deps.local_db, + instance_id, + local_pubkey, + partial_sig, + ) + .await?; + store_committee_endorse_sig_for_pegin( + &deps.local_db, + instance_id, + local_pubkey, + endorse_sig.clone(), + ) + .await?; + (partial_sig, endorse_sig) + } + }; + let mut plan = InstancePlan::default(); + plan.send( + round_message( + instance_id, + GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { + instance_id, + committee_pubkey: local_pubkey, + partial_sig, + endorse_sig, + }), + ) + .await?, + ); + if partial_sigs_complete(deps, instance_id).await? { + then_broadcast(deps, &mut plan); + } + Step::done(plan) +} + +async fn ingest_pegin_confirm_partial_sig( + deps: &JobDeps, + instance_id: Uuid, + params: &[u8], +) -> Result { + let (inbound, message) = decode_inbound(params).await?; + let GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { + instance_id: message_instance_id, + committee_pubkey, + partial_sig, + endorse_sig, + }) = message.content + else { + return invalid("not a PeginConfirmPartialSig message"); + }; + if message_instance_id != instance_id || inbound.instance_id != instance_id { + return invalid("PeginConfirmPartialSig identity mismatch"); + } + ensure_committee_sender(deps, &inbound, &committee_pubkey).await?; + let committee = committee_pubkeys(deps, instance_id).await?; + let pub_nonces_unchecked = + get_committee_pub_nonces_for_instance(&deps.local_db, instance_id).await?; + if pub_nonces_unchecked.len() != committee.len() { + return Step::retry("waiting for committee public nonces", 30); + } + let Some(signer_pub_nonce) = pub_nonces_unchecked + .iter() + .find(|(pubkey, _)| *pubkey == committee_pubkey) + .map(|(_, pub_nonce)| pub_nonce.clone()) + else { + return invalid(format!("missing signer pub nonce for {committee_pubkey}")); + }; + let Some(consensus) = nonce_consensus(deps, instance_id).await? else { + bail!("committee public nonces changed while a partial signature was verified"); + }; + if !votes_complete(deps, instance_id, &consensus).await? { + return Step::retry("waiting for committee PeginConfirm agg nonce consensus", 30); + } + if let Err(error) = verify_pegin_confirm_partial_sig( + &consensus.parameters, + &committee, + &committee_pubkey, + &signer_pub_nonce, + &consensus.agg_nonce, + partial_sig, + ) { + return invalid(format!("invalid partial signature from {committee_pubkey}: {error}")); + } + let pegin_txid = consensus.parameters.build_pegin_tx()?.1.tx().compute_txid(); + match verify_pegin_endorsement( + &deps.goat_client, + instance_id, + &committee_pubkey, + &pegin_txid, + &endorse_sig, + ) + .await + { + Ok(true) => {} + Ok(false) => { + return invalid(format!("invalid endorsement signature from {committee_pubkey}")); + } + Err(error) => { + return Step::retry( + format!("failed to verify committee endorsement signature: {error}"), + 30, + ); + } + } + store_committee_partial_sig_for_instance( + &deps.local_db, + instance_id, + committee_pubkey, + partial_sig, + ) + .await?; + store_committee_endorse_sig_for_pegin( + &deps.local_db, + instance_id, + committee_pubkey, + endorse_sig, + ) + .await?; + let mut plan = InstancePlan::default(); + if partial_sigs_complete(deps, instance_id).await? { + then_broadcast(deps, &mut plan); + } + Step::done(plan) +} + +/// The relayer broadcasts Peg-In-Confirm only before T0 and while no Peg-In-Cancel is +/// confirmed. +async fn pegin_confirm_broadcast_blocked( + deps: &JobDeps, + instance_id: Uuid, +) -> Result> { + let instance = deps.local_db.acquire().await?.find_instance(&instance_id).await?; + let Some(instance) = instance else { + return Ok(Some("instance not found".to_string())); + }; + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Ok(Some(reason)); + } + if let Some(cancel_txid) = instance.pegin_cancel_txid + && let Some(cancel_tx) = deps.btc_client.get_tx_info(&cancel_txid.0).await? + && cancel_tx.status.confirmed + { + return Ok(Some(format!("Peg-In-Cancel {} is confirmed", cancel_txid.0))); + } + Ok(None) +} + +/// Aggregate the committee's partial signatures and broadcast the +/// Peg-In-Confirm. Re-run per block while the transaction is not visible +/// (the apply schedules it from the Bitcoin snapshot, with the snapshot's +/// height as params), so a failed attempt retries only until the next +/// block's job takes over. +async fn broadcast_pegin_confirm(deps: &JobDeps, instance_id: Uuid, params: &[u8]) -> Result { + if !deps.relayer { + return Step::ignore("not a relayer"); + } + // The first attempt (from the signatures) carries no height. + let scanned_height: Option = params.try_into().ok().map(i64::from_le_bytes); + let parameters = get_instance_parameters(&deps.local_db, instance_id) + .await? + .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; + let mut pegin_confirm = parameters.build_pegin_tx()?.1; + let pegin_txid = pegin_confirm.tx().compute_txid(); + if deps.btc_client.get_tx(&pegin_txid).await?.is_some() { + return Step::ignore(format!("Peg-In-Confirm {pegin_txid} is already visible")); + } + if let Some(reason) = pegin_confirm_broadcast_blocked(deps, instance_id).await? { + info!(event = "signing_gate", instance_id = %instance_id, reason = %reason, "not broadcasting Peg-In-Confirm"); + return Step::ignore(reason); + } + let committee = committee_pubkeys(deps, instance_id).await?; + let pub_nonces = get_committee_pub_nonces_for_instance(&deps.local_db, instance_id).await?; + let partial_sigs = get_committee_partial_sigs_for_instance(&deps.local_db, instance_id).await?; + if pub_nonces.len() != committee.len() || partial_sigs.len() != committee.len() { + return Step::ignore(format!( + "{} of {} nonces and {} of {} partial signatures", + pub_nonces.len(), + committee.len(), + partial_sigs.len(), + committee.len() + )); + } + let pub_nonces = order_committee_values(&committee, pub_nonces, "pegin committee pub nonces")?; + let partial_sigs = + order_committee_values(&committee, partial_sigs, "pegin committee partial sigs")?; + let agg_nonce = nonce_aggregation(&pub_nonces); + let context = parameters.get_base_context(); + let full_sig = pegin_confirm + .aggregate_input_0_musig2_signatures(&context, partial_sigs, &agg_nonce) + .map_err(|error| { + anyhow!("Failed to aggregate PeginConfirm signatures for {instance_id}: {error}") + })?; + let connector_z = parameters.connector_z(); + pegin_confirm.push_input_0_signature(&connector_z, full_sig); + let result = broadcast_tx(&deps.btc_client, pegin_confirm.tx()).await; + deps.metrics_state.record_pegin_confirm(result.is_ok()); + match result { + Ok(()) => { + info!(event = "pegin_confirm_broadcast", instance_id = %instance_id, txid = %pegin_txid, "Peg-In-Confirm broadcast"); + Step::done(InstancePlan::noted(format!("broadcast {pegin_txid}"))) + } + Err(error) => { + if let Some(scanned_height) = scanned_height + && i64::from(deps.btc_client.get_height().await?) > scanned_height + { + return Step::ignore(format!( + "broadcast Peg-In-Confirm {pegin_txid} failed ({error:#}); the next block's attempt takes over" + )); + } + Step::retry(format!("broadcast Peg-In-Confirm {pegin_txid}: {error:#}"), 60) + } + } +} + +// ------------------------------------------------------------- post pegin + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PostOutcome { + /// Pegin data is on Goat. + Done, + /// A precondition is not met yet; retry after about one block. + Waiting(String), + /// The gateway refuses the post for good. + Rejected(String), +} + +/// The gateway reverts these for good; everything else is retried. +fn is_permanent_post_rejection(error: &anyhow::Error) -> bool { + let message = format!("{error:#}"); + ["not a pending pegin request", "pegin amount mismatch", "only committee member can call"] + .iter() + .any(|needle| message.contains(needle)) +} + +/// Submit pegin data after Confirm is confirmed, all committee endorsements are +/// present and Goat SPV covers it. Graph publishing uses separate Graph jobs. +pub async fn post_pegin_data(deps: &JobDeps, instance_id: Uuid) -> anyhow::Result { + if !deps.relayer { + warn!(event = "instance_post", instance_id = %instance_id, outcome = "not_relayer", "post job on a node that is not a relayer"); + return Ok(PostOutcome::Done); + } + let goat_client = deps.goat_client.as_ref(); + let btc_client = deps.btc_client.as_ref(); + let committee_pubkeys = goat_client.gateway_get_committee_pubkeys(&instance_id).await?; + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + match pegin_data.status { + PeginStatus::None => { + return Ok(PostOutcome::Rejected("not a pending pegin request on Goat".to_string())); + } + PeginStatus::Pending => { + let instance_params = get_instance_parameters(&deps.local_db, instance_id) + .await? + .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; + let pegin_confirm = instance_params.build_pegin_tx()?.1; + let pegin_txid = pegin_confirm.tx().compute_txid(); + let Some(pegin_tx) = btc_client.get_tx(&pegin_txid).await? else { + return Ok(PostOutcome::Waiting(format!( + "Peg-In-Confirm {pegin_txid} is not available from the Bitcoin backend" + ))); + }; + let endorse_sigs = get_committee_endorse_sigs_for_pegin(&deps.local_db, instance_id) + .await? + .into_iter() + .map(|(_, endorse_sig)| endorse_sig) + .collect::>(); + if endorse_sigs.len() != committee_pubkeys.len() { + return Ok(PostOutcome::Waiting(format!( + "{} of {} committee endorsements of the Peg-In-Confirm", + endorse_sigs.len(), + committee_pubkeys.len() + ))); + } + let Some(pegin_height) = btc_client.get_tx_status(&pegin_txid).await?.block_height + else { + return Ok(PostOutcome::Waiting(format!( + "Peg-In-Confirm {pegin_txid} is not confirmed on Bitcoin" + ))); + }; + let spv_height = goat_client.btc_spv_latest_height().await?; + if spv_height < u64::from(pegin_height) { + return Ok(PostOutcome::Waiting(format!( + "Goat SPV at {spv_height}, Peg-In-Confirm block {pegin_height}" + ))); + } + let result = goat_client + .gateway_post_pegin_data(btc_client, &instance_id, &pegin_tx, &endorse_sigs) + .await; + deps.metrics_state.record_pegin_post(result.is_ok()); + if let Err(error) = result { + if is_permanent_post_rejection(&error) { + return Ok(PostOutcome::Rejected(format!("{error:#}"))); + } + return Err(error); + } + info!(event = "instance_post", instance_id = %instance_id, outcome = "pegin_posted", "postPeginData sent"); + } + _ => { + // Already posted. + } + } + Ok(PostOutcome::Done) +} + +struct PostPeginDataRunner { + deps: Arc, +} + +#[async_trait] +impl JobRunner for PostPeginDataRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let instance_id = match aggregate_uuid(&ctx, "an instance") { + Ok(instance_id) => instance_id, + Err(outcome) => return outcome, + }; + match post_pegin_data(&self.deps, instance_id).await { + Ok(PostOutcome::Done) => JobOutcome::Done(b"posted".to_vec()), + Ok(PostOutcome::Waiting(reason)) => JobOutcome::Transient { + error: reason, + retry_after_secs: Some(avg_block_time_secs(self.deps.btc_client.network()) as i64), + }, + Ok(PostOutcome::Rejected(reason)) => JobOutcome::Failed(reason), + Err(error) => { + JobOutcome::Transient { error: format!("{error:#}"), retry_after_secs: None } + } + } + } +} + +// ------------------------------------------------------------- key delete + +struct DeleteInstanceKeyRunner; + +fn key_delete_done(deleted: bool, detail: impl Into) -> JobOutcome { + let result = KeyDeleteResult { deleted, detail: detail.into() }; + JobOutcome::Done(serde_json::to_vec(&result).expect("KeyDeleteResult serializes")) +} + +#[async_trait] +impl JobRunner for DeleteInstanceKeyRunner { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let instance_id = match aggregate_uuid(&ctx, "an instance") { + Ok(instance_id) => instance_id, + Err(outcome) => return outcome, + }; + if !is_enable_committee_instance_key_delete() { + return key_delete_done(false, "deletion disabled"); + } + let envelope_path = committee_instance_keys_envelope_path(instance_id); + if !envelope_path.exists() { + return key_delete_done(true, "envelope absent"); + } + let committee_master_key = match get_bitvm_key() { + Ok(key) => CommitteeMasterKey::new(key), + Err(error) => { + return JobOutcome::Transient { + error: format!("load committee master key: {error:#}"), + retry_after_secs: None, + }; + } + }; + match committee_master_key.delete_instance_keypair_envelope(instance_id, &envelope_path) { + Ok(()) => { + info!( + event = "instance_key_delete", + instance_id = %instance_id, + path = %envelope_path.display(), + "deleted committee instance key envelope" + ); + key_delete_done(true, envelope_path.display().to_string()) + } + Err(error) => JobOutcome::Failed(format!( + "delete envelope {}: {error:#}", + envelope_path.display() + )), + } + } +} diff --git a/node/src/aggregates/instance_scan.rs b/node/src/aggregates/instance_scan.rs new file mode 100644 index 000000000..277e5697c --- /dev/null +++ b/node/src/aggregates/instance_scan.rs @@ -0,0 +1,490 @@ +//! Collect Instance chain facts outside the apply transaction. +//! +//! Maintenance records answer-window results and Bitcoin snapshots as inbox +//! events. Snapshots include transaction visibility, input spends and tip height +//! for deadline decisions. + +use super::common::ScanMarks; +use super::instance::{AnswerWindow, BtcSnapshot, PeginRequested, TxFact, pegin_request_event}; +use crate::clock::current_time_secs; +use crate::engine_bridge::record_event_in; +use crate::env::{get_instance_maintenance_batch_size, is_enable_committee_instance_key_delete}; +use crate::error::{FailureClass, classify_failure}; +use crate::utils::{ + check_bridge_in_uxto_available_or_self_spent, generate_instance_from_bridge_in_request_event, +}; +use bitcoin::Txid; +use bitvm_lib::actors::Actor; +use client::Utxo; +use client::btc_chain::BTCClient; +use client::goat_chain::GOATClient; +use client::graphs::graph_query::BridgeInRequestEvent; +use engine::EngineHandle; +use indexmap::IndexMap; +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::{LazyLock, Mutex}; +use store::localdb::{InstanceQuery, LocalDB}; +use store::{GoatTxProcessingStatus, GoatTxType, Instance, InstanceBridgeInStatus}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +const TASK_KEY_ANSWER_WINDOW: &str = "instance_answer_window_scan"; +const TASK_KEY_BTC: &str = "instance_btc_scan"; +const SCAN_MARKS_LIMIT: usize = 50_000; + +// ----------------------------------------------------------------- paging + +#[derive(Clone, Debug)] +struct InstancePageState { + watermark: i64, + cursor: Option<(i64, Uuid)>, +} + +static INSTANCE_PAGE_STATES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +fn get_or_init_instance_page_state(task_key: &'static str) -> InstancePageState { + let mut state = INSTANCE_PAGE_STATES.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + state + .entry(task_key) + .or_insert_with(|| InstancePageState { watermark: current_time_secs(), cursor: None }) + .clone() +} + +fn reset_instance_page_state(task_key: &'static str) { + let mut state = INSTANCE_PAGE_STATES.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + state.insert(task_key, InstancePageState { watermark: current_time_secs(), cursor: None }); +} + +fn advance_instance_page_state(task_key: &'static str, watermark: i64, last: (i64, Uuid)) { + let mut state = INSTANCE_PAGE_STATES.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + state.insert(task_key, InstancePageState { watermark, cursor: Some(last) }); +} + +/// One page of instances per tick, in creation order, so a large table is +/// walked across ticks instead of blocking one. +pub(crate) async fn find_one_instance_page( + local_db: &LocalDB, + task_key: &'static str, + mut query: InstanceQuery, + batch_size: u32, +) -> anyhow::Result> { + let page_state = get_or_init_instance_page_state(task_key); + query = query + .with_order("created_at ASC, instance_id ASC".to_string()) + .with_limit(batch_size) + .with_raw_condition(format!("created_at <= {}", page_state.watermark)); + + if let Some((created_at, instance_id)) = page_state.cursor { + query = query.with_raw_condition(format!( + "(created_at > {created_at} OR (created_at = {created_at} AND instance_id > '{instance_id}'))" + )); + } + + let instances = { + let mut storage_processor = local_db.acquire().await?; + let (instances, _) = storage_processor.find_instances(query).await?; + instances + }; + + if instances.is_empty() { + reset_instance_page_state(task_key); + return Ok(instances); + } + + if instances.len() < batch_size as usize { + reset_instance_page_state(task_key); + } else if let Some(last) = instances.last() { + advance_instance_page_state( + task_key, + page_state.watermark, + (last.created_at, last.instance_id), + ); + } + + Ok(instances) +} + +// ------------------------------------------------------------ answer window + +/// Record the committee answers of every `UserInited` instance whose Goat +/// response window is over. The apply decides quorum. +pub async fn scan_answer_windows( + local_db: &LocalDB, + goat_client: &GOATClient, + engine: &EngineHandle, +) -> anyhow::Result<()> { + let window_blocks = goat_client.gateway_get_response_window_blocks().await? as i64; + let goat_height = goat_client.get_latest_block_number().await?; + let instances = find_one_instance_page( + local_db, + TASK_KEY_ANSWER_WINDOW, + InstanceQuery::default() + .with_status(InstanceBridgeInStatus::UserInited.to_string()) + .with_pegin_request_height_threshold(goat_height - window_blocks), + get_instance_maintenance_batch_size(), + ) + .await?; + if instances.is_empty() { + return Ok(()); + } + let quorum = goat_client.committee_mana_quorum_size().await?; + for instance in instances { + let pegin_data = match goat_client.gateway_get_pegin_data(&instance.instance_id).await { + Ok(pegin_data) => pegin_data, + Err(error) => { + warn!( + event = "instance_scan", + instance_id = %instance.instance_id, + error = %error, + "failed to read pegin data for the answer window" + ); + continue; + } + }; + let answers: IndexMap> = pegin_data + .committee_addresses + .iter() + .map(ToString::to_string) + .zip(pegin_data.committee_pubkeys) + .collect(); + let window = AnswerWindow { goat_height, window_blocks, quorum, answers }; + engine.submit(window.event(instance.instance_id)?).await?; + } + Ok(()) +} + +// ------------------------------------------------------------------ bitcoin + +static SCAN_MARKS: LazyLock> = LazyLock::new(ScanMarks::new); + +/// Timeout instances whose Peg-In-Confirm confirmed anyway. The apply reports +/// them once; the row stays `Timeout` for manual handling and there is nothing +/// left to watch, so they leave the scan (until a restart, which reports again). +static SETTLED_TIMEOUTS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + +fn settled_timeout(instance_id: Uuid) -> bool { + SETTLED_TIMEOUTS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).contains(&instance_id) +} + +fn mark_settled_timeout(instance_id: Uuid) { + let mut settled = SETTLED_TIMEOUTS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if settled.len() >= SCAN_MARKS_LIMIT { + settled.clear(); + } + settled.insert(instance_id); +} + +/// Statuses the Bitcoin scan follows. The key-deletion statuses are only +/// followed on committee nodes with deletion enabled. +fn scanned_statuses(actor: &Actor) -> (Vec, bool) { + use InstanceBridgeInStatus::*; + let key_scan = *actor == Actor::Committee && is_enable_committee_instance_key_delete(); + let mut statuses = vec![CommitteesAnswered, UserBroadcastPeginPrepare, Presigned, Timeout]; + if key_scan { + statuses.extend([RelayerL1Broadcasted, RelayerL2Minted]); + } + (statuses.iter().map(ToString::to_string).collect(), key_scan) +} + +/// Record a Bitcoin snapshot for every live instance. +pub async fn scan_instances_on_bitcoin( + local_db: &LocalDB, + btc_client: &BTCClient, + engine: &EngineHandle, + actor: &Actor, +) -> anyhow::Result<()> { + let (statuses, key_scan) = scanned_statuses(actor); + let mut query = InstanceQuery::default().with_statuses(statuses); + if key_scan { + query = query.with_raw_condition(format!( + "(status NOT IN ('{}', '{}') OR key_deleted = 0)", + InstanceBridgeInStatus::RelayerL1Broadcasted, + InstanceBridgeInStatus::RelayerL2Minted + )); + } + let instances = find_one_instance_page( + local_db, + TASK_KEY_BTC, + query, + get_instance_maintenance_batch_size(), + ) + .await?; + if instances.is_empty() { + return Ok(()); + } + let height = i64::from(btc_client.get_height().await?); + for instance in instances { + let instance_id = instance.instance_id; + let Ok(status) = InstanceBridgeInStatus::from_str(&instance.status) else { + warn!(event = "instance_scan", instance_id = %instance_id, status = %instance.status, "unknown instance status"); + continue; + }; + if SCAN_MARKS.seen(instance_id, height, status) { + continue; + } + if status == InstanceBridgeInStatus::Timeout && settled_timeout(instance_id) { + continue; + } + match gather_snapshot(btc_client, &instance, status, height).await { + Ok(Some(snapshot)) => { + let confirm_confirmed = + snapshot.confirm.as_ref().is_some_and(|fact| fact.confirmed); + engine.submit(snapshot.event(instance_id)?).await?; + SCAN_MARKS.mark(instance_id, height, status); + if status == InstanceBridgeInStatus::Timeout && confirm_confirmed { + mark_settled_timeout(instance_id); + } + } + Ok(None) => SCAN_MARKS.forget(instance_id), + Err(error) => warn!( + event = "instance_scan", + instance_id = %instance_id, + status = %status, + height, + error = %error, + "snapshot failed; retried on the next tick" + ), + } + } + Ok(()) +} + +/// One call per transaction: existence, confirmation and height together, so +/// an RPC failure is an error and never mistaken for "not in the mempool". +async fn tx_fact(btc_client: &BTCClient, txid: &Txid) -> anyhow::Result { + Ok(match btc_client.get_tx_info(txid).await? { + None => TxFact::UNKNOWN, + Some(tx) => TxFact { + confirmed: tx.status.confirmed, + block_height: tx.status.block_height.map(i64::from), + in_mempool: !tx.status.confirmed, + }, + }) +} + +/// Gather the facts the apply needs for `status`. `None` when nothing about +/// this status is decided on Bitcoin. +pub async fn gather_snapshot( + btc_client: &BTCClient, + instance: &Instance, + status: InstanceBridgeInStatus, + height: i64, +) -> anyhow::Result> { + use InstanceBridgeInStatus::*; + let mut snapshot = + BtcSnapshot { height, status_at_scan: status.to_string(), ..Default::default() }; + let utxos: Vec = serde_json::from_str(&instance.input_utxos)?; + let prepare_txid = instance.btc_txid.as_ref().map(|txid| txid.0); + let confirm_txid = instance.pegin_confirm_txid.as_ref().map(|txid| txid.0); + let cancel_txid = instance.pegin_cancel_txid.as_ref().map(|txid| txid.0); + + match status { + // Without the Prepare txid a spend cannot be told from the user's own + // Prepare: nothing is decided until it is derived. + CommitteesAnswered => { + if let Some(prepare_txid) = prepare_txid { + let prepare = tx_fact(btc_client, &prepare_txid).await?; + if prepare.confirmed { + // The scan may cross to Confirm or Cancel in one pass. + if let Some(txid) = confirm_txid { + snapshot.confirm = Some(tx_fact(btc_client, &txid).await?); + } + if let Some(txid) = cancel_txid { + snapshot.cancel = Some(tx_fact(btc_client, &txid).await?); + } + } else { + snapshot.inputs_available = Some( + check_bridge_in_uxto_available_or_self_spent( + btc_client, + Some(prepare_txid.to_string()), + &utxos, + ) + .await?, + ); + } + snapshot.prepare = Some(prepare); + } + } + UserBroadcastPeginPrepare | Presigned => { + if let Some(txid) = confirm_txid { + snapshot.confirm = Some(tx_fact(btc_client, &txid).await?); + } + if let Some(txid) = cancel_txid { + snapshot.cancel = Some(tx_fact(btc_client, &txid).await?); + } + if status == UserBroadcastPeginPrepare + && let Some(prepare_txid) = prepare_txid + { + let prepare = tx_fact(btc_client, &prepare_txid).await?; + if !prepare.confirmed { + snapshot.inputs_available = Some( + check_bridge_in_uxto_available_or_self_spent( + btc_client, + Some(prepare_txid.to_string()), + &utxos, + ) + .await?, + ); + } + snapshot.prepare = Some(prepare); + } + } + Timeout => { + if let Some(txid) = cancel_txid { + snapshot.cancel = Some(tx_fact(btc_client, &txid).await?); + } + if let Some(txid) = confirm_txid { + snapshot.confirm = Some(tx_fact(btc_client, &txid).await?); + } + } + RelayerL1Broadcasted | RelayerL2Minted => { + let Some(txid) = confirm_txid.filter(|_| !instance.key_deleted) else { + return Ok(None); + }; + snapshot.confirm = Some(tx_fact(btc_client, &txid).await?); + } + // Nothing of the Answer phase is decided on Bitcoin: the Prepare txid is unknown, so a + // spent input may be the user's own Prepare. + UserInited | UserCanceled | NoEnoughCommitteesAnswered | UserDiscarded => { + return Ok(None); + } + } + debug!( + event = "instance_scan", + instance_id = %instance.instance_id, + status = %status, + height, + prepare = ?snapshot.prepare, + confirm = ?snapshot.confirm, + cancel = ?snapshot.cancel, + inputs_available = ?snapshot.inputs_available, + "bitcoin snapshot" + ); + Ok(Some(snapshot)) +} + +/// Store pending Goat bridge-in requests. Submit pegin_request events only while +/// the response window is open; late requests are handled by the answer-window scan. +pub async fn pegin_request_monitor( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + engine: &EngineHandle, +) -> anyhow::Result<()> { + let tx_records = { + let mut storage_processor = local_db.acquire().await?; + storage_processor + .get_goat_tx_record_by_processing_status( + &GoatTxType::BridgeInRequest.to_string(), + &GoatTxProcessingStatus::Pending.to_string(), + ) + .await? + }; + if tx_records.is_empty() { + return Ok(()); + } + let current_height = goat_client.get_finalized_block_number().await?; + let response_window_blocks = goat_client.gateway_get_response_window_blocks().await? as i64; + for tx_record in tx_records { + let instance_id = tx_record.instance_id; + let event = tx_record + .extra + .as_deref() + .map(serde_json::from_str::) + .transpose()?; + let is_outside_response_window = tx_record.height + response_window_blocks < current_height; + // The row is built from the indexed event and Goat's pegin data; the + // request's inputs are checked by the committee's answer job. + let instance = match event.as_ref() { + Some(event) => { + match generate_instance_from_bridge_in_request_event(btc_client, goat_client, event) + .await + { + Ok(instance) => Some(instance), + Err(error) + if classify_failure(&error) == FailureClass::Transient + && !is_outside_response_window => + { + warn!( + event = "pegin_request", + instance_id = %instance_id, + error = %error, + "could not build the instance of a BridgeInRequest; retried next tick" + ); + continue; + } + Err(error) => { + // A malformed request, a Goat refusal or a request past its + // window: nothing a retry can change. + warn!( + event = "pegin_request", + instance_id = %instance_id, + outside_window = is_outside_response_window, + error = %error, + "could not build the instance of a BridgeInRequest; request skipped" + ); + local_db + .acquire() + .await? + .update_goat_tx_record_processing_status( + &tx_record.graph_id, + &instance_id, + &tx_record.tx_type, + &GoatTxProcessingStatus::Skipped.to_string(), + ) + .await?; + continue; + } + } + } + None => None, + }; + + let mut tx = local_db.start_transaction().await?; + if let Some(instance) = instance { + if is_outside_response_window { + if tx.insert_instance_if_absent(&instance).await? { + info!( + event = "pegin_request", + instance_id = %instance_id, + outcome = "late", + "BridgeInRequest seen after its response window; the answer-window scan decides" + ); + } + } else { + // A re-indexed request refreshes its row; a row that moved on is + // left alone. + tx.upsert_pegin_request_instance(&instance).await?; + record_event_in( + &mut tx, + pegin_request_event( + instance_id, + &PeginRequested { tx_hash: tx_record.tx_hash.clone() }, + )?, + ) + .await?; + } + } else { + warn!( + event = "pegin_request", + instance_id = %instance_id, + "BridgeInRequest record carries no event; skipped" + ); + } + tx.update_goat_tx_record_processing_status( + &tx_record.graph_id, + &instance_id, + &tx_record.tx_type, + &GoatTxProcessingStatus::Processed.to_string(), + ) + .await?; + tx.commit().await?; + if !is_outside_response_window { + engine.wake_for(&crate::aggregates::instance::instance_ref(instance_id)); + } + } + Ok(()) +} diff --git a/node/src/aggregates/instance_tests.rs b/node/src/aggregates/instance_tests.rs new file mode 100644 index 000000000..839b93681 --- /dev/null +++ b/node/src/aggregates/instance_tests.rs @@ -0,0 +1,1108 @@ +//! Instance lifecycle tests using the engine and SQLite, covering failure +//! states, signing deadlines, the abandon buffer and cleanup effects. + +use super::common::{InboundMessage, MembershipProbe, PlannedMessage, content_event_id}; +use super::instance::*; +use crate::utils::{mark_graph_as_endorsed, store_committee_partial_sig_for_instance}; +use async_trait::async_trait; +use bitvm_lib::actors::Actor; +use engine::{ + Backoff, EngineBuilder, EngineConfig, EngineHandle, EngineInboxState, EventSource, JobContext, + JobOutcome, JobRunner, NewEvent, +}; +use indexmap::IndexMap; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use store::localdb::LocalDB; +use store::{ + Graph, GraphStatus, Instance, InstanceBridgeInStatus, NewEngineInboxEvent, NewEngineJob, + NewEngineOutboxMessage, +}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +const DEADLINES: PeginDeadlines = + PeginDeadlines { t0_blocks: 60, buffer_blocks: 6, key_delete_blocks: 32 }; +const PREPARE: i64 = 1_000; +const WAIT: Duration = Duration::from_secs(10); +const LOCAL_PEER: &str = "12D3KooWLocalPeer"; +const OTHER_PEER: &str = "12D3KooWOtherPeer"; + +struct Harness { + db: LocalDB, + _dir: tempfile::TempDir, + handle: EngineHandle, + cancel: CancellationToken, + run: tokio::task::JoinHandle>, + jobs: Arc, +} + +struct StubKeyDelete; + +#[async_trait] +impl JobRunner for StubKeyDelete { + async fn run(&self, _ctx: JobContext) -> JobOutcome { + let result = KeyDeleteResult { deleted: true, detail: "stub".into() }; + JobOutcome::Done(serde_json::to_vec(&result).unwrap()) + } +} + +struct StubPost; + +#[async_trait] +impl JobRunner for StubPost { + async fn run(&self, _ctx: JobContext) -> JobOutcome { + JobOutcome::Done(b"posted".to_vec()) + } +} + +/// Returns the scripted plan for an action and records what it was asked. +#[derive(Default)] +struct Scripted { + plans: Mutex>, + calls: Mutex)>>, +} + +#[async_trait] +impl JobRunner for Scripted { + async fn run(&self, ctx: JobContext) -> JobOutcome { + let params = ctx.job.params.clone().unwrap_or_default(); + self.calls.lock().unwrap().push((ctx.job.action.clone(), params)); + let plan = self.plans.lock().unwrap().remove(&ctx.job.action).unwrap_or_default(); + JobOutcome::Done(plan.encode().unwrap()) + } +} + +/// Every instance is this node's (the production probe looks for the key +/// envelope on disk). +struct Member; + +impl MembershipProbe for Member { + fn holds_instance_key(&self, _instance_id: Uuid) -> bool { + true + } +} + +/// A node outside every committee. +struct Outsider; + +impl MembershipProbe for Outsider { + fn holds_instance_key(&self, _instance_id: Uuid) -> bool { + false + } +} + +/// Every test graph takes its own operator slot (`(operator, nonce)` is unique). +static NEXT_SLOT: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +fn fast_config() -> EngineConfig { + EngineConfig { + worker_count: 1, + thread_stack_bytes: 0, + apply_timeout: Duration::from_secs(5), + worker_poll_interval: Duration::from_millis(30), + recheck_interval: Duration::from_millis(250), + inbox_retry_backoff: Backoff::Steps(vec![1]), + job_lease_secs: 5, + job_poll_interval: Duration::from_millis(30), + outbox_poll_interval: Duration::from_millis(30), + ..EngineConfig::default() + } +} + +async fn start(actor: Actor, relayer: bool, key_delete_enabled: bool) -> Harness { + start_with(actor, relayer, key_delete_enabled, Arc::new(Member)).await +} + +async fn start_with( + actor: Actor, + relayer: bool, + key_delete_enabled: bool, + membership: Arc, +) -> Harness { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("instance.db"); + let db = LocalDB::new_with_busy_timeout( + &format!("sqlite:{}", path.display()), + false, + store::DEFAULT_BUSY_TIMEOUT, + ) + .await; + db.migrate().await; + let jobs = Arc::new(Scripted::default()); + let mut builder = EngineBuilder::new(db.clone(), fast_config()) + .handler( + INSTANCE_KIND, + Arc::new( + InstanceAggregate::new( + actor, + DEADLINES, + relayer, + key_delete_enabled, + LOCAL_PEER.to_string(), + ) + .with_membership(membership), + ), + ) + .job_runner(INSTANCE_KIND, JOB_DELETE_INSTANCE_KEY, Arc::new(StubKeyDelete)) + .job_runner(INSTANCE_KIND, JOB_POST_PEGIN_DATA, Arc::new(StubPost)); + for action in INSTANCE_JOBS { + let runner: Arc = jobs.clone(); + builder = builder.job_runner(INSTANCE_KIND, *action, runner); + } + let (engine, _acks) = builder.build(); + let handle = engine.handle(); + let cancel = CancellationToken::new(); + let run = tokio::spawn(engine.run(cancel.clone())); + Harness { db, _dir: dir, handle, cancel, run, jobs } +} + +fn instance_row(status: InstanceBridgeInStatus, prepare_height: i64) -> Instance { + Instance { + instance_id: Uuid::new_v4(), + network: "regtest".to_string(), + input_utxos: "[]".to_string(), + status: status.to_string(), + goat_tx_height: 100, + btc_height: prepare_height, + created_at: 1, + updated_at: 1, + ..Default::default() + } +} + +impl Harness { + async fn stop(self) { + self.cancel.cancel(); + self.run.await.unwrap().unwrap(); + } + + async fn insert(&self, instance: &Instance) -> Uuid { + let mut storage = self.db.acquire().await.unwrap(); + assert!(storage.insert_instance_if_absent(instance).await.unwrap()); + instance.instance_id + } + + async fn insert_graph(&self, instance_id: Uuid) -> Uuid { + let graph = Graph { + graph_id: Uuid::new_v4(), + instance_id, + status: GraphStatus::OperatorPresigned.to_string(), + definition_hash: "definition".to_string(), + operator_pubkey: "operator".to_string(), + kickoff_index: NEXT_SLOT.fetch_add(1, std::sync::atomic::Ordering::Relaxed), + ..Default::default() + }; + let mut storage = self.db.acquire().await.unwrap(); + storage.upsert_graph_definition(&graph).await.unwrap(); + graph.graph_id + } + + fn script(&self, action: &str, plan: InstancePlan) { + self.jobs.plans.lock().unwrap().insert(action.to_string(), plan); + } + + fn calls(&self) -> Vec<(String, Vec)> { + self.jobs.calls.lock().unwrap().clone() + } + + /// A PeginConfirm round message event as `inbound_event` builds it; the + /// apply only reads the envelope. + fn round_message(&self, kind: &str, author: &str, instance_id: Uuid) -> NewEvent { + let inbound = InboundMessage { + author: author.to_string(), + instance_id, + graph_nonce: None, + operator_pubkey: None, + message: format!("{kind}:{instance_id}:{author}").into_bytes(), + }; + NewEvent::new( + content_event_id(author, kind, &instance_id.to_string(), &inbound.message), + instance_ref(instance_id), + kind, + EventSource::Network, + inbound.encode().unwrap(), + ) + .with_author(author) + } + + async fn jobs_of(&self, instance_id: Uuid) -> Vec { + let mut storage = self.db.acquire().await.unwrap(); + storage + .list_engine_jobs_for_aggregate(INSTANCE_KIND, &instance_id.to_string()) + .await + .unwrap() + } + + async fn wait_job(&self, instance_id: Uuid, action: &str, state: &str) -> store::EngineJob { + wait_for(|| async { + self.jobs_of(instance_id) + .await + .into_iter() + .find(|job| job.action == action && job.state == state) + }) + .await + } + + async fn outbox(&self, outbox_id: &str) -> Option { + self.db.acquire().await.unwrap().find_engine_outbox_message(outbox_id).await.unwrap() + } + + async fn wait_outbox(&self, outbox_id: &str, state: &str) -> store::EngineOutboxMessage { + wait_for(|| async { self.outbox(outbox_id).await.filter(|row| row.state == state) }).await + } + + async fn insert_open_outbox(&self, instance_id: Uuid) -> String { + let outbox_id = format!("outbox:{instance_id}"); + let mut storage = self.db.acquire().await.unwrap(); + storage + .insert_engine_outbox_message( + &NewEngineOutboxMessage { + outbox_id: outbox_id.clone(), + aggregate_kind: INSTANCE_KIND.to_string(), + aggregate_id: instance_id.to_string(), + phase_ref: PHASE_CONFIRM.to_string(), + kind: "PeginConfirmNonce".to_string(), + topic: "topic".to_string(), + payload: b"nonce".to_vec(), + ack_set: None, + first_at: 1, + }, + 1, + ) + .await + .unwrap(); + outbox_id + } + + async fn submit(&self, event: NewEvent) { + self.handle.submit(event).await.unwrap(); + } + + async fn submit_id(&self, event: NewEvent) -> String { + let event_id = event.event_id.clone(); + self.handle.submit(event).await.unwrap(); + event_id + } + + /// A setup session of the operator's graph that has no row yet, with an + /// open delivery and a pending job of that graph. + async fn insert_pending_session(&self, instance_id: Uuid, graph_id: Uuid) -> (String, String) { + use crate::aggregates::graph::GRAPH_KIND; + let mut storage = self.db.acquire().await.unwrap(); + storage.upsert_pending_graph_init(&instance_id, "operator", &graph_id).await.unwrap(); + let outbox_id = format!("graph:{graph_id}:InitGraph"); + storage + .insert_engine_outbox_message( + &NewEngineOutboxMessage { + outbox_id: outbox_id.clone(), + aggregate_kind: GRAPH_KIND.to_string(), + aggregate_id: graph_id.to_string(), + phase_ref: "Setup".to_string(), + kind: "InitGraph".to_string(), + topic: "topic".to_string(), + payload: b"init".to_vec(), + ack_set: None, + first_at: 1, + }, + 1, + ) + .await + .unwrap(); + let job_id = format!("job:{graph_id}"); + let far = crate::clock::current_time_secs() + 3_600; + storage + .insert_engine_job( + &NewEngineJob { + job_id: job_id.clone(), + aggregate_kind: GRAPH_KIND.to_string(), + aggregate_id: graph_id.to_string(), + phase: "graph".to_string(), + action: "freeze_candidates".to_string(), + pool: "Network".to_string(), + params: Vec::new(), + deadline_at: None, + first_at: far, + }, + 1, + ) + .await + .unwrap(); + (outbox_id, job_id) + } + + /// An instance event of a missing instance, parked on `wake_ref` as a + /// waiting start would be; once woken it re-parks on its own ref. + async fn insert_parked_event(&self, wake_ref: &str) -> String { + let event_id = format!("parked:{}", Uuid::new_v4()); + let now = crate::clock::current_time_secs(); + let mut storage = self.db.acquire().await.unwrap(); + storage + .insert_engine_inbox_event( + &NewEngineInboxEvent { + event_id: event_id.clone(), + aggregate_kind: INSTANCE_KIND.to_string(), + aggregate_id: Uuid::new_v4().to_string(), + kind: EVENT_L2_MINTED.to_string(), + source: "local".to_string(), + author: String::new(), + admission_class: String::new(), + payload: Vec::new(), + ack: None, + }, + now, + ) + .await + .unwrap(); + assert!( + storage + .mark_engine_inbox_waiting(&event_id, wake_ref, "test", now + 3_600, now) + .await + .unwrap() + ); + event_id + } + + async fn waiting_ref(&self, event_id: &str) -> Option { + self.db + .acquire() + .await + .unwrap() + .find_engine_inbox_event(event_id) + .await + .unwrap() + .unwrap() + .waiting_ref + } + + async fn snapshot(&self, instance_id: Uuid, snapshot: BtcSnapshot) { + self.submit(snapshot.event(instance_id).unwrap()).await; + } + + async fn instance(&self, instance_id: Uuid) -> Instance { + self.db.acquire().await.unwrap().find_instance(&instance_id).await.unwrap().unwrap() + } + + async fn wait_status(&self, instance_id: Uuid, status: InstanceBridgeInStatus) -> Instance { + wait_for(|| async { + let instance = self.instance(instance_id).await; + (instance.status == status.to_string()).then_some(instance) + }) + .await + } + + async fn wait_key_deleted(&self, instance_id: Uuid) { + wait_for(|| async { self.instance(instance_id).await.key_deleted.then_some(()) }).await; + } + + async fn wait_applied(&self, event_id: &str) { + wait_for(|| async { + let mut storage = self.db.acquire().await.unwrap(); + storage + .find_engine_inbox_event(event_id) + .await + .unwrap() + .filter(|row| row.state == EngineInboxState::Applied.to_string()) + .map(|_| ()) + }) + .await; + } + + async fn graph_status(&self, graph_id: Uuid) -> String { + self.db.acquire().await.unwrap().find_graph(&graph_id).await.unwrap().unwrap().status + } + + async fn outbox_state(&self, outbox_id: &str) -> String { + self.db + .acquire() + .await + .unwrap() + .find_engine_outbox_message(outbox_id) + .await + .unwrap() + .unwrap() + .state + } + + async fn job_actions(&self, instance_id: Uuid) -> Vec { + let mut storage = self.db.acquire().await.unwrap(); + storage + .list_engine_jobs_for_aggregate(INSTANCE_KIND, &instance_id.to_string()) + .await + .unwrap() + .into_iter() + .map(|job| job.action) + .collect() + } + + async fn engine_status(&self, instance_id: Uuid) -> (String, String) { + let row = self.handle.aggregate(&instance_ref(instance_id)).await.unwrap().unwrap(); + (row.status, row.phase) + } +} + +async fn wait_for(mut probe: F) -> T +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let started = tokio::time::Instant::now(); + loop { + if let Some(value) = probe().await { + return value; + } + assert!(started.elapsed() < WAIT, "condition not met within {WAIT:?}"); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +async fn settle() { + tokio::time::sleep(Duration::from_millis(400)).await; +} + +fn at(height: i64, status: InstanceBridgeInStatus) -> BtcSnapshot { + BtcSnapshot { height, status_at_scan: status.to_string(), ..Default::default() } +} + +fn presign_snapshot(height: i64, confirm: TxFact, cancel: TxFact) -> BtcSnapshot { + BtcSnapshot { + confirm: Some(confirm), + cancel: Some(cancel), + ..at(height, InstanceBridgeInStatus::Presigned) + } +} + +// ----------------------------------------------------------------- failures + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_discarded_enters_failure_with_every_effect() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, true).await; + let id = t.insert(&instance_row(CommitteesAnswered, 0)).await; + let graph_id = t.insert_graph(id).await; + let outbox_id = t.insert_open_outbox(id).await; + + let mut snapshot = at(10, CommitteesAnswered); + snapshot.inputs_available = Some(false); + t.snapshot(id, snapshot).await; + + t.wait_status(id, UserDiscarded).await; + assert_eq!(t.graph_status(graph_id).await, GraphStatus::Obsoleted.to_string()); + assert_eq!(t.outbox_state(&outbox_id).await, "Closed"); + t.wait_key_deleted(id).await; + assert_eq!(t.job_actions(id).await, vec![JOB_DELETE_INSTANCE_KEY.to_string()]); + assert_eq!(t.engine_status(id).await, (UserDiscarded.to_string(), PHASE_FAILED.to_string())); + + // Failure is terminal: a later Prepare confirmation changes nothing. + let mut late = at(20, UserDiscarded); + late.prepare = Some(TxFact::confirmed_at(19)); + t.snapshot(id, late).await; + settle().await; + assert_eq!(t.instance(id).await.status, UserDiscarded.to_string()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn no_enough_committees_answered_when_the_window_closes_below_quorum() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, true).await; + let id = t.insert(&instance_row(UserInited, 0)).await; + let mut answers = IndexMap::new(); + answers.insert("0xa".to_string(), vec![1u8; 33]); + let window = AnswerWindow { goat_height: 200, window_blocks: 50, quorum: 2, answers }; + t.submit(window.event(id).unwrap()).await; + let instance = t.wait_status(id, NoEnoughCommitteesAnswered).await; + assert_eq!(instance.committees_answers.len(), 1); + t.wait_key_deleted(id).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn committees_answered_at_quorum_keeps_the_key() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, true).await; + let id = t.insert(&instance_row(UserInited, 0)).await; + let mut answers = IndexMap::new(); + answers.insert("0xa".to_string(), vec![1u8; 33]); + answers.insert("0xb".to_string(), vec![2u8; 33]); + let window = AnswerWindow { goat_height: 200, window_blocks: 50, quorum: 2, answers }; + t.submit(window.event(id).unwrap()).await; + let instance = t.wait_status(id, CommitteesAnswered).await; + assert_eq!(instance.committees_answers.len(), 2); + assert!(!instance.key_deleted); + assert!(t.job_actions(id).await.is_empty()); + assert_eq!(t.engine_status(id).await.1, PHASE_AWAIT_PREPARE); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_canceled_from_the_confirm_phase_deletes_the_key() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, true, true).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + let graph_id = t.insert_graph(id).await; + t.snapshot(id, presign_snapshot(1_070, TxFact::UNKNOWN, TxFact::confirmed_at(1_069))).await; + t.wait_status(id, UserCanceled).await; + assert_eq!(t.graph_status(graph_id).await, GraphStatus::Obsoleted.to_string()); + t.wait_key_deleted(id).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failed_instance_keeps_the_settlement_jobs_of_its_graphs() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, true, true).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + let graph_id = t.insert_graph(id).await; + let job = |job_id: &str, phase: &str, action: &str| store::NewEngineJob { + job_id: job_id.to_string(), + aggregate_kind: "graph".to_string(), + aggregate_id: graph_id.to_string(), + phase: phase.to_string(), + action: action.to_string(), + pool: "network".to_string(), + params: Vec::new(), + deadline_at: None, + first_at: i64::MAX, + }; + { + let mut storage = t.db.acquire().await.unwrap(); + for (job_id, phase, action) in [ + ("setup", crate::aggregates::graph::JOB_PHASE, "committee_presign"), + ("post", crate::aggregates::graph::PHASE_POST, "post_graph_data"), + ("challenge", "Settlement", "challenge_kickoff"), + ] { + assert!( + storage + .insert_engine_job(&job(job_id, phase, action), 1) + .await + .unwrap() + .is_inserted() + ); + } + } + t.snapshot(id, presign_snapshot(1_070, TxFact::UNKNOWN, TxFact::confirmed_at(1_069))).await; + t.wait_status(id, UserCanceled).await; + assert_eq!(t.graph_status(graph_id).await, GraphStatus::Obsoleted.to_string()); + let mut storage = t.db.acquire().await.unwrap(); + for (job_id, state) in [("setup", "Cancelled"), ("post", "Cancelled"), ("challenge", "Pending")] + { + assert_eq!( + storage.find_engine_job(job_id).await.unwrap().unwrap().state, + state, + "{job_id}" + ); + } + drop(storage); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn timeout_keeps_the_key_until_cancel_confirms() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, true, true).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + let graph_id = t.insert_graph(id).await; + let outbox_id = t.insert_open_outbox(id).await; + + // Buffer not over: nothing happens. + t.snapshot(id, presign_snapshot(1_065, TxFact::UNKNOWN, TxFact::UNKNOWN)).await; + settle().await; + assert_eq!(t.instance(id).await.status, Presigned.to_string()); + + // Abandon height, Peg-In-Confirm neither confirmed nor in the mempool. + t.snapshot(id, presign_snapshot(1_066, TxFact::UNKNOWN, TxFact::UNKNOWN)).await; + let instance = t.wait_status(id, Timeout).await; + assert!(!instance.key_deleted); + assert_eq!(t.graph_status(graph_id).await, GraphStatus::Obsoleted.to_string()); + assert_eq!(t.outbox_state(&outbox_id).await, "Closed"); + assert!(t.job_actions(id).await.is_empty(), "Timeout must not delete the key"); + + // A late Peg-In-Confirm only alerts. + let mut late = at(1_080, Timeout); + late.confirm = Some(TxFact::confirmed_at(1_079)); + late.cancel = Some(TxFact::UNKNOWN); + t.snapshot(id, late).await; + settle().await; + assert_eq!(t.instance(id).await.status, Timeout.to_string()); + + // Cancel confirmed: UserCanceled and the key goes. + let mut cancel = at(1_090, Timeout); + cancel.confirm = Some(TxFact::UNKNOWN); + cancel.cancel = Some(TxFact::confirmed_at(1_089)); + t.snapshot(id, cancel).await; + t.wait_status(id, UserCanceled).await; + t.wait_key_deleted(id).await; + t.stop().await; +} + +// ------------------------------------------------------------------ success + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn mempool_visibility_defers_abandon_and_confirmation_completes_the_pegin() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, true, true).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + + // Past the abandon height but visible in the mempool: keep waiting. + t.snapshot(id, presign_snapshot(1_066, TxFact::mempool(), TxFact::UNKNOWN)).await; + t.snapshot(id, presign_snapshot(1_100, TxFact::mempool(), TxFact::UNKNOWN)).await; + settle().await; + assert_eq!(t.instance(id).await.status, Presigned.to_string()); + + // Confirmed late: L1 broadcast state, relayer post job, then the mint. + t.snapshot(id, presign_snapshot(1_101, TxFact::confirmed_at(1_101), TxFact::UNKNOWN)).await; + t.wait_status(id, RelayerL1Broadcasted).await; + assert_eq!(t.engine_status(id).await.1, PHASE_POST); + wait_for(|| async { + t.job_actions(id).await.contains(&JOB_POST_PEGIN_DATA.to_string()).then_some(()) + }) + .await; + + t.submit(l2_minted_event(id, "0xmint").unwrap()).await; + let instance = t.wait_status(id, RelayerL2Minted).await; + assert_eq!(instance.post_pegin_txhash.as_deref(), Some("0xmint")); + assert_eq!(t.engine_status(id).await.1, PHASE_DONE); + + // Success-side key deletion once the depth is reached. + let mut deep = at(1_133, RelayerL2Minted); + deep.confirm = Some(TxFact::confirmed_at(1_101)); + t.snapshot(id, deep).await; + t.wait_key_deleted(id).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn prepare_confirmation_records_the_height_and_triggers_confirm_instance() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Operator, false, false).await; + let id = t.insert(&instance_row(CommitteesAnswered, 0)).await; + let mut snapshot = at(500, CommitteesAnswered); + snapshot.prepare = Some(TxFact::confirmed_at(480)); + t.snapshot(id, snapshot).await; + let instance = t.wait_status(id, UserBroadcastPeginPrepare).await; + assert_eq!(instance.btc_height, 480); + // Every role stores the parameters through the Presign-phase job. + let job = t.wait_job(id, JOB_CONFIRM_INSTANCE, "Done").await; + assert_eq!(job.phase, PHASE_PRESIGN); + assert_eq!(t.engine_status(id).await.1, PHASE_PRESIGN); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_confirm_plan_opens_the_operator_session_and_starts_the_graph() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Operator, false, false).await; + let id = t.insert(&instance_row(CommitteesAnswered, 0)).await; + let graph_id = Uuid::new_v4(); + let plan = InstancePlan { + start_graph: Some(StartGraph { graph_id, operator_pubkey: "operator".to_string() }), + ..Default::default() + }; + t.script(JOB_CONFIRM_INSTANCE, plan); + let mut snapshot = at(500, CommitteesAnswered); + snapshot.prepare = Some(TxFact::confirmed_at(480)); + t.snapshot(id, snapshot).await; + t.wait_job(id, JOB_CONFIRM_INSTANCE, "Done").await; + // The session and the Graph aggregate's start event are committed together. + let start_id = + crate::aggregates::graph::operator_start_event(id, graph_id, "operator").unwrap().event_id; + wait_for(|| async { + let mut storage = t.db.acquire().await.unwrap(); + storage.find_engine_inbox_event(&start_id).await.unwrap().map(|_| ()) + }) + .await; + let session = + t.db.acquire() + .await + .unwrap() + .find_pending_graph_init_by_instance_and_operator_pubkey(&id, "operator") + .await + .unwrap() + .expect("setup session opened"); + assert_eq!(session.graph_id, graph_id); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_pegin_request_is_answered_by_committee_nodes_until_the_window_closes() { + use InstanceBridgeInStatus::*; + let request = PeginRequested { tx_hash: "0xreq".to_string() }; + + let t = start(Actor::Operator, false, false).await; + let id = t.insert(&instance_row(UserInited, 0)).await; + let event = pegin_request_event(id, &request).unwrap(); + let event_id = event.event_id.clone(); + t.submit(event).await; + t.wait_applied(&event_id).await; + assert!(t.job_actions(id).await.is_empty(), "only committee nodes answer"); + t.stop().await; + + let t = start(Actor::Committee, false, false).await; + let id = t.insert(&instance_row(UserInited, 0)).await; + t.submit(pegin_request_event(id, &request).unwrap()).await; + let job = t.wait_job(id, JOB_ANSWER_PEGIN_REQUEST, "Done").await; + assert_eq!(job.phase, PHASE_ANSWER); + assert_eq!(t.calls().len(), 1); + // A request seen again after the Answer phase changes nothing. + let mut answers = IndexMap::new(); + answers.insert("0xa".to_string(), vec![1u8; 33]); + let window = AnswerWindow { goat_height: 200, window_blocks: 50, quorum: 1, answers }; + t.submit(window.event(id).unwrap()).await; + t.wait_status(id, CommitteesAnswered).await; + let late = PeginRequested { tx_hash: "0xagain".to_string() }; + let event = pegin_request_event(id, &late).unwrap(); + let event_id = event.event_id.clone(); + t.submit(event).await; + t.wait_applied(&event_id).await; + assert_eq!(t.calls().len(), 1, "no second answer after the window"); + t.stop().await; +} + +// -------------------------------------------------------- pegin confirm round + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_round_message_becomes_an_ingest_job_whose_plan_is_sent_until_the_confirm() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, false).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + // Entering Presigned is what starts the round; here the row starts there, + // so the start job is not enqueued: the member's nonce arrives first. + let outbox_id = format!("instance:{id}:PeginConfirmNonceConsensus"); + let mut plan = InstancePlan::default(); + plan.send(PlannedMessage { + outbox_id: outbox_id.clone(), + phase_ref: PHASE_CONFIRM.to_string(), + kind: "PeginConfirmNonceConsensus".to_string(), + actor: Actor::Committee.to_string(), + payload: b"vote".to_vec(), + ack_set: None, + }) + .then(JOB_PEGIN_CONFIRM_VOTE, Vec::new(), None); + t.script(JOB_INGEST_PEGIN_CONFIRM_NONCE, plan); + + t.submit(t.round_message("PeginConfirmNonce", OTHER_PEER, id)).await; + let ingest = t.wait_job(id, JOB_INGEST_PEGIN_CONFIRM_NONCE, "Done").await; + assert_eq!(ingest.phase, PHASE_CONFIRM); + assert!(ingest.deadline_at.is_some(), "a parked message does not wait forever"); + let row = t.wait_outbox(&outbox_id, "Open").await; + assert_eq!(row.phase_ref, PHASE_CONFIRM); + assert_eq!(row.aggregate_id, id.to_string()); + t.wait_job(id, JOB_PEGIN_CONFIRM_VOTE, "Done").await; + + // The same content again is a duplicate: no second ingest. + let duplicate = t.round_message("PeginConfirmNonce", OTHER_PEER, id); + t.submit(duplicate).await; + settle().await; + assert_eq!( + t.calls().iter().filter(|(action, _)| action == JOB_INGEST_PEGIN_CONFIRM_NONCE).count(), + 1 + ); + + // Peg-In-Confirm confirmed: the Confirm phase's deliveries close. + t.snapshot(id, presign_snapshot(1_050, TxFact::confirmed_at(1_049), TxFact::UNKNOWN)).await; + t.wait_status(id, RelayerL1Broadcasted).await; + t.wait_outbox(&outbox_id, "Closed").await; + // A late partial signature is still ingested (it carries the endorsement + // the relayer's post needs); a late nonce is not. + t.submit(t.round_message("PeginConfirmPartialSig", OTHER_PEER, id)).await; + t.wait_job(id, JOB_INGEST_PEGIN_CONFIRM_PARTIAL_SIG, "Done").await; + let late = t.round_message("PeginConfirmNonceConsensus", OTHER_PEER, id); + let late_id = t.submit_id(late).await; + t.wait_applied(&late_id).await; + assert!(!t.job_actions(id).await.contains(&JOB_INGEST_PEGIN_CONFIRM_VOTE.to_string())); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn round_messages_from_this_node_outsiders_and_closed_rounds_are_ignored() { + use InstanceBridgeInStatus::*; + // This node's own message. + let t = start(Actor::Committee, false, false).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + let own = t.submit_id(t.round_message("PeginConfirmNonce", LOCAL_PEER, id)).await; + t.wait_applied(&own).await; + assert!(t.job_actions(id).await.is_empty()); + // A round that is over. + let done = t.insert(&instance_row(RelayerL2Minted, PREPARE)).await; + let late = t.submit_id(t.round_message("PeginConfirmPartialSig", OTHER_PEER, done)).await; + t.wait_applied(&late).await; + assert!(t.job_actions(done).await.is_empty()); + let failed = t.insert(&instance_row(UserCanceled, PREPARE)).await; + let late = t.submit_id(t.round_message("PeginConfirmNonce", OTHER_PEER, failed)).await; + t.wait_applied(&late).await; + assert!(t.job_actions(failed).await.is_empty()); + t.stop().await; + + // A committee node outside the instance's committee. + let t = start_with(Actor::Committee, false, false, Arc::new(Outsider)).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + let event = t.submit_id(t.round_message("PeginConfirmNonce", OTHER_PEER, id)).await; + t.wait_applied(&event).await; + assert!(t.job_actions(id).await.is_empty()); + t.stop().await; + + // A relayer outside the committee still collects the round: it + // aggregates and broadcasts for committees it does not sign in. + let t = start_with(Actor::Committee, true, false, Arc::new(Outsider)).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + let event = t.submit_id(t.round_message("PeginConfirmNonce", OTHER_PEER, id)).await; + t.wait_applied(&event).await; + t.wait_job(id, JOB_INGEST_PEGIN_CONFIRM_NONCE, "Done").await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn entering_presigned_starts_the_round_and_failure_cancels_it() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, true).await; + let id = t.insert(&instance_row(UserBroadcastPeginPrepare, PREPARE)).await; + let graph_id = t.insert_graph(id).await; + mark_graph_as_endorsed(&t.db, id, graph_id).await.unwrap(); + let mut plan = InstancePlan::default(); + plan.send(PlannedMessage { + outbox_id: format!("instance:{id}:PeginConfirmNonce"), + phase_ref: PHASE_CONFIRM.to_string(), + kind: "PeginConfirmNonce".to_string(), + actor: Actor::Committee.to_string(), + payload: b"nonce".to_vec(), + ack_set: None, + }); + t.script(JOB_START_PEGIN_CONFIRM, plan); + t.submit(graph_endorsed_event(id, graph_id).unwrap()).await; + t.wait_status(id, Presigned).await; + t.wait_job(id, JOB_START_PEGIN_CONFIRM, "Done").await; + t.wait_outbox(&format!("instance:{id}:PeginConfirmNonce"), "Open").await; + + // The user cancels: the round's deliveries close and its jobs go. + let pending = t.submit_id(t.round_message("PeginConfirmNonce", OTHER_PEER, id)).await; + t.wait_applied(&pending).await; + t.snapshot(id, presign_snapshot(1_070, TxFact::UNKNOWN, TxFact::confirmed_at(1_069))).await; + t.wait_status(id, UserCanceled).await; + t.wait_outbox(&format!("instance:{id}:PeginConfirmNonce"), "Closed").await; + // The round's ingest job ends Cancelled, or Done when it had already run; + // its finish may still be waiting for the write lock when the outbox closes. + wait_for(|| async { + let jobs = t.jobs_of(id).await; + let ingest: Vec<_> = + jobs.iter().filter(|job| job.action == JOB_INGEST_PEGIN_CONFIRM_NONCE).collect(); + (!ingest.is_empty() + && ingest.iter().all(|job| job.state == "Cancelled" || job.state == "Done")) + .then_some(()) + }) + .await; + t.wait_key_deleted(id).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_relayer_rebroadcasts_an_invisible_confirm_once_per_block() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, true, false).await; + let mut row = instance_row(Presigned, PREPARE); + row.committees_answers.insert("0xa".to_string(), vec![1u8; 33]); + let id = t.insert(&row).await; + + // Not every partial signature is stored: nothing to aggregate. + t.snapshot(id, presign_snapshot(1_010, TxFact::UNKNOWN, TxFact::UNKNOWN)).await; + settle().await; + assert!(t.job_actions(id).await.is_empty()); + + let secp = bitcoin::secp256k1::Secp256k1::new(); + let keypair = bitcoin::secp256k1::Keypair::from_secret_key( + &secp, + &bitcoin::secp256k1::SecretKey::from_slice(&[9; 32]).unwrap(), + ); + store_committee_partial_sig_for_instance( + &t.db, + id, + bitcoin::PublicKey::new(keypair.public_key()), + musig2::PartialSignature::one(), + ) + .await + .unwrap(); + t.snapshot(id, presign_snapshot(1_011, TxFact::UNKNOWN, TxFact::UNKNOWN)).await; + t.wait_job(id, JOB_BROADCAST_PEGIN_CONFIRM, "Done").await; + // Another block, still invisible: another broadcast; visible: none. + t.snapshot(id, presign_snapshot(1_012, TxFact::UNKNOWN, TxFact::UNKNOWN)).await; + wait_for(|| async { + (t.calls().iter().filter(|(action, _)| action == JOB_BROADCAST_PEGIN_CONFIRM).count() == 2) + .then_some(()) + }) + .await; + t.snapshot(id, presign_snapshot(1_013, TxFact::mempool(), TxFact::UNKNOWN)).await; + settle().await; + assert_eq!( + t.calls().iter().filter(|(action, _)| action == JOB_BROADCAST_PEGIN_CONFIRM).count(), + 2 + ); + // A non-relayer committee node never broadcasts. + t.stop().await; + let t = start(Actor::Committee, false, false).await; + let mut row = instance_row(Presigned, PREPARE); + row.committees_answers.insert("0xa".to_string(), vec![1u8; 33]); + let id = t.insert(&row).await; + store_committee_partial_sig_for_instance( + &t.db, + id, + bitcoin::PublicKey::new(keypair.public_key()), + musig2::PartialSignature::one(), + ) + .await + .unwrap(); + t.snapshot(id, presign_snapshot(1_011, TxFact::UNKNOWN, TxFact::UNKNOWN)).await; + settle().await; + assert!(t.job_actions(id).await.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_endorsed_graph_reaches_presigned_only_from_the_presign_phase() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, false).await; + let id = t.insert(&instance_row(UserBroadcastPeginPrepare, PREPARE)).await; + let graph_id = t.insert_graph(id).await; + + // Not endorsed yet: the threshold is not met. + t.submit(graph_endorsed_event(id, graph_id).unwrap()).await; + settle().await; + assert_eq!(t.instance(id).await.status, UserBroadcastPeginPrepare.to_string()); + + mark_graph_as_endorsed(&t.db, id, graph_id).await.unwrap(); + let other = t.insert_graph(id).await; + t.submit(graph_endorsed_event(id, other).unwrap()).await; + t.wait_status(id, Presigned).await; + assert_eq!(t.engine_status(id).await.1, PHASE_CONFIRM); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_endorsement_before_the_presign_phase_waits_for_it() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, false).await; + let id = t.insert(&instance_row(CommitteesAnswered, 0)).await; + let graph_id = t.insert_graph(id).await; + mark_graph_as_endorsed(&t.db, id, graph_id).await.unwrap(); + let event = graph_endorsed_event(id, graph_id).unwrap(); + let event_id = event.event_id.clone(); + t.submit(event).await; + // Parked on the instance until it reaches the Presign phase. + let waiting = wait_for(|| async { t.waiting_ref(&event_id).await }).await; + assert_eq!(waiting, instance_ref(id).wake_ref()); + assert_eq!(t.instance(id).await.status, CommitteesAnswered.to_string()); + // Prepare confirms: the status change wakes the endorsement, which now counts. + let mut snapshot = at(500, CommitteesAnswered); + snapshot.prepare = Some(TxFact::confirmed_at(480)); + t.snapshot(id, snapshot).await; + t.wait_status(id, Presigned).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_mint_for_a_failed_instance_leaves_it_failed() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, true, true).await; + let id = t.insert(&instance_row(UserDiscarded, PREPARE)).await; + let event = l2_minted_event(id, "0xmint").unwrap(); + let event_id = event.event_id.clone(); + t.submit(event).await; + t.wait_applied(&event_id).await; + let instance = t.instance(id).await; + assert_eq!(instance.status, UserDiscarded.to_string()); + assert_eq!(instance.post_pegin_txhash, None); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn only_committee_nodes_delete_keys() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Operator, false, true).await; + let id = t.insert(&instance_row(CommitteesAnswered, 0)).await; + let mut snapshot = at(10, CommitteesAnswered); + snapshot.inputs_available = Some(false); + t.snapshot(id, snapshot).await; + let instance = t.wait_status(id, UserDiscarded).await; + settle().await; + assert!(!instance.key_deleted); + assert!(t.job_actions(id).await.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn disabled_key_deletion_only_records_the_intent() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, false).await; + let id = t.insert(&instance_row(Presigned, PREPARE)).await; + t.snapshot(id, presign_snapshot(1_070, TxFact::UNKNOWN, TxFact::confirmed_at(1_069))).await; + t.wait_status(id, UserCanceled).await; + settle().await; + assert!(!t.instance(id).await.key_deleted); + assert!(t.job_actions(id).await.is_empty()); + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_missing_instance_row_parks_the_event_until_it_exists() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Committee, false, false).await; + let instance = instance_row(Presigned, PREPARE); + let id = instance.instance_id; + let event = + presign_snapshot(1_070, TxFact::UNKNOWN, TxFact::confirmed_at(1_069)).event(id).unwrap(); + let event_id = event.event_id.clone(); + t.submit(event).await; + wait_for(|| async { + let mut storage = t.db.acquire().await.unwrap(); + storage + .find_engine_inbox_event(&event_id) + .await + .unwrap() + .filter(|row| row.waiting_ref.as_deref() == Some(instance_ref(id).wake_ref().as_str())) + .map(|_| ()) + }) + .await; + t.insert(&instance).await; + t.handle.notify(&[instance_ref(id).wake_ref()]).await.unwrap(); + t.wait_status(id, UserCanceled).await; + t.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failed_instance_releases_its_pending_setup_session() { + use InstanceBridgeInStatus::*; + let t = start(Actor::Operator, false, false).await; + let id = t.insert(&instance_row(CommitteesAnswered, 0)).await; + let graph_id = Uuid::new_v4(); + let (outbox_id, job_id) = t.insert_pending_session(id, graph_id).await; + // The next start of this operator waits on the slot the session is taking. + let slot = crate::aggregates::graph::slot_ref("operator", 0); + let parked = t.insert_parked_event(&slot).await; + assert_eq!(t.waiting_ref(&parked).await.as_deref(), Some(slot.as_str())); + + let mut snapshot = at(10, CommitteesAnswered); + snapshot.inputs_available = Some(false); + t.snapshot(id, snapshot).await; + t.wait_status(id, UserDiscarded).await; + + // The session is gone with its delivery and job, and the slot is released: + // the parked event was woken, re-applied, and parked on its own ref. + assert!( + t.db.acquire() + .await + .unwrap() + .find_pending_graph_init_by_graph_id(&graph_id) + .await + .unwrap() + .is_none() + ); + assert_eq!(t.outbox_state(&outbox_id).await, "Closed"); + let job = t.db.acquire().await.unwrap().find_engine_job(&job_id).await.unwrap().unwrap(); + assert_eq!(job.state, "Cancelled"); + wait_for(|| async { + let woken = t.waiting_ref(&parked).await; + (woken.as_deref() != Some(slot.as_str())).then_some(()) + }) + .await; + t.stop().await; +} diff --git a/node/src/aggregates/jobs.rs b/node/src/aggregates/jobs.rs new file mode 100644 index 000000000..6a9d1e878 --- /dev/null +++ b/node/src/aggregates/jobs.rs @@ -0,0 +1,173 @@ +//! What the Instance and Graph jobs share: their handles, the step a job +//! returns and the lookups every round makes. + +use super::common::InboundMessage; +use super::instance::{SigningGate, signing_gate}; +use crate::env::get_bitvm_key; +use crate::messages::GOATMessage; +use crate::metrics_service::MetricsState; +use crate::middleware::publisher::NetworkPublisher; +use crate::utils::{CommitteeMembership, local_committee_membership, validate_committee}; +use anyhow::{Context, Result}; +use bitcoin::PublicKey; +use bitcoin::key::Keypair; +use bitvm_lib::actors::Actor; +use bitvm_lib::babe_adapter::BabeBundleBuilder; +use bitvm_lib::keys::OperatorMasterKey; +use client::btc_chain::BTCClient; +use client::goat_chain::GOATClient; +use client::http_client::async_client::HttpAsyncClient; +use engine::{JobContext, JobOutcome}; +use libp2p::PeerId; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::str::FromStr; +use std::sync::Arc; +use store::InstanceBridgeInStatus; +use store::localdb::LocalDB; +use tracing::debug; +use uuid::Uuid; + +/// The handles a job runs with. +pub struct JobDeps { + pub actor: Actor, + pub relayer: bool, + pub local_db: LocalDB, + pub btc_client: Arc, + pub goat_client: Arc, + /// Proof builder requests of the peg-out jobs. + pub http_client: Arc, + pub soldering_builder: Option>, + pub metrics_state: MetricsState, + pub publisher: NetworkPublisher, +} + +/// What a job decided, encoded as its result for the apply. +pub trait Plan: Serialize { + /// A plan that only notes why nothing is done. + fn noted(note: impl Into) -> Self; +} + +/// What one step decided. +pub enum Step

{ + Plan(Box

), + Retry { reason: String, after_secs: i64 }, +} + +impl Step

{ + pub fn done(plan: P) -> Result { + Ok(Step::Plan(Box::new(plan))) + } + + pub fn ignore(note: impl Into) -> Result { + Self::done(P::noted(note)) + } + + pub fn retry(reason: impl Into, after_secs: i64) -> Result { + Ok(Step::Retry { reason: reason.into(), after_secs }) + } + + pub fn into_outcome(self) -> JobOutcome { + match self { + Step::Plan(plan) => plan_outcome(&*plan), + Step::Retry { reason, after_secs } => { + JobOutcome::Transient { error: reason, retry_after_secs: Some(after_secs) } + } + } + } +} + +/// The outcome that hands a plan to the apply. +pub fn plan_outcome(plan: &impl Serialize) -> JobOutcome { + match bincode::serialize(plan) { + Ok(bytes) => JobOutcome::Done(bytes), + Err(error) => JobOutcome::Failed(format!("encode plan: {error}")), + } +} + +/// The aggregate id of a job as a UUID; `what` names the aggregate in the +/// failure ("a graph", "an instance"). +pub(super) fn aggregate_uuid(ctx: &JobContext, what: &str) -> Result { + Uuid::parse_str(&ctx.job.aggregate_id).map_err(|error| { + JobOutcome::Failed(format!( + "job aggregate id {} is not {what} id: {error}", + ctx.job.aggregate_id + )) + }) +} + +// ------------------------------------------------------------------ lookups + +pub(super) async fn decode_inbound(params: &[u8]) -> Result<(InboundMessage, GOATMessage)> { + let inbound = InboundMessage::decode(params).context("decode inbound message")?; + let message = GOATMessage::deserialize_message(&inbound.message).await?; + Ok((inbound, message)) +} + +pub(super) fn author_peer(inbound: &InboundMessage) -> Result { + PeerId::from_str(&inbound.author).context("decode author peer id") +} + +pub(super) async fn instance_status( + deps: &JobDeps, + instance_id: Uuid, +) -> Result> { + Ok(deps + .local_db + .acquire() + .await? + .find_instance(&instance_id) + .await? + .and_then(|instance| InstanceBridgeInStatus::from_str(&instance.status).ok())) +} + +/// The instance keypair when this node is in the instance's committee on +/// Goat; `None` means the instance is not this node's to sign for. Only the +/// node's own steps ask: a value from a member is stored by whoever the +/// apply let it through to (the members, and a relayer for every round). +pub(super) async fn local_member(deps: &JobDeps, instance_id: Uuid) -> Result> { + match local_committee_membership(&deps.goat_client, instance_id).await? { + CommitteeMembership::Member(keypair) => Ok(Some(keypair)), + CommitteeMembership::NotMember(reason) => { + debug!(event = "job", instance_id = %instance_id, reason, "not in this instance's committee"); + Ok(None) + } + } +} + +pub(super) async fn committee_pubkeys(deps: &JobDeps, instance_id: Uuid) -> Result> { + deps.goat_client.gateway_get_committee_pubkeys(&instance_id).await +} + +/// The T0 gate for the committee's signing steps: `None` when open. +pub(super) async fn gate_closed(deps: &JobDeps, instance_id: Uuid) -> Result> { + Ok(match signing_gate(&deps.local_db, &deps.btc_client, instance_id).await? { + SigningGate::Open => None, + SigningGate::Closed(reason) => Some(reason), + }) +} + +/// The local operator master key and its public key. +pub(super) fn local_operator() -> Result<(OperatorMasterKey, PublicKey)> { + let master = OperatorMasterKey::new(get_bitvm_key()?); + let pubkey = master.master_keypair().public_key().into(); + Ok((master, pubkey)) +} + +pub(super) fn signing_nonce_id(label: &str, bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(label.as_bytes()); + hasher.update(bytes); + hex::encode(hasher.finalize()) +} + +/// The committee value really comes from the member it names (pubkey and +/// peer id bound on Goat). +pub(super) async fn ensure_committee_sender( + deps: &JobDeps, + inbound: &InboundMessage, + committee_pubkey: &PublicKey, +) -> Result<()> { + let peer = author_peer(inbound)?; + validate_committee(&deps.goat_client, &peer, inbound.instance_id, committee_pubkey).await +} diff --git a/node/src/aggregates/mod.rs b/node/src/aggregates/mod.rs new file mode 100644 index 000000000..7d2dc0072 --- /dev/null +++ b/node/src/aggregates/mod.rs @@ -0,0 +1,22 @@ +//! Instance and Graph event handlers and their jobs. +//! +//! Business data stays in the instance and graph tables; the engine manages +//! pending events, jobs, outgoing messages and dependency waits. + +pub mod common; +pub mod graph; +pub mod graph_fetch; +pub mod graph_jobs; +pub mod graph_pegout; +pub mod graph_pegout_jobs; +pub mod graph_scan; +pub mod graph_setup; +pub mod instance; +pub mod instance_jobs; +pub mod instance_scan; +pub mod jobs; + +#[cfg(test)] +mod graph_tests; +#[cfg(test)] +mod instance_tests; diff --git a/node/src/bin/db_inject.rs b/node/src/bin/db_inject.rs deleted file mode 100644 index f7c086008..000000000 --- a/node/src/bin/db_inject.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! update-db: inject a GOAT message into the local DB. -//! -//! Purpose: -//! - Insert a GOATMessageContent payload into the local SQLite DB so the node -//! can process it as if it were received from the network. -//! -//! Key args: -//! - --db-path: local SQLite path (e.g., sqlite:/tmp/bitvm-node.db) -//! - --actor: Committee | Operator | Verifier | Watchtower | All -//! - --message-json or --message-file (one required) -//! -//! Example: -//! - cargo run -p bitvm-noded --bin update-db -- \ -//! --db-path sqlite:/tmp/bitvm-node.db \ -//! --actor Operator \ -//! --message-file ./message.json -use std::fs; -use std::path::PathBuf; -use std::str::FromStr; - -use anyhow::{Context, Result, anyhow}; -use clap::Parser; - -use bitvm_lib::actors::Actor; -use bitvm_noded::action::*; -use bitvm_noded::utils::upsert_message; -use store::create_local_db; - -#[derive(Parser, Debug)] -#[command(name = "db-inject", about = "Insert a GOAT message into the local DB for processing")] -struct Args { - /// db path - #[arg(long)] - db_path: String, - - /// Target actor (Committee, Operator, Verifier, Watchtower, All) - #[arg(long, value_parser = parse_actor)] - actor: Actor, - - /// from_peer column, defaults to "Manual" - #[arg(long, default_value = "Manual")] - from_peer: String, - - /// Message weight (default 0) - #[arg(long, default_value_t = 0)] - weight: i64, - - /// Lock time in seconds before the message becomes processable (default 0) - #[arg(long, default_value_t = 0)] - lock_secs: i64, - - /// Skip update if the same message_id already exists (default false => upsert) - #[arg(long, default_value_t = false)] - skip_if_exists: bool, - - /// Inline JSON for GOATMessageContent (mutually exclusive with --message-file) - #[arg(long, conflicts_with = "message_file")] - message_json: Option, - - /// Path to a JSON file containing GOATMessageContent - #[arg(long)] - message_file: Option, -} - -fn parse_actor(raw: &str) -> std::result::Result { - Actor::from_str(raw).map_err(|_| format!("invalid actor: {raw}")) -} - -fn load_message_json(args: &Args) -> Result { - if let Some(ref inline) = args.message_json { - return Ok(inline.clone()); - } - if let Some(ref path) = args.message_file { - return fs::read_to_string(path).context("read message file"); - } - Err(anyhow!("either --message-json or --message-file is required")) -} - -#[tokio::main] -async fn main() -> Result<()> { - dotenv::dotenv().ok(); - let args = Args::parse(); - let raw_json = load_message_json(&args)?; - let content: GOATMessageContent = - serde_json::from_str(&raw_json).context("parse GOATMessageContent JSON")?; - - let actor = args.actor; - let message_type = content.event_type(); - let local_db = create_local_db(&args.db_path).await; - let mut storage_processor = local_db.acquire().await?; - let is_update = !args.skip_if_exists; - - upsert_message( - &mut storage_processor, - is_update, - args.from_peer.clone(), - actor.clone(), - content, - args.weight, - args.lock_secs, - ) - .await?; - - println!( - "Inserted message for actor={actor} message_type={} db_path={} update={} lock_secs={} weight={}", - message_type, args.db_path, is_update, args.lock_secs, args.weight - ); - Ok(()) -} diff --git a/node/src/bin/mock_proof_builder.rs b/node/src/bin/mock_proof_builder.rs new file mode 100644 index 000000000..9ac900d69 --- /dev/null +++ b/node/src/bin/mock_proof_builder.rs @@ -0,0 +1,473 @@ +//! Mock proof builder for the localnet (`mock-gc` only). +//! +//! Serves the exact HTTP routes and JSON the bitvm-noded Operator and Watchtower call, returning +//! proof data the node accepts. It performs *no* real proving: the operator proof is the repo's +//! structurally valid Groth16 mock with 96 public-value bytes patched to match the request (so the +//! node's `convert_ark_imm_wrap_vk` decoding, vk-hash, version and constant checks all pass), and +//! the watchtower proof is dummy bytes of the exact lengths the node validates. +//! +//! Realistic waiting: for each `(kind, graph, ...)` the first request is recorded, and the builder +//! answers `{"proof_data": null, "error": "..."}` until the configured delay has elapsed, then +//! serves the proof. The delay is a "ready_at" timestamp, never a slow HTTP response (the node has +//! no request timeout and dispatches messages one at a time). +//! +//! Auth headers are accepted without verification (logged at debug). +//! +//! ## Environment +//! - `MOCK_PROOF_DELAY_SECS` default "not ready" window for every proof (default 0) +//! - `MOCK_PROOF_OPERATOR_DELAY_SECS` override for operator proofs +//! - `MOCK_PROOF_WATCHTOWER_DELAY_SECS` override for watchtower proofs +//! - `MOCK_PROOF_RESPONSE_LATENCY_MS` optional artificial per-response latency (default 0; keep small) +//! - `GENESIS_SEQUENCER_COMMIT_TXID` REQUIRED for operator proofs (same value as the nodes) +//! - `MOCK_PROOF_OPERATOR_VK` operator proof vk-hash string (default = localnet program vk) +//! - `MOCK_PROOF_OPERATOR_ZKM_VERSION` operator proof Ziren version (default = SDK circuit version) +//! - `MOCK_PROOF_WATCHTOWER_VK` watchtower proof vk-hash string (default documented value) +//! - `MOCK_PROOF_WATCHTOWER_ZKM_VERSION`watchtower proof Ziren version (default "v1.2.7") + +use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use axum::Router; +use axum::extract::State; +use axum::http::HeaderMap; +use axum::response::Json; +use axum::routing::{get, post}; +use clap::Parser; +use proof_builder::{ + OperatorProofRequest, OperatorProofResponse, ProofData, ProofDescResponse, + WatchtowerProofRequest, WatchtowerProofResponse, WatchtowerProofTimeoutUpdateRequest, + WatchtowerProofTimeoutUpdateResponse, +}; +use zkm_sdk::{ZKMProofWithPublicValues, ZKMPublicValues}; + +/// Structurally valid curve points from the former localnet fixture. This is not a valid +/// proof of the patched public values; mock-gc deliberately bypasses the pairing verdict. +const MOCK_OPERATOR_ENCODED_PROOF: &str = "2c88102ff14026ed35c76229319008d52e8bac2cef9d6812ae73769f4b9987d52f44e70c6a42cf43142978af0aaf407cb725a8138c02cdcf7f8326967fd6d6c616e39d0cd24351eed4ca6419720f80d63d23d949b306ac950838474a25d95eef2f671368683c3ac9d59f438b3f796189c2881c62331905f1b0c66bfa5391865f1add2ca47995b8a0c63d07e8b2c2eebdc60d82f04b916008f52498d23bacdb54096a4f7725cc8956ec3b99e9397354e0ffd69162904d9d09c4d2dd28965fb1c82b92022ac1d18d7a368705eef8b1ede50f33f3e37137f9ff620fdb7ec71ead742d47f4bdc501dab4c3449f8f6d400cd9d9372b50ebb5b12dd32cc6fad30395a0"; + +fn mock_operator_template() -> ZKMProofWithPublicValues { + use sha2::{Digest, Sha256}; + let vk_hash: [u8; 32] = Sha256::digest(&*zkm_verifier::IMM_GROTH16_VK_BYTES).into(); + let groth16 = serde_json::from_value(serde_json::json!({ + "public_inputs": ["", "", ""], + "encoded_proof": MOCK_OPERATOR_ENCODED_PROOF, + "raw_proof": "", + "groth16_vkey_hash": vk_hash, + })) + .expect("mock Groth16 proof structure"); + ZKMProofWithPublicValues { + proof: zkm_sdk::ZKMProof::Groth16(groth16), + public_values: ZKMPublicValues::new(), + zkm_version: zkm_sdk::ZKM_CIRCUIT_VERSION.to_string(), + } +} +/// Default localnet operator program identity; override it to match the node configuration. +const DEFAULT_OPERATOR_VK: &str = + "0x00ba1ff974eb6e5890f238f8a11c1aeff2d5f4a68a860274bcb602c3c5c681b7"; +/// A valid 66-char vk-hash string for the watchtower proof (only the length/format is checked). +const DEFAULT_WATCHTOWER_VK: &str = + "0x0074ea3cc3dc181f156a1808459846d54700a7c4b2f6fd58a18499f2d7910e58"; +const WATCHTOWER_PROOF_SIZE: usize = 260; +const WATCHTOWER_PUBLIC_INPUTS_SIZE: usize = 36; + +#[derive(Parser, Debug)] +#[command(name = "mock-proof-builder", about = "Localnet mock proof builder for bitvm-noded")] +struct Args { + /// Address to listen on. Keep it path-less so the node's URL joins agree. + #[arg(long, default_value = "127.0.0.1:22950")] + listen: SocketAddr, +} + +struct AppState { + /// First-seen instant per proof task key -> used to gate readiness. + seen: Mutex>, + /// Watchtower task keys marked failed by the timeout endpoint (stay "not ready" forever). + failed_watchtower: Mutex>, + /// Pre-parsed operator proof template. + operator_template: ZKMProofWithPublicValues, + operator_vk: String, + operator_zkm_version: String, + watchtower_vk: String, + watchtower_zkm_version: String, + /// Internal-byte-order genesis txid, required to compute the operator constant. + genesis_txid_internal: Option<[u8; 32]>, +} + +fn env_secs(name: &str) -> Option { + let secs = std::env::var(name).ok()?.trim().parse::().ok()?; + (secs.is_finite() && secs > 0.0).then(|| Duration::from_secs_f64(secs)) +} + +fn operator_delay() -> Duration { + env_secs("MOCK_PROOF_OPERATOR_DELAY_SECS") + .or_else(|| env_secs("MOCK_PROOF_DELAY_SECS")) + .unwrap_or(Duration::ZERO) +} + +fn watchtower_delay() -> Duration { + env_secs("MOCK_PROOF_WATCHTOWER_DELAY_SECS") + .or_else(|| env_secs("MOCK_PROOF_DELAY_SECS")) + .unwrap_or(Duration::ZERO) +} + +async fn response_latency() { + let ms = std::env::var("MOCK_PROOF_RESPONSE_LATENCY_MS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(0); + if ms > 0 { + tokio::time::sleep(Duration::from_millis(ms)).await; + } +} + +/// Records the first-seen time for `key` and reports whether `delay` has elapsed since then. +fn task_ready(state: &AppState, key: &str, delay: Duration) -> bool { + let mut seen = state.seen.lock().expect("seen mutex poisoned"); + let first = seen.entry(key.to_string()).or_insert_with(Instant::now); + first.elapsed() >= delay +} + +fn log_auth_headers(kind: &str, headers: &HeaderMap) { + let pubkey = + headers.get("x-proof-auth-public-key").and_then(|v| v.to_str().ok()).unwrap_or(""); + tracing::debug!( + kind, + auth_public_key = pubkey, + "accepting proof request without auth verification" + ); +} + +/// Parse a 64-hex display txid/blockhash into internal (reversed) byte order. +fn display_hex_to_internal(display: &str) -> Option<[u8; 32]> { + let raw = hex::decode(display.trim_start_matches("0x")).ok()?; + let mut bytes: [u8; 32] = raw.try_into().ok()?; + bytes.reverse(); + Some(bytes) +} + +/// Compute the 96-byte operator guest public values for a request: +/// `btc_best_block_hash(internal) || operator_constant || included_watchtowers_bitmap`. +fn build_operator_public_values( + state: &AppState, + req: &OperatorProofRequest, +) -> Result<[u8; 96], String> { + let genesis = state + .genesis_txid_internal + .ok_or_else(|| "GENESIS_SEQUENCER_COMMIT_TXID is not set".to_string())?; + + let graph_uuid = + uuid::Uuid::parse_str(&req.graph_id).map_err(|e| format!("invalid graph_id: {e}"))?; + let graph_id_bytes = *graph_uuid.as_bytes(); + + let wci_internal = display_hex_to_internal(&req.watchtower_challenge_init_txid) + .ok_or_else(|| "invalid watchtower_challenge_init_txid".to_string())?; + + let mut xonly_keys = Vec::with_capacity(req.watchtower_challenge_pubkeys.len()); + for key in &req.watchtower_challenge_pubkeys { + let raw = hex::decode(key.trim_start_matches("0x")) + .map_err(|e| format!("invalid watchtower pubkey hex: {e}"))?; + // 33-byte compressed key -> drop the parity prefix to get the 32-byte x-only key. + let xonly: [u8; 32] = raw + .get(raw.len().saturating_sub(32)..) + .and_then(|s| s.try_into().ok()) + .ok_or_else(|| "watchtower pubkey is too short".to_string())?; + xonly_keys.push(xonly); + } + + let constant = bitcoin_light_client_circuit::hash_operator_constant( + graph_id_bytes, + genesis, + wci_internal, + &xonly_keys, + ); + + let blockhash = display_hex_to_internal(&req.operator_committed_blockhash) + .ok_or_else(|| "invalid operator_committed_blockhash".to_string())?; + + let mut bitmap = [0u8; 32]; + for (i, included) in req.included_watchtowers.iter().enumerate() { + if *included && i < 256 { + bitmap[i / 8] |= 1 << (i % 8); + } + } + + let mut pubin = [0u8; 96]; + pubin[..32].copy_from_slice(&blockhash); + pubin[32..64].copy_from_slice(&constant); + pubin[64..].copy_from_slice(&bitmap); + Ok(pubin) +} + +fn operator_proof_data(state: &AppState, req: &OperatorProofRequest) -> Result { + let public_values = build_operator_public_values(state, req)?; + let mut proof = state.operator_template.clone(); + proof.public_values = ZKMPublicValues::from(&public_values); + let proof_bytes = bincode::serialize(&proof) + .map_err(|e| format!("failed to serialize patched operator proof: {e}"))?; + Ok(ProofData { + proof: proof_bytes, + vk: state.operator_vk.clone(), + public_inputs: public_values.to_vec(), + zkm_version: state.operator_zkm_version.clone(), + }) +} + +fn watchtower_proof_data(state: &AppState, key: &str) -> ProofData { + // Deterministic dummy bytes of exactly the lengths the node validates; content is not checked. + let seed = { + use sha2::{Digest, Sha256}; + let d: [u8; 32] = Sha256::digest(key.as_bytes()).into(); + d + }; + let mut proof = Vec::with_capacity(WATCHTOWER_PROOF_SIZE); + while proof.len() < WATCHTOWER_PROOF_SIZE { + proof.extend_from_slice(&seed); + } + proof.truncate(WATCHTOWER_PROOF_SIZE); + let mut public_inputs = Vec::with_capacity(WATCHTOWER_PUBLIC_INPUTS_SIZE); + while public_inputs.len() < WATCHTOWER_PUBLIC_INPUTS_SIZE { + public_inputs.extend_from_slice(&seed); + } + public_inputs.truncate(WATCHTOWER_PUBLIC_INPUTS_SIZE); + ProofData { + proof, + vk: state.watchtower_vk.clone(), + public_inputs, + zkm_version: state.watchtower_zkm_version.clone(), + } +} + +async fn handle_operator_proof( + State(state): State>, + headers: HeaderMap, + Json(req): Json, +) -> Json { + log_auth_headers("operator", &headers); + response_latency().await; + let key = format!("operator:{}:{}", req.instance_id, req.graph_id); + if !task_ready(&state, &key, operator_delay()) { + return Json(OperatorProofResponse { + proof_data: None, + error: Some("operator proof is not ready".to_string()), + }); + } + match operator_proof_data(&state, &req) { + Ok(proof_data) => { + tracing::info!(graph_id = %req.graph_id, "serving mock operator proof"); + Json(OperatorProofResponse { proof_data: Some(proof_data), error: None }) + } + Err(error) => { + tracing::warn!(graph_id = %req.graph_id, %error, "cannot build mock operator proof"); + Json(OperatorProofResponse { proof_data: None, error: Some(error) }) + } + } +} + +async fn handle_watchtower_proof( + State(state): State>, + headers: HeaderMap, + Json(req): Json, +) -> Json { + log_auth_headers("watchtower", &headers); + response_latency().await; + let key = format!("watchtower:{}:{}:{}", req.instance_id, req.graph_id, req.public_key); + if state.failed_watchtower.lock().expect("failed set poisoned").contains(&key) { + return Json(WatchtowerProofResponse { + proof_data: None, + error: Some("watchtower proof task was marked failed".to_string()), + }); + } + if !task_ready(&state, &key, watchtower_delay()) { + return Json(WatchtowerProofResponse { + proof_data: None, + error: Some("watchtower proof is not ready".to_string()), + }); + } + tracing::info!(graph_id = %req.graph_id, "serving mock watchtower proof"); + Json(WatchtowerProofResponse { + proof_data: Some(watchtower_proof_data(&state, &key)), + error: None, + }) +} + +async fn handle_watchtower_timeout( + State(state): State>, + headers: HeaderMap, + Json(req): Json, +) -> Json { + log_auth_headers("watchtower-timeout", &headers); + let key = format!("watchtower:{}:{}:{}", req.instance_id, req.graph_id, req.public_key); + state.failed_watchtower.lock().expect("failed set poisoned").insert(key); + Json(WatchtowerProofTimeoutUpdateResponse { + instance_id: req.instance_id, + graph_id: req.graph_id, + public_key: req.public_key, + data: Some("1 rows affected".to_string()), + error: None, + }) +} + +async fn handle_proof_desc() -> Json { + Json(ProofDescResponse { proof_desc: None, error: Some("No proof found".to_string()) }) +} + +async fn handle_root() -> &'static str { + "Hello, World!" +} + +fn build_state() -> AppState { + let operator_template = mock_operator_template(); + let operator_zkm_version = std::env::var("MOCK_PROOF_OPERATOR_ZKM_VERSION") + .unwrap_or_else(|_| operator_template.zkm_version.clone()); + let operator_vk = + std::env::var("MOCK_PROOF_OPERATOR_VK").unwrap_or_else(|_| DEFAULT_OPERATOR_VK.to_string()); + let watchtower_vk = std::env::var("MOCK_PROOF_WATCHTOWER_VK") + .unwrap_or_else(|_| DEFAULT_WATCHTOWER_VK.to_string()); + let watchtower_zkm_version = + std::env::var("MOCK_PROOF_WATCHTOWER_ZKM_VERSION").unwrap_or_else(|_| "v1.2.7".to_string()); + + let genesis_txid_internal = match std::env::var("GENESIS_SEQUENCER_COMMIT_TXID") { + Ok(value) => match display_hex_to_internal(&value) { + Some(bytes) => Some(bytes), + None => { + tracing::warn!( + "GENESIS_SEQUENCER_COMMIT_TXID is set but not a 32-byte hex txid; operator proofs will fail" + ); + None + } + }, + Err(_) => { + tracing::warn!( + "GENESIS_SEQUENCER_COMMIT_TXID is not set; operator proofs cannot be served until it is" + ); + None + } + }; + + AppState { + seen: Mutex::new(HashMap::new()), + failed_watchtower: Mutex::new(HashSet::new()), + operator_template, + operator_vk, + operator_zkm_version, + watchtower_vk, + watchtower_zkm_version, + genesis_txid_internal, + } +} + +fn router(state: Arc) -> Router { + Router::new() + .route("/", get(handle_root)) + .route("/v1/proofs/operator_proofs", post(handle_operator_proof)) + .route("/v1/proofs/watchtower_proofs", post(handle_watchtower_proof)) + .route("/v1/proofs/watchtower_proofs_timeout", post(handle_watchtower_timeout)) + .route("/v1/proofs/chain_proofs_desc", get(handle_proof_desc)) + .route("/v1/proofs/operator_proofs_desc", get(handle_proof_desc)) + .with_state(state) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + + let args = Args::parse(); + let state = Arc::new(build_state()); + let app = router(state); + + let listener = tokio::net::TcpListener::bind(args.listen).await?; + tracing::info!(listen = %args.listen, "mock proof builder listening"); + axum::serve(listener, app).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_state() -> AppState { + let operator_template = mock_operator_template(); + AppState { + seen: Mutex::new(HashMap::new()), + failed_watchtower: Mutex::new(HashSet::new()), + operator_template, + operator_vk: DEFAULT_OPERATOR_VK.to_string(), + operator_zkm_version: zkm_sdk::ZKM_CIRCUIT_VERSION.to_string(), + watchtower_vk: DEFAULT_WATCHTOWER_VK.to_string(), + watchtower_zkm_version: "v1.2.7".to_string(), + genesis_txid_internal: Some([7u8; 32]), + } + } + + fn sample_operator_request() -> OperatorProofRequest { + OperatorProofRequest { + instance_id: "11111111-1111-1111-1111-111111111111".to_string(), + graph_id: "22222222-2222-2222-2222-222222222222".to_string(), + gateway_address: None, + operator_committed_blockhash: "aa".repeat(32), + execution_layer_block_number: 5, + watchtower_challenge_txids: vec![Some("bb".repeat(32))], + included_watchtowers: vec![true], + watchtower_challenge_init_txid: "cc".repeat(32), + watchtower_challenge_pubkeys: vec![format!("02{}", "dd".repeat(32))], + } + } + + #[test] + fn readiness_gate_defers_then_serves() { + let state = test_state(); + let key = "operator:i:g"; + // With a positive delay the first look is not ready; with zero delay it is ready. + assert!(!task_ready(&state, key, Duration::from_secs(3600))); + assert!(task_ready(&state, "operator:i:g2", Duration::ZERO)); + } + + #[test] + fn operator_proof_reencodes_and_is_node_decodable() { + let state = test_state(); + let req = sample_operator_request(); + let data = operator_proof_data(&state, &req).expect("operator proof data"); + assert_eq!(data.vk, DEFAULT_OPERATOR_VK); + assert_eq!(data.public_inputs.len(), 96); + // The node deserializes the proof bytes back into a ZKMProofWithPublicValues. + let proof: ZKMProofWithPublicValues = bincode::deserialize(&data.proof).unwrap(); + assert_eq!(proof.public_values.as_slice(), data.public_inputs.as_slice()); + // public_values[32..64] is the operator constant computed from the request fields. + let expected = build_operator_public_values(&state, &req).unwrap(); + assert_eq!(proof.public_values.as_slice(), &expected); + let part_vk = zkm_verifier::Groth16Verifier::try_get_part_stark_vk(&proof.zkm_version) + .expect("bundled partial STARK key"); + let converted = zkm_verifier::convert_ark_imm_wrap_vk( + &proof, + &data.vk, + &zkm_verifier::IMM_GROTH16_VK_BYTES, + part_vk, + ) + .expect("mock proof must pass the node's curve-point and public-input conversion"); + assert_eq!(converted.public_inputs.len(), bitvm_lib::babe_adapter::NUM_PUBLIC_INPUTS); + } + + #[test] + fn operator_proof_requires_genesis_txid() { + let mut state = test_state(); + state.genesis_txid_internal = None; + let err = operator_proof_data(&state, &sample_operator_request()).unwrap_err(); + assert!(err.contains("GENESIS_SEQUENCER_COMMIT_TXID")); + } + + #[test] + fn watchtower_proof_has_expected_lengths() { + let state = test_state(); + let data = watchtower_proof_data(&state, "watchtower:i:g:pk"); + assert_eq!(data.proof.len(), WATCHTOWER_PROOF_SIZE); + assert_eq!(data.public_inputs.len(), WATCHTOWER_PUBLIC_INPUTS_SIZE); + assert_eq!(data.vk.len(), 66); + assert!(!data.zkm_version.is_empty()); + } +} diff --git a/node/src/bin/mock_rpc.rs b/node/src/bin/mock_rpc.rs index 21cd05267..2db32a44a 100644 --- a/node/src/bin/mock_rpc.rs +++ b/node/src/bin/mock_rpc.rs @@ -11,12 +11,13 @@ use alloy::primitives::U256; use anyhow::{Context, Result}; use bitcoin::Network; use bitvm_lib::actors::Actor; +use bitvm_noded::clock::current_time_secs; use bitvm_noded::env::{ ENV_BITCOIN_NETWORK, ENV_BITVM_SECRET, ENV_GOAT_ADDRESS, ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, ENV_GOAT_NETWORK, ENV_GOAT_SWAP_CONTRACT_ADDRESS, }; use bitvm_noded::metrics_service::MetricsState; -use bitvm_noded::rpc_service::{self, AppState, current_time_secs}; +use bitvm_noded::rpc_service::{self, AppState}; use bitvm_noded::utils::{generate_local_key, get_rand_btc_address_p2wpkh, get_rand_goat_address}; use clap::Parser; use client::Utxo; diff --git a/node/src/bin/rpc_sign.rs b/node/src/bin/rpc_sign.rs new file mode 100644 index 000000000..305f4b060 --- /dev/null +++ b/node/src/bin/rpc_sign.rs @@ -0,0 +1,25 @@ +//! Localnet helper: print the signed RPC auth headers for one request so a shell script can +//! call a signed route (e.g. `/v1/debug/engine/clear-poison`) with curl. +//! +//! Usage: BITVM_SECRET= rpc-sign [body] +//! Prints three lines: `x-auth-timestamp: ..`, `x-auth-nonce: ..`, `x-auth-signature: ..`. +use bitvm_noded::rpc_service::auth::{ + AUTH_NONCE_HEADER, AUTH_SIGNATURE_HEADER, AUTH_TIMESTAMP_HEADER, sign_request_auth, +}; +use http::Method; + +fn main() -> anyhow::Result<()> { + let args: Vec = std::env::args().collect(); + if args.len() < 3 { + eprintln!("usage: rpc-sign [body]"); + std::process::exit(2); + } + let method = Method::from_bytes(args[1].as_bytes())?; + let body = args.get(3).map(|s| s.as_bytes().to_vec()).unwrap_or_default(); + let keypair = bitvm_noded::env::get_bitvm_key()?; + let (timestamp, nonce, signature) = sign_request_auth(&keypair, &method, &args[2], &body); + println!("{AUTH_TIMESTAMP_HEADER}: {timestamp}"); + println!("{AUTH_NONCE_HEADER}: {nonce}"); + println!("{AUTH_SIGNATURE_HEADER}: {signature}"); + Ok(()) +} diff --git a/node/src/clock.rs b/node/src/clock.rs new file mode 100644 index 000000000..0fbd7dbff --- /dev/null +++ b/node/src/clock.rs @@ -0,0 +1,8 @@ +//! The wall clock as the node records it: Unix seconds. + +use std::time::UNIX_EPOCH; + +#[inline(always)] +pub fn current_time_secs() -> i64 { + std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 +} diff --git a/node/src/control.rs b/node/src/control.rs new file mode 100644 index 000000000..1213a74a3 --- /dev/null +++ b/node/src/control.rs @@ -0,0 +1,214 @@ +//! Control message handling: ACKs settle the engine outbox, NodeInfo messages +//! update peer information and registration. Every other message kind is +//! recorded for the engine by `p2p_msg_handler::handle_inbound_p2p_message`. + +use crate::clock::current_time_secs; +use crate::messages::*; +use crate::middleware::publisher::MessagePublisher; +use crate::middleware::publisher::publish_to_role; +use crate::utils::*; +use anyhow::Result; +use bitvm_lib::actors::Actor; +use client::goat_chain::GOATClient; +use libp2p::PeerId; +use std::str::FromStr; +use std::sync::Arc; +use store::localdb::LocalDB; + +pub struct ControlContext<'a> { + /// `Sync` so handlers may hold the context across awaits in a `Send` task. + pub publisher: &'a mut (dyn MessagePublisher + Sync), + pub local_db: &'a LocalDB, + pub goat_client: &'a Arc, + pub from_peer_id: PeerId, +} + +pub async fn dispatch_control( + ctx: &mut ControlContext<'_>, + content: &GOATMessageContent, +) -> Result<()> { + match content { + GOATMessageContent::GraphSetupAck(GraphSetupAck { delivery_id }) => { + // ACKs settle the engine outbox. + let Some(engine) = crate::engine_bridge::engine_handle() else { + return Ok(()); + }; + let outcome = engine.acknowledge(&ctx.from_peer_id.to_string(), delivery_id).await?; + tracing::debug!( + delivery_id, + from_peer_id = %ctx.from_peer_id, + outcome = ?outcome, + "processed GraphSetupAck" + ); + Ok(()) + } + GOATMessageContent::RequestNodeInfo(node_info) => { + handle_request_node_info(ctx, node_info).await + } + GOATMessageContent::ResponseNodeInfo(node_info) => { + handle_response_node_info(ctx, node_info).await + } + _ => Ok(()), + } +} + +/// A `NodeInfo` payload is entirely self-declared, so only accept it as the +/// sender's own record: the registry is keyed by `peer_id`, and without this +/// check any peer could overwrite any other node's row. +fn accept_node_info(ctx: &ControlContext<'_>, node_info: &NodeInfo, message_kind: &str) -> bool { + let Err(reason) = check_node_info(ctx, node_info) else { + return true; + }; + // Log registered-peer failures individually and aggregate unregistered-peer failures. + let registered = crate::p2p_admission::peer_registry() + .sender_class(&ctx.from_peer_id, std::time::Instant::now()) + .is_registered(); + if registered { + tracing::warn!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); + } else { + crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::NodeInfoRejected); + tracing::debug!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); + } + false +} + +fn check_node_info( + ctx: &ControlContext<'_>, + node_info: &NodeInfo, +) -> std::result::Result<(), String> { + if !node_info_matches_sender(node_info, &ctx.from_peer_id) { + return Err(format!("payload claims peer_id {}", node_info.peer_id)); + } + validate_node_info_payload(node_info)?; + if node_info.binding_sig.is_empty() { + return Ok(()); + } + // A present-but-invalid binding is a forgery attempt (claiming another key's + // authorisation), not a benign omission, so reject the whole payload. + if crate::messages::verify_node_info_binding(node_info).is_none() { + return Err("node info binding signature is invalid".to_string()); + } + // `binding_issued_at` orders re-bindings, newest winning. One dated far ahead + // would win against every honest re-binding until that date. + if node_info.binding_issued_at + > current_time_secs() + crate::messages::NODE_INFO_BINDING_MAX_FUTURE_SECS + { + return Err("node info binding is dated in the future".to_string()); + } + Ok(()) +} + +/// Record verified operator bindings by x-only key; confirm stake asynchronously. +async fn learn_node_info_binding( + local_db: &LocalDB, + goat_client: &Arc, + node_info: &NodeInfo, +) { + if node_info.actor != Actor::Operator.to_string() { + return; + } + let Some(operator_key) = crate::messages::verify_node_info_binding(node_info) else { + return; + }; + let operator_key = crate::p2p_admission::operator_key(&operator_key); + if let Ok(peer_id) = PeerId::from_str(&node_info.peer_id) { + let released_confirmed_key = crate::p2p_admission::peer_registry() + .observe_operator_binding(&peer_id, &operator_key, node_info.binding_issued_at); + if released_confirmed_key { + // The peer moved to another key: what was confirmed for the old one + // must not be there to be restored after a restart. + crate::p2p_admission::revoke_persisted_operator(local_db, &peer_id).await; + } + crate::p2p_admission::refresh_operator_binding_in_background( + local_db, + goat_client, + peer_id, + operator_key, + ); + } +} + +/// Store accepted NodeInfo within the unregistered-row limit and learn its binding. +async fn record_node_info(ctx: &ControlContext<'_>, node_info: &NodeInfo) -> Result<()> { + let registered = crate::p2p_admission::peer_registry() + .sender_class(&ctx.from_peer_id, std::time::Instant::now()) + .is_registered(); + let admissible = registered + || ctx + .local_db + .acquire() + .await? + .node_row_admissible( + &node_info.peer_id, + crate::p2p_admission::NODE_TABLE_MAX_UNREGISTERED_ROWS, + current_time_secs() - crate::p2p_admission::NODE_TABLE_EVICTABLE_AFTER_SECS, + &crate::env::get_peer_id(), + ) + .await?; + if admissible { + save_node_info(ctx.local_db, node_info).await?; + } else { + crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::NodeTableFull); + } + // The binding is learned either way: it lives in the bounded registry, and + // an operator must be able to become registered while the table is full. + learn_node_info_binding(ctx.local_db, ctx.goat_client, node_info).await; + // A self-declared committee node is a candidate for the registry's + // committee lookup (within its budget); a durable message never is. + if node_info.actor == Actor::Committee.to_string() { + crate::p2p_admission::refresh_registry_in_background( + ctx.local_db, + ctx.goat_client, + Some(ctx.from_peer_id), + ); + } + Ok(()) +} + +async fn handle_request_node_info( + ctx: &mut ControlContext<'_>, + node_info: &NodeInfo, +) -> Result<()> { + if accept_node_info(ctx, node_info, "RequestNodeInfo") { + record_node_info(ctx, node_info).await?; + } + // Answer regardless: the response only carries this node's own public info, + // and staying silent would break discovery for a misconfigured peer. + // Coalesce recent requests into one broadcast response. + if crate::p2p_admission::node_info_response_gate().try_respond(std::time::Instant::now()) { + publish_node_info_response(ctx.publisher).await?; + } + Ok(()) +} + +async fn publish_node_info_response(swarm: &mut dyn MessagePublisher) -> Result<()> { + let message_content = GOATMessageContent::ResponseNodeInfo(crate::env::get_local_node_info()); + publish_to_role(swarm, GOATMessage::new(Actor::All, message_content)).await?; + Ok(()) +} + +/// Answer the `RequestNodeInfo`s that arrived while the previous response was +/// still cooling down. Called from the regular tick. +pub async fn flush_deferred_node_info_response(swarm: &mut dyn MessagePublisher) { + if !crate::p2p_admission::node_info_response_gate().take_pending(std::time::Instant::now()) { + return; + } + if let Err(error) = publish_node_info_response(swarm).await { + tracing::debug!( + event = "p2p_node_info", + outcome = "deferred_response_failed", + error = %error, + "failed to publish the deferred node info response; peers will ask again" + ); + } +} + +async fn handle_response_node_info( + ctx: &mut ControlContext<'_>, + node_info: &NodeInfo, +) -> Result<()> { + if accept_node_info(ctx, node_info, "ResponseNodeInfo") { + record_node_info(ctx, node_info).await?; + } + Ok(()) +} diff --git a/node/src/engine_bridge.rs b/node/src/engine_bridge.rs new file mode 100644 index 000000000..cf4eb5f99 --- /dev/null +++ b/node/src/engine_bridge.rs @@ -0,0 +1,221 @@ +//! Adapters between the execution engine (`engine` crate) and the node: +//! gossip transport for the outbox, the ACK forwarder for the return channel, +//! configuration from the environment, aggregate registration, and the +//! process-wide handle producers use to record events. + +use crate::aggregates::graph::{GRAPH_KIND, GraphAggregate}; +use crate::aggregates::graph_jobs::register_graph_jobs; +use crate::aggregates::instance::{INSTANCE_KIND, InstanceAggregate}; +use crate::aggregates::instance_jobs::register_instance_jobs; +use crate::aggregates::jobs::JobDeps; +use crate::clock::current_time_secs; +use crate::env; +use crate::messages::{GOATMessage, GOATMessageContent, GraphSetupAck}; +use crate::metrics_service::{MetricsState, node_metrics_state}; +use crate::middleware::publisher::{MessagePublisher, NetworkPublisher, publish_to_role}; +use async_trait::async_trait; +use bitvm_lib::actors::Actor; +use bitvm_lib::babe_adapter::BabeBundleBuilder; +use client::btc_chain::BTCClient; +use client::goat_chain::GOATClient; +use client::http_client::async_client::HttpAsyncClient; +use engine::{ + AckReceiver, Engine, EngineBuilder, EngineConfig, EngineHandle, EngineMetrics, NewEvent, + SubmitOutcome, Transport, +}; +use libp2p::gossipsub::IdentTopic; +use libp2p_metrics::Registry; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; +use store::localdb::{LocalDB, StorageProcessor}; +use tokio_util::sync::CancellationToken; + +/// Publishes outbox payloads on their gossip topic through the swarm. +pub struct GossipTransport { + publisher: NetworkPublisher, +} + +impl GossipTransport { + pub fn new(publisher: NetworkPublisher) -> Self { + Self { publisher } + } +} + +#[async_trait] +impl Transport for GossipTransport { + async fn publish(&self, topic: &str, payload: &[u8]) -> anyhow::Result<()> { + crate::middleware::publisher::note_oversized_payload(payload, topic, "outbox"); + let mut publisher = self.publisher.clone(); + let result = publisher.publish_message(IdentTopic::new(topic), payload.to_vec()).await; + if let Some(metrics) = node_metrics_state() { + metrics.record_p2p_publish(result.is_ok()); + } + result.map(|_| ()) + } +} + +/// Engine settings from the environment, on top of the crate defaults. +pub fn engine_config_from_env() -> EngineConfig { + let defaults = EngineConfig::default(); + EngineConfig { + worker_count: env::get_engine_worker_count(defaults.worker_count), + cpu_pool_slots: env::get_engine_cpu_pool_slots(defaults.cpu_pool_slots), + pegout_pool_slots: env::get_engine_pegout_pool_slots(defaults.pegout_pool_slots), + network_pool_slots: env::get_engine_network_pool_slots(defaults.network_pool_slots), + apply_timeout: Duration::from_secs(env::get_engine_apply_timeout_secs( + defaults.apply_timeout.as_secs(), + )), + recheck_interval: Duration::from_secs(env::get_engine_recheck_interval_secs( + defaults.recheck_interval.as_secs(), + )), + ..defaults + } +} + +/// SQLite busy timeout for the node's database: every writer waits at least +/// as long as the engine's apply may hold the write lock, so a long apply +/// delays other writers instead of failing them. +pub fn db_busy_timeout() -> Duration { + engine_config_from_env().apply_timeout.max(store::DEFAULT_BUSY_TIMEOUT) +} + +/// Register the engine's metrics; done before the registry is shared. +pub fn register_engine_metrics(registry: &mut Registry) -> EngineMetrics { + let metrics = EngineMetrics::new(); + metrics.register(registry); + metrics +} + +/// What the aggregates and their jobs need from the node. +pub struct EngineDeps { + pub actor: Actor, + pub local_peer: String, + pub local_db: LocalDB, + pub btc_client: Arc, + pub goat_client: Arc, + pub http_client: Arc, + pub soldering_builder: Option>, + pub metrics_state: MetricsState, +} + +/// Register Instance and Graph handlers, jobs and the network transport. +pub fn build_engine( + publisher: NetworkPublisher, + metrics: EngineMetrics, + deps: EngineDeps, +) -> (Engine, AckReceiver) { + let builder = EngineBuilder::new(deps.local_db.clone(), engine_config_from_env()) + .transport(Arc::new(GossipTransport::new(publisher.clone()))) + .metrics(metrics) + .handler(INSTANCE_KIND, Arc::new(InstanceAggregate::from_env(deps.actor.clone()))) + .handler( + GRAPH_KIND, + Arc::new( + GraphAggregate::new(deps.actor.clone(), env::is_relayer(), deps.local_peer.clone()) + .with_local_operator(own_operator_pubkey(&deps.actor)), + ), + ); + let jobs = Arc::new(JobDeps { + actor: deps.actor, + relayer: env::is_relayer(), + local_db: deps.local_db, + btc_client: deps.btc_client, + goat_client: deps.goat_client, + http_client: deps.http_client, + soldering_builder: deps.soldering_builder, + metrics_state: deps.metrics_state, + publisher, + }); + let builder = register_instance_jobs(builder, jobs.clone()); + register_graph_jobs(builder, jobs).build() +} + +/// The operator key this node withdraws with, as graphs store it. +fn own_operator_pubkey(actor: &Actor) -> Option { + if *actor != Actor::Operator { + return None; + } + match env::get_node_pubkey() { + Ok(pubkey) => Some(pubkey.to_string()), + Err(error) => { + tracing::warn!(error = %error, "operator key unavailable; peg-out jobs for own graphs are disabled"); + None + } + } +} + +static ENGINE_HANDLE: OnceLock = OnceLock::new(); + +/// Make the running engine reachable from producers that have no handle of +/// their own (the event watcher, the admission path). +pub fn install_engine_handle(handle: EngineHandle) { + let _ = ENGINE_HANDLE.set(handle); +} + +pub fn engine_handle() -> Option<&'static EngineHandle> { + ENGINE_HANDLE.get() +} + +/// Record an event inside the caller's transaction. The worker picks it up on +/// its poll after the commit, or sooner through [`wake`]. +pub async fn record_event_in( + storage: &mut StorageProcessor<'_>, + event: NewEvent, +) -> anyhow::Result<()> { + match engine_handle() { + Some(engine) => { + engine.submit_in(storage, event).await?; + } + None => { + storage.insert_engine_inbox_event(&event.into_store(), current_time_secs()).await?; + } + } + Ok(()) +} + +/// Record an event on its own connection and wake its worker. Without an +/// engine handle (tests) the row is inserted directly and counts as new. +pub async fn submit_event(local_db: &LocalDB, event: NewEvent) -> anyhow::Result { + match engine_handle() { + Some(engine) => engine.submit(event).await, + None => { + let mut storage = local_db.acquire().await?; + storage.insert_engine_inbox_event(&event.into_store(), current_time_secs()).await?; + Ok(SubmitOutcome::Inserted) + } + } +} + +/// Turn ACK requests from the engine into `GraphSetupAck` messages. The engine +/// only emits an ACK after the material is durably applied or parked. +pub async fn run_ack_forwarder( + mut acks: AckReceiver, + mut publisher: NetworkPublisher, + cancel: CancellationToken, +) -> anyhow::Result { + loop { + let request = tokio::select! { + biased; + _ = cancel.cancelled() => return Ok("engine ack forwarder stopped".to_string()), + request = acks.recv() => request, + }; + let Some(request) = request else { + return Ok("engine ack channel closed".to_string()); + }; + let message = GOATMessage::new( + Actor::All, + GOATMessageContent::GraphSetupAck(GraphSetupAck { + delivery_id: request.delivery_id.clone(), + }), + ); + if let Err(error) = publish_to_role(&mut publisher, message).await { + tracing::debug!( + event = "engine_ack", + outcome = "publish_failed", + peer = %request.peer, + error = %error, + "ACK publish failed; the sender will resend and be ACKed again" + ); + } + } +} diff --git a/node/src/env.rs b/node/src/env.rs index 6b209fb02..59f795322 100644 --- a/node/src/env.rs +++ b/node/src/env.rs @@ -1,5 +1,4 @@ -#![allow(dead_code)] -use crate::action::NodeInfo; +use crate::messages::NodeInfo; use alloy::primitives::Address as EvmAddress; use alloy::primitives::Address; use alloy::providers::{Provider, ProviderBuilder}; @@ -8,11 +7,14 @@ use base64::Engine; use bitcoin::{Network, PublicKey, key::Keypair}; use bitvm_lib::actors::Actor; use bitvm_lib::keys::NodeMasterKey; -use bitvm_lib::timelocks::{default_connector_z_timelock_blocks, estimated_block_interval_secs}; +use bitvm_lib::timelocks::{ + default_connector_z_timelock_blocks, default_pegin_abandon_buffer_blocks, +}; use client::goat_chain::utils::{ get_committee_management_contract, get_gateway_relay_contracts, is_validate_committee, }; use client::goat_chain::{GoatInitConfig, GoatNetwork}; +use client::timeout_config::get_goat_rpc_timeout_secs; use libp2p::PeerId; use reqwest::Url; use sha2::{Digest, Sha256}; @@ -59,19 +61,13 @@ pub const ENV_BITVM_SECRET: &str = "BITVM_SECRET"; pub const ENV_PEER_KEY: &str = "PEER_KEY"; pub const ENV_PROOF_SEVER_URL: &str = "PROOF_SEVER_URL"; pub const ENV_ACTOR: &str = "ACTOR"; -pub const ENV_IPFS_ENDPOINT: &str = "IPFS_ENDPOINT"; pub const ENV_COMMITTEE_NUM: &str = "COMMITTEE_NUM"; pub const ENV_MIN_REQUIRED_VERIFIER: &str = "MIN_REQUIRED_VERIFIER"; pub const ENV_VERIFIER_CANDIDATE_BACKUP_COUNT: &str = "VERIFIER_CANDIDATE_BACKUP_COUNT"; pub const ENV_VERIFIER_CANDIDATE_COLLECTION_WINDOW_SECS: &str = "VERIFIER_CANDIDATE_COLLECTION_WINDOW_SECS"; -pub const ENV_P2P_GRAPH_SETUP_RETRY_INTERVAL_SECS: &str = "P2P_GRAPH_SETUP_RETRY_INTERVAL_SECS"; -pub const ENV_P2P_GRAPH_SETUP_RETRY_WINDOW_SECS: &str = "P2P_GRAPH_SETUP_RETRY_WINDOW_SECS"; pub const ENV_EXTERNAL_SOCKET_ADDR: &str = "EXTERNAL_SOCKET_ADDR"; pub const COMMITTEE_INSTANCE_KEYS_DIR: &str = "cache/committee-instance-keys/"; -pub const SCRIPT_CACHE_FILE_NAME: &str = "cache/partial_script.bin"; -pub const ASSERT_COMMITS_CACHE_DIR: &str = "cache/assert_commits_cache/"; -pub const IPFS_GRAPH_CACHE_DIR: &str = "cache/graph_cache/"; pub const DUST_AMOUNT: u64 = goat::transactions::base::DUST_AMOUNT; pub const MAX_CUSTOM_INPUTS: usize = 100; @@ -79,8 +75,6 @@ pub const DEFAULT_CONFIRMATION_TARGET: u16 = 1; pub const DEFAULT_MIN_REQUIRED_VERIFIER: usize = 1; pub const DEFAULT_VERIFIER_CANDIDATE_BACKUP_COUNT: usize = 2; pub const DEFAULT_VERIFIER_CANDIDATE_COLLECTION_WINDOW_SECS: i64 = 15; -pub const DEFAULT_P2P_GRAPH_SETUP_RETRY_INTERVAL_SECS: i64 = 10; -pub const DEFAULT_P2P_GRAPH_SETUP_RETRY_WINDOW_SECS: i64 = 600; pub const ENV_BITCOIN_NETWORK: &str = "BITCOIN_NETWORK"; pub const ENV_GOAT_NETWORK: &str = "GOAT_NETWORK"; @@ -91,18 +85,26 @@ pub const ENV_OPERATOR_VK_HASH: &str = "OPERATOR_VK_HASH"; pub const ENV_OPERATOR_ZKM_VERSION: &str = "OPERATOR_ZKM_VERSION"; pub const DEFAULT_WATCHTOWER_PROOF_WAIT_SECS: usize = 60; pub const DEFAULT_OPERATOR_PROOF_WAIT_SECS: usize = 60; +// Only read by the real `get_babe_gc_asset_paths`; unused under `mock-gc`. +#[cfg(not(feature = "mock-gc"))] pub const ENV_FGC_GATES_PATH: &str = "FGC_GATES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_FGC_OUT_INDICES_PATH: &str = "FGC_OUT_INDICES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_SGC_GATES_PATH: &str = "SGC_GATES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_SGC_OUT_INDICES_PATH: &str = "SGC_OUT_INDICES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_FGC_COMPACT_GATES_PATH: &str = "FGC_COMPACT_GATES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_FGC_COMPACT_OUT_INDICES_PATH: &str = "FGC_COMPACT_OUT_INDICES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_SGC_COMPACT_GATES_PATH: &str = "SGC_COMPACT_GATES_PATH"; +#[cfg(not(feature = "mock-gc"))] pub const ENV_SGC_COMPACT_OUT_INDICES_PATH: &str = "SGC_COMPACT_OUT_INDICES_PATH"; pub const ENV_BABE_SETUP_STATE_DIR: &str = "BABE_SETUP_STATE_DIR"; pub const ENV_SOLDERING_PROOF_PAYLOAD_STORE_PATH: &str = "SOLDERING_PROOF_PAYLOAD_STORE_PATH"; -pub const ENV_ALWAYS_CHALLENGE: &str = "ALWAYS_CHALLENGE"; pub const ENV_GENESIS_SEQUENCER_COMMIT_TXID: &str = "GENESIS_SEQUENCER_COMMIT_TXID"; pub const ENV_SEQUENCER_SET_MONITOR_START_COSMOS_BLOCK: &str = "SEQUENCER_SET_MONITOR_START_COSMOS_BLOCK"; @@ -111,33 +113,11 @@ pub const DEFAULT_COSMOS_RPC_URL: &str = "https://rpc.testnet3.goat.network/goat // Conservative fee-reserve estimates, not exact serialized transaction sizes. pub const CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE: u64 = 100; -pub const CHECKSIG_P2TR_INPUT_VBYTES_ESTIMATE: u64 = 100; pub const P2WSH_OUTPUT_VBYTES: u64 = 50; -pub const P2TR_OUTPUT_VBYTES: u64 = 50; -pub const P2A_OUTPUT_VBYTES: u64 = 50; pub const PRE_KICKOFF_BASE_VBYTES: u64 = 300; -pub const PEGIN_BASE_VBYTES: u64 = 300; -pub const CHALLENGE_BASE_VBYTES: u64 = 300; pub const ANCHOR_CHILD_BASE_VBYTES: u64 = 200; // reduce costs to facilitate testing -pub const MIN_SATKE_AMOUNT: u64 = 4_000_000; // 0.04 BTC -pub const MIN_CHALLENGE_AMOUNT: u64 = 1_000_000; // 0.01 BTC -pub const STAKE_RATE: u64 = 0; // 0% -pub const CHALLENGE_RATE: u64 = 0; // 0% - -const COMMITTEE_MEMBER_NUMBER: usize = 2; - -pub const MESSAGE_BROADCAST_MAX_TIMES: i64 = 3; -pub const MESSAGE_RESEND_INTERVAL_SECOND: i64 = 60 * 5; -pub const MESSAGE_EXPIRE_TIME: i64 = 3600 * 24 * 15; // 15 days for test - -pub const MESSAGE_SAVE_INTERVAL_SECOND: i64 = 3600 * 24 * 3; - -pub const GRAPH_OPERATOR_DATA_UPLOAD_TIME_EXPIRED: i64 = 3600 * 48; - -pub const SYNC_GRAPH_INTERVAL: u64 = 3; -pub const SYNC_GRAPH_MAX_WAIT_SECS: u64 = 30; // use to judge load history event thread is dead pub const LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS: i64 = 600; @@ -149,8 +129,6 @@ pub const ENV_INSTANCE_MAINTENANCE_BATCH_SIZE: &str = "INSTANCE_MAINTENANCE_BATC pub const DEFAULT_INSTANCE_MAINTENANCE_BATCH_SIZE: u32 = 50; pub const ENV_MAINTENANCE_RUN_TIMEOUT_SECS: &str = "MAINTENANCE_RUN_TIMEOUT_SECS"; pub const DEFAULT_MAINTENANCE_RUN_TIMEOUT_SECS: u64 = 60; -pub const ENV_P2P_INBOX_BATCH_SIZE: &str = "P2P_INBOX_BATCH_SIZE"; -pub const DEFAULT_P2P_INBOX_BATCH_SIZE: i64 = 16; /// Maximum decoded JSON gossip size. pub const ENV_P2P_MAX_JSON_MESSAGE_BYTES: &str = "P2P_MAX_JSON_MESSAGE_BYTES"; pub const DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES: usize = 2 * 1024 * 1024; @@ -172,15 +150,6 @@ pub const ENV_P2P_RESERVED_PEERS: &str = "P2P_RESERVED_PEERS"; /// Comma-separated `=` bindings. /// Stake is verified on chain at startup. pub const ENV_P2P_TRUSTED_OPERATOR_BINDINGS: &str = "P2P_TRUSTED_OPERATOR_BINDINGS"; -/// Longest wait between two re-publishes of a signing-round outbox row. -pub const ENV_P2P_PROTOCOL_RETRY_MAX_SECS: &str = "P2P_PROTOCOL_RETRY_MAX_SECS"; -pub const DEFAULT_P2P_PROTOCOL_RETRY_MAX_SECS: i64 = 600; -/// Handler execution timeout on the business/control worker. -/// Timeout cancels at an await; durable messages are retried. -pub const ENV_P2P_HANDLER_TIMEOUT_SECS: &str = "P2P_HANDLER_TIMEOUT_SECS"; -pub const DEFAULT_P2P_HANDLER_TIMEOUT_SECS: u64 = 300; -/// Comma-separated peer IDs whose persisted replay marks are cleared at startup. -pub const ENV_P2P_REPLAY_MARK_RESET_PEERS: &str = "P2P_REPLAY_MARK_RESET_PEERS"; pub const ENV_P2P_MAX_PENDING_INCOMING_CONNECTIONS: &str = "P2P_MAX_PENDING_INCOMING_CONNECTIONS"; pub const DEFAULT_P2P_MAX_PENDING_INCOMING_CONNECTIONS: u32 = 128; pub const ENV_P2P_MAX_CONNECTIONS_PER_PEER: &str = "P2P_MAX_CONNECTIONS_PER_PEER"; @@ -207,10 +176,27 @@ pub fn get_network() -> Network { } } -pub fn get_instance_presigned_time_expired_secs() -> i64 { - let network = get_network(); - let connector_z_blocks = default_connector_z_timelock_blocks(network) as i64; - estimated_block_interval_secs(network) * connector_z_blocks * 2 / 3 +pub const ENV_PEGIN_ABANDON_BUFFER_BLOCKS: &str = "PEGIN_ABANDON_BUFFER_BLOCKS"; +pub const ENV_OPERATOR_GRAPH_START_CUTOFF_BLOCKS: &str = "OPERATOR_GRAPH_START_CUTOFF_BLOCKS"; + +/// `B`, the peg-in abandon buffer in blocks after `Prepare height + T0`. Defaults to the +/// network's confirmed-action window. +pub fn get_pegin_abandon_buffer_blocks() -> i64 { + std::env::var(ENV_PEGIN_ABANDON_BUFFER_BLOCKS) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|blocks| blocks.clamp(1, 100_000)) + .unwrap_or_else(|| i64::from(default_pegin_abandon_buffer_blocks(get_network()))) +} + +/// Local operator policy: no new graph for an instance once the tip is past +/// `Prepare height + cutoff`. Defaults to two thirds of `T0`. +pub fn get_operator_graph_start_cutoff_blocks() -> i64 { + std::env::var(ENV_OPERATOR_GRAPH_START_CUTOFF_BLOCKS) + .ok() + .and_then(|value| value.parse::().ok()) + .map(|blocks| blocks.clamp(0, 100_000)) + .unwrap_or_else(|| i64::from(default_connector_z_timelock_blocks(get_network())) * 2 / 3) } pub fn get_goat_network() -> GoatNetwork { @@ -388,7 +374,7 @@ pub fn get_local_node_info() -> NodeInfo { .unwrap_or_default(); let binding_sig = match get_bitvm_key() { Ok(keypair) => { - crate::action::sign_node_info_binding(&peer_key, &pubkey_str, issued_at, &keypair) + crate::messages::sign_node_info_binding(&peer_key, &pubkey_str, issued_at, &keypair) } Err(error) => { tracing::error!("failed to sign node info binding: {error}"); @@ -408,9 +394,6 @@ pub fn get_local_node_info() -> NodeInfo { binding_issued_at: issued_at, } } -pub fn get_committee_member_num() -> usize { - COMMITTEE_MEMBER_NUMBER -} #[derive(Clone, Copy, Display, EnumString)] pub enum GraphBtcTxName { @@ -514,6 +497,44 @@ pub fn get_goat_swap_the_graph_urls_from_env() -> String { .unwrap_or("https://graph.goat.network/subgraphs/name/bitvm_escrow_manager_dev".to_string()) } +const GOAT_BOOT_RETRY_ATTEMPTS: u32 = 12; +const GOAT_BOOT_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); + +/// Retries a Goat read the node needs to boot, so a restart during a Goat +/// outage waits for the RPC (about a minute) instead of dying at once. +async fn retry_at_boot(what: &str, rpc_url: &Url, mut call: F) -> Result +where + E: std::fmt::Display, + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + // The boot provider has no request timeout of its own: bound every call + // so a peer that accepts the connection but never answers still retries. + let timeout = std::time::Duration::from_secs(get_goat_rpc_timeout_secs()); + let mut attempt = 1; + loop { + let error = match tokio::time::timeout(timeout, call()).await { + Ok(Ok(value)) => return Ok(value), + Ok(Err(error)) => error.to_string(), + Err(_) => format!("timed out after {} s", timeout.as_secs()), + }; + if attempt >= GOAT_BOOT_RETRY_ATTEMPTS { + return Err(error); + } + warn!( + event = "goat_boot_retry", + what, + attempt, + max_attempts = GOAT_BOOT_RETRY_ATTEMPTS, + rpc_url = %rpc_url, + error = %error, + "Goat RPC unavailable at boot; retrying" + ); + tokio::time::sleep(GOAT_BOOT_RETRY_DELAY).await; + attempt += 1; + } +} + pub async fn goat_config_from_env() -> GoatInitConfig { let rpc_url = get_goat_url_from_env(); let private_key = std::env::var(ENV_GOAT_PRIVATE_KEY).ok(); @@ -526,11 +547,13 @@ pub async fn goat_config_from_env() -> GoatInitConfig { peg_btc_address, ) = { let provider = ProviderBuilder::new().connect_http(rpc_url.clone()); - let chain_id = provider - .get_chain_id() + let chain_id = retry_at_boot("eth_chainId", &rpc_url, || provider.get_chain_id()) .await - .unwrap_or_else(|_| panic!("cannot get chain_id from {rpc_url}")) - as u32; + .unwrap_or_else(|error| { + panic!( + "cannot get chain_id from {rpc_url} after {GOAT_BOOT_RETRY_ATTEMPTS} attempts: {error}" + ) + }) as u32; let ( committee_management_address, @@ -539,9 +562,13 @@ pub async fn goat_config_from_env() -> GoatInitConfig { peg_btc_address, ) = if let Some(gateway_address) = gateway_address { let (committee_management_address, stake_management_address, btc_spv_address, peg_btc_address) = - get_gateway_relay_contracts(&provider, gateway_address).await.expect( - "fail to get committee, stake management, btc spv, peg btc contract online addresses", - ); + retry_at_boot("gateway relay contracts", &rpc_url, || { + get_gateway_relay_contracts(&provider, gateway_address) + }) + .await + .expect( + "fail to get committee, stake management, btc spv, peg btc contract online addresses", + ); ( Some(committee_management_address), Some(stake_management_address), @@ -657,22 +684,6 @@ pub fn get_verifier_candidate_collection_window_secs() -> i64 { .unwrap_or(DEFAULT_VERIFIER_CANDIDATE_COLLECTION_WINDOW_SECS) } -pub fn get_p2p_graph_setup_retry_interval_secs() -> i64 { - std::env::var(ENV_P2P_GRAPH_SETUP_RETRY_INTERVAL_SECS) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|value| value.clamp(1, 300)) - .unwrap_or(DEFAULT_P2P_GRAPH_SETUP_RETRY_INTERVAL_SECS) -} - -pub fn get_p2p_graph_setup_retry_window_secs() -> i64 { - std::env::var(ENV_P2P_GRAPH_SETUP_RETRY_WINDOW_SECS) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|value| value.clamp(10, 3600)) - .unwrap_or(DEFAULT_P2P_GRAPH_SETUP_RETRY_WINDOW_SECS) -} - pub fn get_soldering_proof_payload_store_path() -> anyhow::Result { let value = std::env::var(ENV_SOLDERING_PROOF_PAYLOAD_STORE_PATH) .map_err(|_| anyhow::anyhow!("{ENV_SOLDERING_PROOF_PAYLOAD_STORE_PATH} needs to be set"))?; @@ -723,6 +734,23 @@ pub struct BabeGcAssetPaths { pub sgc_compact_out_indices: PathBuf, } +/// Under `mock-gc` the 3.3 GB GC gate files are never loaded, so this returns empty paths and +/// requires no env vars. The two callers (`handle.rs` InitGraph / CutCircuits) discard the value. +#[cfg(feature = "mock-gc")] +pub fn get_babe_gc_asset_paths() -> anyhow::Result { + Ok(BabeGcAssetPaths { + fgc_gates: PathBuf::new(), + fgc_out_indices: PathBuf::new(), + sgc_gates: PathBuf::new(), + sgc_out_indices: PathBuf::new(), + fgc_compact_gates: PathBuf::new(), + fgc_compact_out_indices: PathBuf::new(), + sgc_compact_gates: PathBuf::new(), + sgc_compact_out_indices: PathBuf::new(), + }) +} + +#[cfg(not(feature = "mock-gc"))] pub fn get_babe_gc_asset_paths() -> anyhow::Result { fn read_path(name: &str) -> anyhow::Result { let path = @@ -760,14 +788,6 @@ pub fn get_maintenance_run_timeout_secs() -> u64 { .unwrap_or(DEFAULT_MAINTENANCE_RUN_TIMEOUT_SECS) } -pub fn get_p2p_inbox_batch_size() -> i64 { - std::env::var(ENV_P2P_INBOX_BATCH_SIZE) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|size| size.clamp(1, 128)) - .unwrap_or(DEFAULT_P2P_INBOX_BATCH_SIZE) -} - pub fn get_p2p_max_json_message_bytes() -> usize { std::env::var(ENV_P2P_MAX_JSON_MESSAGE_BYTES) .ok() @@ -813,27 +833,6 @@ pub fn parse_reserved_peers(value: &str) -> std::collections::HashSet i64 { - std::env::var(ENV_P2P_PROTOCOL_RETRY_MAX_SECS) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|secs| secs.clamp(30, 3600)) - .unwrap_or_else(|| { - (get_instance_presigned_time_expired_secs() / 4) - .clamp(30, DEFAULT_P2P_PROTOCOL_RETRY_MAX_SECS) - }) -} - -pub fn get_p2p_handler_timeout() -> std::time::Duration { - let secs = std::env::var(ENV_P2P_HANDLER_TIMEOUT_SECS) - .ok() - .and_then(|value| value.parse::().ok()) - .map(|secs| secs.clamp(5, 3600)) - .unwrap_or(DEFAULT_P2P_HANDLER_TIMEOUT_SECS); - std::time::Duration::from_secs(secs) -} - /// Parse `P2P_TRUSTED_OPERATOR_BINDINGS`. A malformed entry is reported and /// skipped: a typo must not stop the node, nor silently trust something else. pub fn get_p2p_trusted_operator_bindings() -> Vec<(libp2p::PeerId, [u8; 32])> { @@ -864,19 +863,6 @@ pub fn parse_trusted_operator_bindings(value: &str) -> Vec<(libp2p::PeerId, [u8; .collect() } -pub fn get_p2p_replay_mark_reset_peers() -> Vec { - std::env::var(ENV_P2P_REPLAY_MARK_RESET_PEERS) - .map(|peers| { - peers - .split(',') - .map(str::trim) - .filter(|peer| !peer.is_empty()) - .map(String::from) - .collect() - }) - .unwrap_or_default() -} - pub fn get_p2p_max_pending_incoming() -> u32 { p2p_connection_limit( ENV_P2P_MAX_PENDING_INCOMING_CONNECTIONS, @@ -888,7 +874,7 @@ pub fn get_p2p_max_per_peer() -> u32 { p2p_connection_limit(ENV_P2P_MAX_CONNECTIONS_PER_PEER, DEFAULT_P2P_MAX_CONNECTIONS_PER_PEER) } -pub fn get_p2p_inbox_max_queued_bytes() -> i64 { +pub fn get_p2p_inbox_max_bytes() -> i64 { std::env::var(ENV_P2P_INBOX_MAX_QUEUED_BYTES) .ok() .and_then(|value| value.parse::().ok()) @@ -896,16 +882,13 @@ pub fn get_p2p_inbox_max_queued_bytes() -> i64 { .unwrap_or(DEFAULT_P2P_INBOX_MAX_QUEUED_BYTES) } +/// Enable committee key deletion on terminal failure except Timeout, or 32 blocks +/// after Confirm confirmation. Disabled deletion leaves key_deleted false. pub fn is_enable_committee_instance_key_delete() -> bool { - // TODO: enable this feature when ready - tracing::warn!( - "Committee key delete not activated, ignore ENV_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE" - ); - false - // std::env::var(ENV_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE) - // .ok() - // .map(|value| value.eq_ignore_ascii_case("true")) - // .unwrap_or(DEFAULT_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE) + std::env::var(ENV_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE) + .ok() + .map(|value| value.eq_ignore_ascii_case("true")) + .unwrap_or(DEFAULT_ENABLE_COMMITTEE_INSTANCE_KEY_DELETE) } pub fn get_committee_instance_key_delete_timelock_blocks() -> i64 { @@ -916,13 +899,46 @@ pub fn get_committee_instance_key_delete_timelock_blocks() -> i64 { .unwrap_or(DEFAULT_COMMITTEE_INSTANCE_KEY_DELETE_TIMELOCK_BLOCKS) } -pub fn should_always_challenge() -> bool { - match std::env::var(ENV_ALWAYS_CHALLENGE) { - Ok(val) => val.to_lowercase() == "true", - Err(_) => false, +pub const ENV_ENGINE_WORKER_COUNT: &str = "ENGINE_WORKER_COUNT"; +pub const ENV_ENGINE_CPU_POOL_SLOTS: &str = "ENGINE_CPU_POOL_SLOTS"; +pub const ENV_ENGINE_PEGOUT_POOL_SLOTS: &str = "ENGINE_PEGOUT_POOL_SLOTS"; +pub const ENV_ENGINE_NETWORK_POOL_SLOTS: &str = "ENGINE_NETWORK_POOL_SLOTS"; +pub const ENV_ENGINE_APPLY_TIMEOUT_SECS: &str = "ENGINE_APPLY_TIMEOUT_SECS"; +pub const ENV_ENGINE_RECHECK_INTERVAL_SECS: &str = "ENGINE_RECHECK_INTERVAL_SECS"; + +fn engine_setting(name: &str, default: T, min: T, max: T) -> T { + match std::env::var(name).ok().and_then(|value| value.parse::().ok()) { + Some(value) if value < min => min, + Some(value) if value > max => max, + Some(value) => value, + None => default, } } +pub fn get_engine_worker_count(default: usize) -> usize { + engine_setting(ENV_ENGINE_WORKER_COUNT, default, 1, 64) +} + +pub fn get_engine_cpu_pool_slots(default: usize) -> usize { + engine_setting(ENV_ENGINE_CPU_POOL_SLOTS, default, 1, 64) +} + +pub fn get_engine_pegout_pool_slots(default: usize) -> usize { + engine_setting(ENV_ENGINE_PEGOUT_POOL_SLOTS, default, 1, 64) +} + +pub fn get_engine_network_pool_slots(default: usize) -> usize { + engine_setting(ENV_ENGINE_NETWORK_POOL_SLOTS, default, 1, 256) +} + +pub fn get_engine_apply_timeout_secs(default: u64) -> u64 { + engine_setting(ENV_ENGINE_APPLY_TIMEOUT_SECS, default, 1, 3600) +} + +pub fn get_engine_recheck_interval_secs(default: u64) -> u64 { + engine_setting(ENV_ENGINE_RECHECK_INTERVAL_SECS, default, 1, 3600) +} + pub fn get_genesis_sequencer_commit_id() -> [u8; 32] { use bitcoin::hashes::Hash; let hexed = std::env::var(ENV_GENESIS_SEQUENCER_COMMIT_TXID).unwrap(); diff --git a/node/src/error.rs b/node/src/error.rs index 6da661095..3f972d511 100644 --- a/node/src/error.rs +++ b/node/src/error.rs @@ -1,6 +1,5 @@ use thiserror::Error; -#[allow(dead_code)] #[derive(Debug, Error)] pub enum SpecialError { #[error("Invalid Pegin Request: {0}")] @@ -12,15 +11,114 @@ pub enum SpecialError { #[error("Invalid Graph: {0}")] InvalidGraph(String), - #[error("Contract Call Reverted: {0}")] - EvmReverted(String), - #[error("Invalid Committee: {0}")] InvalidCommittee(String), #[error("Insufficient Balance: {0}")] InsufficientBalance(String), +} + +/// How a failed step is retried. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FailureClass { + /// A business rejection: the input or the call is wrong, retrying cannot help. + Permanent(String), + /// The chain, the network or the database did not answer; retried without limit. + Transient, + /// Anything else, a bug or a state nobody foresaw; setup steps retry it a + /// bounded number of times, peg-out steps keep retrying with an alert. + Unknown, +} + +pub fn classify_failure(error: &anyhow::Error) -> FailureClass { + if let Some(reason) = permanent_rejection(error) { + return FailureClass::Permanent(reason); + } + if is_transient(error) { + return FailureClass::Transient; + } + FailureClass::Unknown +} + +/// The message or call was rejected for good: a malformed or forged input, a +/// committee mismatch or a contract revert. +pub fn permanent_rejection(error: &anyhow::Error) -> Option { + for cause in error.chain() { + if let Some(special) = cause.downcast_ref::() { + return match special { + SpecialError::InvalidPeginRequest(message) + | SpecialError::InvalidPeginData(message) + | SpecialError::InvalidGraph(message) + | SpecialError::InvalidCommittee(message) => Some(message.clone()), + SpecialError::InsufficientBalance(_) => None, + }; + } + if let Some(message) = revert_message(cause) { + return Some(format!("contract call reverted: {message}")); + } + } + None +} + +/// Whether the contract refused the call (an `execution reverted` answer from +/// a Goat node anywhere in the chain). +pub fn is_contract_revert(error: &anyhow::Error) -> bool { + error.chain().any(|cause| revert_message(cause).is_some()) +} + +fn revert_message(cause: &(dyn std::error::Error + 'static)) -> Option { + if cause.downcast_ref::().is_none() + && cause.downcast_ref::().is_none() + { + return None; + } + let message = cause.to_string(); + (message.contains("execution reverted") || message.contains("revert")).then_some(message) +} + +fn is_transient(error: &anyhow::Error) -> bool { + store::is_storage_error(error) + || error.chain().any(|cause| { + cause.downcast_ref::().is_some() + || cause.downcast_ref::().is_some() + || cause.downcast_ref::().is_some() + || cause.downcast_ref::().is_some() + || cause.downcast_ref::().is_some_and(transport_down) + || cause.downcast_ref::().is_some_and(|contract| { + matches!(contract, alloy::contract::Error::TransportError(inner) if transport_down(inner)) + }) + || transient_message(&cause.to_string()) + }) +} + +fn transport_down(error: &alloy::transports::TransportError) -> bool { + matches!( + error, + alloy::transports::RpcError::Transport(_) | alloy::transports::RpcError::NullResp + ) +} - #[error("Other Error: {0}")] - Other(String), +/// Errors the clients wrap into plain strings: network and backend failures by +/// their usual wording. +fn transient_message(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + [ + "timed out", + "timeout", + "connection", + "error sending request", + "transport(", + "temporarily", + "database is locked", + "database is busy", + "dns error", + "broken pipe", + "reset by peer", + "429 too many", + "502 bad gateway", + "503 service", + "504 gateway", + ] + .iter() + .any(|needle| message.contains(needle)) } diff --git a/node/src/handle.rs b/node/src/handle.rs deleted file mode 100644 index a4dba61e8..000000000 --- a/node/src/handle.rs +++ /dev/null @@ -1,8234 +0,0 @@ -use crate::action::*; -use crate::env::{ - COMMITTEE_INSTANCE_KEYS_DIR, get_babe_gc_asset_paths, get_bitvm_key, get_node_goat_address, - get_peer_id, get_soldering_proof_payload_store_path, get_verifier_candidate_backup_count, - get_verifier_candidate_collection_window_secs, is_relayer, -}; -use crate::error::SpecialError; -use crate::metrics_service::MetricsState; -use crate::middleware::publisher::MessagePublisher; -use crate::rpc_service::current_time_secs; -use crate::scheduled_tasks::graph_maintenance_tasks::ChallengeSubStatus; -use crate::soldering_payload_store::{ - read_soldering_proof_store_payload, soldering_proof_payload_store_path, -}; -use crate::utils::*; -use anyhow::{Context, Result, anyhow, bail, ensure}; -use ark_serialize::CanonicalSerialize; -use bitcoin::{Amount, OutPoint, Txid, hashes::Hash, key::Keypair}; -use bitcoin::{PublicKey, XOnlyPublicKey}; -use bitvm_lib::actors::Actor; -use bitvm_lib::babe_adapter::{ - BABE_M_CC, BABE_N_CC, BabeBundleBuilder, BabeChallengeAssertWitness, BabeProverState, - CACSetupPackage, ChallengeAssertWitnessRaw, CompactSolderingProofPayload, - FinalizedInstanceData, SolderingData, TxAssertWitness, WOTS_SIG_COUNT, assert_wots_message, - build_assert_witness, build_real_challenge_assert_witness, build_real_setup_package, - derive_finalized_indices, expand_compact_soldering_proof_payload, extract_gc_circuit_data, - open_real_setup_and_solder, recover_operator_proof_from_assert_witness, - recover_real_wrongly_challenged_witness, verify_real_setup, -}; -use bitvm_lib::committee::*; -use bitvm_lib::keys::*; -use bitvm_lib::operator::*; -use bitvm_lib::types::{ - BitvmGcCircuitData, BitvmGcGraph, BitvmGcInstanceParameters, SimplifiedBitvmGcGraph, -}; -use bitvm_lib::verifier::*; -use client::goat_chain::{DisproveTxType, PeginStatus, WithdrawStatus}; -use client::graphs::graph_query::BridgeInRequestEvent; -use client::http_client::async_client::HttpAsyncClient; -use client::{btc_chain::BTCClient, goat_chain::GOATClient}; -use goat::transactions::base::output_topology; -use goat::transactions::pre_signed::PreSignedTransaction; -use goat::transactions::pre_signed_musig2::verify_public_nonce; -use goat::wots::{Wots, Wots96}; -use libp2p::PeerId; -use libp2p::gossipsub::MessageId; -use secp256k1::{Message as SecpMessage, SECP256K1}; -use std::str::FromStr; -use std::sync::Arc; -use std::time::Instant; -use store::localdb::LocalDB; -use store::{GoatTxType, GraphStatus, SerializableTxid}; -use uuid::Uuid; - -pub struct HandlerContext<'a> { - pub swarm: &'a mut dyn MessagePublisher, - pub local_db: &'a LocalDB, - pub btc_client: &'a Arc, - pub goat_client: &'a Arc, - pub http_client: &'a HttpAsyncClient, - pub soldering_builder: &'a Option>, - pub metrics_state: &'a MetricsState, - pub actor: Actor, - pub from_peer_id: PeerId, - pub id: MessageId, - pub is_self_peer: bool, -} - -/// Owned resources for work that must not run on the swarm event loop. It -/// deliberately excludes `Swarm`: heavy work persists its follow-up protocol -/// notifications to the durable P2P outbox instead. -pub(crate) struct HeavyTaskContext { - pub local_db: LocalDB, - pub btc_client: Arc, - pub goat_client: Arc, - pub soldering_builder: Option>, - pub metrics_state: MetricsState, - pub from_peer_id: PeerId, -} - -pub(crate) enum HeavyTask { - GenerateVerifierSetup(InitGraph), - GenerateSolderingProof(CutCircuits), - ValidateVerifierGraph(Box), - VerifySolderingProof(SolderingProofReady), -} - -impl HeavyTask { - pub(crate) fn kind(&self) -> &'static str { - match self { - Self::GenerateVerifierSetup(_) => "generate_verifier_setup", - Self::GenerateSolderingProof(_) => "generate_soldering_proof", - Self::ValidateVerifierGraph(_) => "validate_verifier_graph", - Self::VerifySolderingProof(_) => "verify_soldering_proof", - } - } - - pub(crate) fn graph_id(&self) -> Uuid { - match self { - Self::GenerateVerifierSetup(message) => message.graph_id, - Self::GenerateSolderingProof(message) => message.graph_id, - Self::ValidateVerifierGraph(message) => message.graph_id, - Self::VerifySolderingProof(message) => message.graph_id, - } - } -} - -pub(crate) fn heavy_task_from_content( - content: &GOATMessageContent, - actor: &Actor, -) -> Option { - match (content, actor) { - (GOATMessageContent::InitGraph(message), Actor::Verifier) => { - Some(HeavyTask::GenerateVerifierSetup(message.clone())) - } - (GOATMessageContent::CutCircuits(message), Actor::Verifier) => { - Some(HeavyTask::GenerateSolderingProof(message.clone())) - } - (GOATMessageContent::CreateGraph(message), Actor::Verifier) => { - Some(HeavyTask::ValidateVerifierGraph(Box::new(message.clone()))) - } - (GOATMessageContent::SolderingProofReady(message), Actor::Operator) => { - Some(HeavyTask::VerifySolderingProof(message.clone())) - } - _ => None, - } -} - -pub(crate) async fn run_heavy_task(context: &HeavyTaskContext, task: HeavyTask) -> Result<()> { - match task { - HeavyTask::GenerateVerifierSetup(message) => { - handle_init_graph_verifier(context, message).await - } - HeavyTask::GenerateSolderingProof(message) => { - handle_cut_circuits_verifier( - context, - message.instance_id, - message.graph_id, - &message.verifier_pubkey, - message.candidate_index, - &message.selected_circuit_indexes, - ) - .await - } - HeavyTask::ValidateVerifierGraph(message) => { - handle_create_graph_verifier( - context, - message.instance_id, - message.graph_id, - message.graph_nonce, - &message.graph, - ) - .await - } - HeavyTask::VerifySolderingProof(message) => { - handle_soldering_proof_ready_operator( - context, - message.instance_id, - message.graph_id, - message.candidate_index, - message.payload_hash, - message.total_len, - ) - .await - } - } -} - -fn committee_instance_keys_envelope_path(instance_id: Uuid) -> std::path::PathBuf { - let mut path = std::path::PathBuf::from(COMMITTEE_INSTANCE_KEYS_DIR); - path.push(format!("{instance_id}.json")); - path -} - -fn is_io_not_found_error(err: &anyhow::Error) -> bool { - err.chain().any(|cause| { - cause - .downcast_ref::() - .is_some_and(|io_err| io_err.kind() == std::io::ErrorKind::NotFound) - }) -} - -fn load_or_create_committee_instance_keypair( - committee_master_key: &CommitteeMasterKey, - instance_id: Uuid, -) -> Result { - let envelope_path = committee_instance_keys_envelope_path(instance_id); - match committee_master_key.load_instance_keypair(instance_id, &envelope_path) { - Ok(keypair) => Ok(keypair), - Err(load_err) => { - if !is_io_not_found_error(&load_err) { - return Err(load_err).with_context(|| { - format!( - "load committee instance keypair failed for {} at {} (refuse auto-overwrite for non-missing envelope)", - instance_id, - envelope_path.display() - ) - }); - } - tracing::info!( - "committee instance key not found for {} at {}: {}, creating new envelope", - instance_id, - envelope_path.display(), - load_err - ); - committee_master_key - .create_instance_keypair_envelope(instance_id, &envelope_path) - .with_context(|| { - format!( - "create committee instance key envelope failed for {} at {}", - instance_id, - envelope_path.display() - ) - })?; - committee_master_key.load_instance_keypair(instance_id, &envelope_path).with_context( - || { - format!( - "reload committee instance key envelope failed for {} at {}", - instance_id, - envelope_path.display() - ) - }, - ) - } - } -} - -pub async fn dispatch(ctx: &mut HandlerContext<'_>, content: &GOATMessageContent) -> Result<()> { - match (content, &ctx.actor) { - (GOATMessageContent::GraphSetupAck(GraphSetupAck { delivery_id }), _) => { - let source_peer_id = ctx.from_peer_id.to_string(); - let acknowledged = ctx - .local_db - .acquire() - .await? - .acknowledge_p2p_outbox_message(delivery_id, &source_peer_id) - .await?; - tracing::debug!( - delivery_id, - from_peer_id = %ctx.from_peer_id, - acknowledged, - "processed GraphSetupAck" - ); - Ok(()) - } - ( - GOATMessageContent::PeginRequest(PeginRequest { - instance_id, - pegin_request_tx_hash, - pegin_request_height, - .. - }), - Actor::Committee, - ) => { - handle_pegin_request_committee( - ctx, - *instance_id, - pegin_request_tx_hash, - *pegin_request_height, - ) - .await - } - ( - GOATMessageContent::PeginRequest(PeginRequest { - instance_id, - pegin_request_tx_hash, - pegin_request_height, - .. - }), - _, - ) => { - handle_pegin_request_default( - ctx, - *instance_id, - pegin_request_tx_hash, - *pegin_request_height, - ) - .await - } - (GOATMessageContent::ConfirmInstance(ConfirmInstance { instance_id }), Actor::Operator) => { - handle_confirm_instance_operator(ctx, *instance_id).await - } - (GOATMessageContent::ConfirmInstance(ConfirmInstance { instance_id }), _) => { - handle_confirm_instance_default(ctx, *instance_id).await - } - ( - GOATMessageContent::GenCircuits(GenCircuits { - instance_id, - graph_id, - verifier_pubkey, - setup_package, - }), - Actor::Operator, - ) => { - handle_gen_circuits_operator( - ctx, - *instance_id, - *graph_id, - verifier_pubkey, - setup_package, - ) - .await - } - (content, actor) if heavy_task_from_content(content, actor).is_some() => { - bail!("heavy task must be dispatched through the durable P2P inbox") - } - ( - GOATMessageContent::CreateGraph(CreateGraph { - instance_id, - graph_id, - graph_nonce, - graph, - }), - Actor::Committee, - ) => { - handle_create_graph_committee( - ctx, - *instance_id, - *graph_id, - *graph_nonce, - graph, - content, - ) - .await - } - ( - GOATMessageContent::VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement { - instance_id, - graph_id, - verifier_pubkey, - verifier_index, - canonical_graph_params_hash, - signature, - }), - Actor::Committee, - ) => { - handle_verifier_graph_params_endorsement_committee( - ctx, - *instance_id, - *graph_id, - verifier_pubkey, - *verifier_index, - *canonical_graph_params_hash, - signature, - content, - ) - .await - } - ( - GOATMessageContent::NonceGeneration(NonceGeneration { - instance_id, - graph_id, - committee_pubkey: received_committee_pubkey, - pub_nonces, - nonce_sigs, - }), - Actor::Committee, - ) => { - handle_nonce_generation_committee( - ctx, - *instance_id, - *graph_id, - received_committee_pubkey, - pub_nonces, - nonce_sigs, - content, - ) - .await - } - ( - GOATMessageContent::NonceGeneration(NonceGeneration { - instance_id, - graph_id, - committee_pubkey: received_committee_pubkey, - pub_nonces, - nonce_sigs, - }), - Actor::Operator, - ) => { - handle_nonce_generation_operator( - ctx, - *instance_id, - *graph_id, - received_committee_pubkey, - pub_nonces, - nonce_sigs, - ) - .await - } - ( - GOATMessageContent::AggNonceConsensus(AggNonceConsensus { - instance_id, - graph_id, - committee_pubkey: received_committee_pubkey, - consensus_hash, - signature, - }), - Actor::Committee, - ) => { - handle_agg_nonce_consensus_committee( - ctx, - *instance_id, - *graph_id, - received_committee_pubkey, - *consensus_hash, - signature, - content, - ) - .await - } - ( - GOATMessageContent::CommitteePresign(CommitteePresign { - instance_id, - graph_id, - committee_pubkey: received_committee_pubkey, - committee_partial_sigs, - agg_nonces, - }), - Actor::Committee, - ) => { - handle_committee_presign_committee( - ctx, - *instance_id, - *graph_id, - received_committee_pubkey, - committee_partial_sigs, - agg_nonces, - content, - ) - .await - } - ( - GOATMessageContent::CommitteePresign(CommitteePresign { - instance_id, - graph_id, - committee_pubkey: received_committee_pubkey, - committee_partial_sigs, - agg_nonces, - }), - Actor::Operator, - ) => { - handle_committee_presign_operator( - ctx, - *instance_id, - *graph_id, - received_committee_pubkey, - committee_partial_sigs, - agg_nonces, - content, - ) - .await - } - ( - GOATMessageContent::EndorseGraph(EndorseGraph { - instance_id, - graph_id, - committee_pubkey: received_committee_pubkey, - committee_sig_for_graph, - committee_sig_for_params, - committee_evm_address, - }), - Actor::Operator, - ) => { - handle_endorse_graph_operator( - ctx, - *instance_id, - *graph_id, - received_committee_pubkey, - committee_sig_for_graph, - committee_sig_for_params, - committee_evm_address, - ) - .await - } - ( - GOATMessageContent::GraphFinalize(GraphFinalize { - instance_id, - graph_id, - graph_nonce, - graph, - endorse_sigs, - params_endorse_sigs, - }), - Actor::Committee, - ) => { - handle_graph_finalize_committee( - ctx, - *instance_id, - *graph_id, - *graph_nonce, - graph, - endorse_sigs, - params_endorse_sigs, - ) - .await - } - ( - GOATMessageContent::GraphFinalize(GraphFinalize { - instance_id, - graph_id, - graph_nonce, - graph, - endorse_sigs, - params_endorse_sigs, - }), - _, - ) => { - handle_graph_finalize_default( - ctx, - *instance_id, - *graph_id, - *graph_nonce, - graph, - endorse_sigs, - params_endorse_sigs, - ) - .await - } - ( - GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { - instance_id, - committee_pubkey: received_committee_pubkey, - pub_nonce, - nonce_sig, - }), - Actor::Committee, - ) => { - handle_pegin_confirm_nonce_committee( - ctx, - *instance_id, - received_committee_pubkey, - pub_nonce, - nonce_sig, - content, - ) - .await - } - ( - GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { - instance_id, - committee_pubkey: received_committee_pubkey, - consensus_hash, - signature, - }), - Actor::Committee, - ) => { - handle_pegin_confirm_nonce_consensus_committee( - ctx, - *instance_id, - received_committee_pubkey, - *consensus_hash, - signature, - content, - ) - .await - } - ( - GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { - instance_id, - committee_pubkey: received_committee_pubkey, - partial_sig, - endorse_sig, - }), - Actor::Committee, - ) => { - handle_pegin_confirm_partial_sig_committee( - ctx, - *instance_id, - received_committee_pubkey, - partial_sig, - endorse_sig, - content, - ) - .await - } - (GOATMessageContent::PostReady(PostReady { instance_id }), Actor::Committee) => { - handle_post_ready(ctx, *instance_id).await - } - ( - GOATMessageContent::KickoffReady(KickoffReady { instance_id, graph_id }), - Actor::Operator, - ) => handle_kickoff_ready_operator(ctx, *instance_id, *graph_id, content).await, - ( - GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), - Actor::Committee, - ) => handle_kickoff_sent_committee(ctx, *instance_id, *graph_id, content).await, - ( - GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), - Actor::Verifier, - ) => handle_kickoff_sent_verifier(ctx, *instance_id, *graph_id, content).await, - (GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), _) => { - handle_kickoff_sent_default(ctx, *instance_id, *graph_id, content).await - } - ( - GOATMessageContent::PreKickoffSent(PreKickoffSent { instance_id, graph_id }), - Actor::Verifier, - ) => handle_prekickoff_sent_verifier(ctx, *instance_id, *graph_id, content).await, - (GOATMessageContent::PreKickoffSent(PreKickoffSent { instance_id, graph_id }), _) => { - handle_prekickoff_sent_default(ctx, *instance_id, *graph_id).await - } - ( - GOATMessageContent::ChallengeSent(ChallengeSent { - instance_id, - graph_id, - challenge_txid, - }), - Actor::Operator, - ) => { - handle_challenge_sent_operator(ctx, *instance_id, *graph_id, *challenge_txid, content) - .await - } - (GOATMessageContent::ChallengeSent(ChallengeSent { instance_id, graph_id, .. }), _) => { - handle_challenge_sent_default(ctx, *instance_id, *graph_id).await - } - ( - GOATMessageContent::WatchtowerChallengeInitSent(WatchtowerChallengeInitSent { - instance_id, - graph_id, - }), - Actor::Watchtower, - ) => { - handle_watchtower_challenge_init_sent_watchtower(ctx, *instance_id, *graph_id, content) - .await - } - ( - GOATMessageContent::WatchtowerChallengeSent(WatchtowerChallengeSent { - instance_id, - graph_id, - watchtower_index, - }), - Actor::Operator, - ) => { - handle_watchtower_challenge_sent_operator( - ctx, - *instance_id, - *graph_id, - *watchtower_index, - content, - ) - .await - } - ( - GOATMessageContent::WatchtowerChallengeTimeout(WatchtowerChallengeTimeout { - instance_id, - graph_id, - }), - Actor::Operator, - ) => { - handle_watchtower_challenge_timeout_operator(ctx, *instance_id, *graph_id, content) - .await - } - (GOATMessageContent::NackReady(NackReady { instance_id, graph_id }), Actor::Verifier) => { - handle_nack_ready_verifier(ctx, *instance_id, *graph_id, content).await - } - ( - GOATMessageContent::OperatorCommitPubinReady(OperatorCommitPubinReady { - instance_id, - graph_id, - }), - Actor::Operator, - ) => { - handle_operator_commit_pubin_ready_operator(ctx, *instance_id, *graph_id, content).await - } - ( - GOATMessageContent::OperatorCommitPubinTimeout(OperatorCommitPubinTimeout { - instance_id, - graph_id, - }), - Actor::Verifier, - ) => { - handle_operator_commit_pubin_timeout_verifier(ctx, *instance_id, *graph_id, content) - .await - } - ( - GOATMessageContent::AssertReady(AssertReady { instance_id, graph_id }), - Actor::Operator, - ) => handle_assert_ready_operator(ctx, *instance_id, *graph_id).await, - ( - GOATMessageContent::AssertSent(AssertSent { instance_id, graph_id, assert_txid }), - Actor::Verifier, - ) => handle_assert_sent_verifier(ctx, *instance_id, *graph_id, *assert_txid, content).await, - ( - GOATMessageContent::ChallengeAssertSent(ChallengeAssertSent { - instance_id, - graph_id, - challenge_assert_txid, - verifier_index, - .. - }), - Actor::Operator, - ) => { - handle_challenge_assert_sent_operator( - ctx, - *instance_id, - *graph_id, - *challenge_assert_txid, - *verifier_index, - content, - ) - .await - } - ( - GOATMessageContent::WronglyChallengeTimeout(WronglyChallengeTimeout { - instance_id, - graph_id, - challenge_assert_txid, - verifier_index, - .. - }), - Actor::Verifier, - ) => { - handle_wrongly_challenge_timeout_verifier( - ctx, - *instance_id, - *graph_id, - *challenge_assert_txid, - *verifier_index, - content, - ) - .await - } - ( - GOATMessageContent::DisproveSent(DisproveSent { - instance_id, - graph_id, - disprove_type, - index, - challenge_finish_txid, - .. - }), - Actor::Committee, - ) => { - handle_disprove_sent_committee( - ctx, - *instance_id, - *graph_id, - *disprove_type, - *index, - *challenge_finish_txid, - content, - ) - .await - } - (GOATMessageContent::DisproveSent(DisproveSent { instance_id, graph_id, .. }), _) => { - handle_disprove_sent_default(ctx, *instance_id, *graph_id).await - } - (GOATMessageContent::Take1Ready(Take1Ready { instance_id, graph_id }), Actor::Operator) => { - handle_take1_ready_operator(ctx, *instance_id, *graph_id, content).await - } - (GOATMessageContent::Take1Sent(Take1Sent { instance_id, graph_id }), Actor::Committee) => { - handle_take1_sent_committee(ctx, *instance_id, *graph_id, content).await - } - (GOATMessageContent::Take1Sent(Take1Sent { instance_id, graph_id }), _) => { - handle_take1_sent_default(ctx, *instance_id, *graph_id).await - } - (GOATMessageContent::Take2Ready(Take2Ready { instance_id, graph_id }), Actor::Operator) => { - handle_take2_ready_operator(ctx, *instance_id, *graph_id, content).await - } - (GOATMessageContent::Take2Sent(Take2Sent { instance_id, graph_id }), Actor::Committee) => { - handle_take2_sent_committee(ctx, *instance_id, *graph_id, content).await - } - (GOATMessageContent::Take2Sent(Take2Sent { instance_id, graph_id }), _) => { - handle_take2_sent_default(ctx, *instance_id, *graph_id).await - } - ( - GOATMessageContent::SyncGraphRequest(SyncGraphRequest { instance_id, graph_id }), - Actor::Committee, - ) => handle_sync_graph_request(ctx, *instance_id, *graph_id).await, - (GOATMessageContent::SyncGraph(SyncGraph { instance_id, graph_id, graph }), _) => { - handle_sync_graph(ctx, *instance_id, *graph_id, graph).await - } - (GOATMessageContent::RequestNodeInfo(node_info), _) => { - handle_request_node_info(ctx, node_info).await - } - (GOATMessageContent::ResponseNodeInfo(node_info), _) => { - handle_response_node_info(ctx, node_info).await - } - _ => Ok(()), - } -} - -/// Apply the business-key cooldown to stored-value recovery. -async fn publish_stored( - ctx: &mut HandlerContext<'_>, - gate_key: &str, - target: Actor, - content: GOATMessageContent, - origin: StoredValue, -) -> Result<()> { - let now = std::time::Instant::now(); - let gate = crate::p2p_admission::protocol_republish_gate(); - let message = GOATMessage::new(target, content); - if origin == StoredValue::Found && gate.is_cooling(gate_key, now) { - return Ok(()); - } - send_protocol_message(ctx.swarm, ctx.local_db, message, origin).await?; - gate.allow(gate_key, now); - Ok(()) -} - -/// Check local graph-signing completion; missing, foreign and Obsoleted rows are not complete. -async fn graph_signing_round_is_over( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, -) -> Result { - let Some(row) = local_db.acquire().await?.find_graph(&graph_id).await? else { - return Ok(false); - }; - if row.instance_id != instance_id { - return Ok(false); - } - Ok(GraphStatus::from_str(&row.status).is_ok_and(|status| match status { - GraphStatus::OperatorPresigned | GraphStatus::Obsoleted => false, - GraphStatus::CommitteePresigned - | GraphStatus::OperatorDataPushed - | GraphStatus::PreKickoff - | GraphStatus::OperatorKickOff - | GraphStatus::Challenge - | GraphStatus::Disprove - | GraphStatus::Skipped - | GraphStatus::OperatorTake1 - | GraphStatus::OperatorTake2 => true, - })) -} - -/// The committee key this node signs with for `instance_id`. -fn local_committee_pubkey(instance_id: Uuid) -> Result { - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - Ok(load_committee_instance_keypair(&committee_master_key, instance_id)?.public_key().into()) -} - -/// Recover this node's own PeginConfirm value; leave live outbox rows on their retry schedule. -async fn redeliver_own_pegin_confirm_value( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - gate_key: &str, - sender: &PublicKey, - content: &GOATMessageContent, -) -> Result<()> { - if ctx.id != GOATMessage::default_message_id() - || *sender != local_committee_pubkey(instance_id)? - { - return Ok(()); - } - publish_stored(ctx, gate_key, Actor::Committee, content.clone(), StoredValue::Found).await -} - -fn make_message(ctx: &HandlerContext<'_>, content: &GOATMessageContent) -> GOATMessage { - GOATMessage::new(ctx.actor.clone(), content.clone()) -} - -fn build_signed_init_graph( - instance_id: Uuid, - graph_id: Uuid, - operator_keypair: &Keypair, -) -> Result { - let operator_peer_id = PeerId::from_str(&get_peer_id()) - .context("decode local operator peer id for InitGraph")? - .to_bytes(); - Ok(sign_init_graph(instance_id, graph_id, operator_keypair, operator_peer_id)) -} - -/// A graph-bearing message has two identity representations: its envelope and -/// the signed graph parameters. Never use one to read/write local state while -/// using the other to reconstruct or scan the graph. -fn message_identity_matches( - message_kind: &str, - message_instance_id: Uuid, - message_graph_id: Uuid, - message_graph_nonce: Option, - graph_instance_id: Uuid, - graph_graph_id: Uuid, - graph_nonce: u64, -) -> bool { - let nonce_matches = message_graph_nonce.is_none_or(|nonce| nonce == graph_nonce); - if message_instance_id == graph_instance_id - && message_graph_id == graph_graph_id - && nonce_matches - { - return true; - } - - tracing::warn!( - "Ignore {message_kind}: message identity {message_instance_id}:{message_graph_id}:{message_graph_nonce:?} does not match graph parameters {graph_instance_id}:{graph_graph_id}:{graph_nonce}" - ); - false -} - -/// Closes a bounded candidate collection window and assigns provisional slots. -/// Final graph slots are assigned only after valid soldering proofs are collected. -fn freeze_operator_candidates(state: &mut OperatorBabeSetupState) -> Result<()> { - if state.candidate_verifier_pubkeys.is_some() { - return Ok(()); - } - if state.candidates.len() < min_required_verifier() { - bail!( - "cannot freeze {} verifier candidates before reaching target {}", - state.candidates.len(), - min_required_verifier() - ); - } - let mut verifier_peer_ids = std::collections::HashSet::new(); - for candidate in &state.candidates { - if !verifier_peer_ids.insert(&candidate.verifier_peer_id) { - bail!("cannot freeze duplicate verifier peer id"); - } - } - state.candidates.sort_by_key(|candidate| candidate.verifier_pubkey.to_bytes()); - for (candidate_index, candidate) in state.candidates.iter_mut().enumerate() { - candidate.candidate_index = Some(candidate_index); - candidate.selected_circuit_indexes = - derive_finalized_indices(&candidate.setup_package, BABE_M_CC)?; - } - state.candidate_verifier_pubkeys = - Some(state.candidates.iter().map(|candidate| candidate.verifier_pubkey).collect()); - Ok(()) -} - -fn seal_selected_verifiers( - state: &mut OperatorBabeSetupState, - selected_verifier_pubkeys: Vec, -) -> Result<()> { - if let Some(existing) = &state.selected_verifier_pubkeys { - if existing != &selected_verifier_pubkeys { - bail!("refuse to replace the sealed verifier selection"); - } - return Ok(()); - } - if selected_verifier_pubkeys.len() != min_required_verifier() { - bail!( - "selected verifier count {} does not match required {}", - selected_verifier_pubkeys.len(), - min_required_verifier() - ); - } - if selected_verifier_pubkeys.windows(2).any(|pair| pair[0].to_bytes() >= pair[1].to_bytes()) { - bail!("selected verifier public keys must be unique and canonically ordered"); - } - if selected_verifier_pubkeys.iter().any(|selected| { - !state.candidates.iter().any(|candidate| candidate.verifier_pubkey == *selected) - }) { - bail!("sealed verifier selection contains a non-candidate public key"); - } - state.selected_verifier_pubkeys = Some(selected_verifier_pubkeys); - Ok(()) -} - -fn selected_gc_data(state: &OperatorBabeSetupState) -> Result> { - let selected = state - .selected_verifier_pubkeys - .as_ref() - .ok_or_else(|| anyhow!("verifier selection is not sealed"))?; - selected - .iter() - .map(|verifier_pubkey| { - state - .candidates - .iter() - .find(|candidate| candidate.verifier_pubkey == *verifier_pubkey) - .and_then(|candidate| candidate.gc_data.clone()) - .ok_or_else(|| { - anyhow!("missing verified GC data for sealed verifier {verifier_pubkey}") - }) - }) - .collect() -} - -fn selected_candidate_for_graph_index( - state: &OperatorBabeSetupState, - verifier_index: usize, -) -> Result<&OperatorVerifierCandidate> { - let verifier_pubkey = state - .selected_verifier_pubkeys - .as_ref() - .and_then(|selected| selected.get(verifier_index)) - .ok_or_else(|| anyhow!("missing sealed verifier slot {verifier_index}"))?; - state - .candidates - .iter() - .find(|candidate| candidate.verifier_pubkey == *verifier_pubkey) - .ok_or_else(|| anyhow!("missing candidate for sealed verifier slot {verifier_index}")) -} - -/// Stores verified proof data against its immutable candidate slot. -fn record_candidate_gc_data( - state: &mut OperatorBabeSetupState, - verifier_pubkey: PublicKey, - candidate_index: usize, - setup_package: &CACSetupPackage, - gc_data: BitvmGcCircuitData, - prover_state: &BabeProverState, - soldering_proof_ready: SolderingProofReady, -) -> Result<()> { - if gc_data.verifier_pubkey != verifier_pubkey { - bail!("GC slot owner does not match soldering proof verifier"); - } - let candidate = state - .candidates - .iter_mut() - .find(|candidate| candidate.verifier_pubkey == verifier_pubkey) - .ok_or_else(|| anyhow!("selected verifier candidate is missing"))?; - if candidate.candidate_index != Some(candidate_index) { - bail!("candidate index does not match soldering proof slot"); - } - if candidate.setup_package != *setup_package { - bail!("soldering proof setup package does not match selected verifier candidate"); - } - if prover_state.package != *setup_package { - bail!("BABE prover state setup package does not match selected verifier candidate"); - } - if candidate.selected_circuit_indexes != prover_state.soldering.finalized_indices { - bail!("BABE prover state finalized indices do not match selected verifier cut"); - } - if soldering_proof_ready.candidate_index != candidate_index { - bail!("soldering proof reference candidate index does not match selected verifier"); - } - if prover_state.finalized.len() != BABE_M_CC || prover_state.h_msgs.len() != BABE_M_CC { - bail!("BABE prover state must contain exactly {BABE_M_CC} finalized instances and hashes"); - } - if prover_state.h_msgs != gc_data.final_msg_hashlocks { - bail!("BABE prover state hashes do not match GC slot hashes"); - } - if let Some(existing) = &candidate.gc_data - && existing != &gc_data - { - bail!("conflicting GC data received for verifier candidate"); - } - if let Some(existing) = &candidate.soldering_proof_ready - && existing != &soldering_proof_ready - { - bail!("conflicting soldering proof reference received for verifier candidate"); - } - if candidate.gc_data.is_none() { - candidate.gc_data = Some(gc_data); - } - if candidate.soldering_proof_ready.is_none() { - candidate.soldering_proof_ready = Some(soldering_proof_ready); - } - Ok(()) -} - -fn build_babe_prover_state( - setup_package: &CACSetupPackage, - finalized: Vec, - soldering: SolderingData, -) -> Result { - let h_msgs = finalized - .iter() - .map(|finalized| { - setup_package - .commits - .get(finalized.index) - .map(|commit| commit.h_msg) - .ok_or_else(|| anyhow!("finalized BABE instance index is out of range")) - }) - .collect::>>()?; - Ok(BabeProverState { package: setup_package.clone(), finalized, soldering, h_msgs }) -} - -fn validate_verifier_slot_lengths( - verifier_index: usize, - gc_data_len: usize, - verifier_asserts_len: usize, - disproves_len: usize, -) -> Result<()> { - if gc_data_len == 0 { - bail!("graph has no verifier GC slots"); - } - if gc_data_len != verifier_asserts_len || gc_data_len != disproves_len { - bail!( - "graph verifier branch lengths differ: gc_data={gc_data_len}, verifier_asserts={verifier_asserts_len}, disproves={disproves_len}" - ); - } - if verifier_index >= gc_data_len { - bail!("verifier index {verifier_index} out of range for {gc_data_len} slots"); - } - Ok(()) -} - -fn validate_verifier_slot(graph: &BitvmGcGraph, verifier_index: usize) -> Result<()> { - validate_verifier_slot_lengths( - verifier_index, - graph.parameters.gc_data.len(), - graph.verifier_asserts.len(), - graph.disproves.len(), - ) -} - -fn validate_watchtower_branches(graph: &BitvmGcGraph) -> Result { - let watchtower_num = graph.parameters.watchtower_pubkeys.len(); - if watchtower_num == 0 { - bail!("graph has no watchtower slots"); - } - if graph.parameters.watchtower_ack_hashlocks.len() != watchtower_num - || graph.watchtower_challenge_timeouts.len() != watchtower_num - || graph.operator_challenge_nacks.len() != watchtower_num - { - bail!( - "graph watchtower branch lengths differ: watchtowers={}, ack_hashlocks={}, timeouts={}, nacks={}", - watchtower_num, - graph.parameters.watchtower_ack_hashlocks.len(), - graph.watchtower_challenge_timeouts.len(), - graph.operator_challenge_nacks.len() - ); - } - Ok(watchtower_num) -} - -fn find_verifier_index_by_pubkey( - gc_data: &[BitvmGcCircuitData], - verifier_pubkey: &PublicKey, -) -> Result> { - let matches = gc_data - .iter() - .enumerate() - .filter_map(|(index, data)| (data.verifier_pubkey == *verifier_pubkey).then_some(index)) - .collect::>(); - match matches.as_slice() { - [] => Ok(None), - [index] => Ok(Some(*index)), - _ => bail!("graph contains duplicate slots for verifier {verifier_pubkey}"), - } -} - -fn validate_challenge_witness_index( - verifier_index: usize, - challenge_witness: &BabeChallengeAssertWitness, -) -> Result<()> { - if challenge_witness.verifier_index != verifier_index { - bail!( - "challenge witness verifier index {} does not match message verifier index {verifier_index}", - challenge_witness.verifier_index - ); - } - Ok(()) -} - -fn validate_expected_challenge_assert_txid( - graph: &BitvmGcGraph, - verifier_index: usize, - challenge_assert_txid: Txid, -) -> Result<()> { - validate_verifier_slot(graph, verifier_index)?; - let expected_txid = graph.verifier_asserts[verifier_index].tx().compute_txid(); - if challenge_assert_txid != expected_txid { - bail!( - "challenge assert txid {challenge_assert_txid} does not match graph verifier assert txid {expected_txid}" - ); - } - Ok(()) -} - -/// Resolve the request metadata a `PeginRequest` carries from the canonical -/// `BridgeInRequest` event this node indexed itself. -/// -/// The message is gossiped, so its tx hash, height and timestamp are all -/// sender-controlled, and the height alone decides -/// whether the instance ever matches the response-window query in -/// `instance_window_expiration_monitor`. The event watcher already stores the -/// canonical request transaction in `goat_tx_record`, so take all three values -/// from there and never from the message. -/// -/// Returns `None` when the event watcher has not indexed the request yet. The -/// current trigger is ignored in that case: once the watcher observes the -/// event, `instance_answers_monitor` creates a canonical local `PeginRequest`. -async fn resolve_pegin_request_metadata( - local_db: &LocalDB, - instance_id: Uuid, - pegin_request_tx_hash: &str, - pegin_request_height: i64, -) -> Result> { - let tx_record = { - let mut storage_processor = local_db.acquire().await?; - storage_processor - .find_graph_goat_tx_record( - &instance_id, - &Uuid::nil(), - &GoatTxType::BridgeInRequest.to_string(), - ) - .await? - }; - let Some(tx_record) = tx_record else { - tracing::info!( - "Ignore PeginRequest for {instance_id}: no local BridgeInRequest event record yet; \ - the event watcher will enqueue the canonical request" - ); - return Ok(None); - }; - if tx_record.tx_hash != pegin_request_tx_hash || tx_record.height != pegin_request_height { - // Not fatal - the canonical record wins either way - but a mismatch is - // the signature of a forged or stale request, so make it visible. - tracing::warn!( - "PeginRequest for {instance_id} claims tx {pegin_request_tx_hash} at height \ - {pegin_request_height}, the indexed BridgeInRequest is tx {} at height {}", - tx_record.tx_hash, - tx_record.height - ); - } - // The block timestamp comes from the same indexed event; fall back to now - // rather than to anything the sender supplied. - let timestamp = tx_record - .extra - .as_deref() - .and_then(|extra| serde_json::from_str::(extra).ok()) - .and_then(|event| event.block_timestamp.parse::().ok()) - .unwrap_or_else(current_time_secs); - Ok(Some((tx_record.tx_hash, tx_record.height, timestamp))) -} - -fn should_ignore_invalid_pegin_request(e: &anyhow::Error, instance_id: Uuid) -> bool { - if let Some(SpecialError::InvalidPeginRequest(err_msg)) = e.downcast_ref::() { - tracing::warn!("Ignore PeginRequest for {instance_id}: {err_msg}"); - return true; - } - false -} - -fn should_ignore_invalid_pegin_data(e: &anyhow::Error, instance_id: Uuid) -> bool { - if let Some(SpecialError::InvalidPeginData(err_msg)) = e.downcast_ref::() { - tracing::warn!("Ignore ConfirmInstance for {instance_id}: {err_msg}"); - return true; - } - false -} - -fn should_ignore_invalid_graph( - e: &anyhow::Error, - instance_id: Uuid, - graph_id: Uuid, - context: &str, - from_peer: Option<&PeerId>, -) -> bool { - if let Some(SpecialError::InvalidGraph(err_msg)) = e.downcast_ref::() { - if let Some(peer_id) = from_peer { - tracing::warn!( - "Ignore {context} for {instance_id}:{graph_id} from {}: {err_msg}", - peer_id.to_string() - ); - } else { - tracing::warn!("Ignore {context} for {instance_id}:{graph_id}: {err_msg}"); - } - return true; - } - false -} - -fn should_ignore_committee_error( - e: &anyhow::Error, - instance_id: Uuid, - graph_id: Option, - ctx: &HandlerContext<'_>, - context: &str, -) -> bool { - let peer = ctx.from_peer_id.to_string(); - match e.downcast_ref::() { - Some(SpecialError::InvalidCommittee(err_msg)) => { - if let Some(graph_id) = graph_id { - tracing::warn!( - "Ignore {context} for {instance_id}:{graph_id} from {peer}: {err_msg}" - ); - } else { - tracing::warn!("Ignore {context} for {instance_id} from {peer}: {err_msg}"); - } - true - } - Some(SpecialError::EvmReverted(err_msg)) => { - if let Some(graph_id) = graph_id { - tracing::warn!( - "Ignore {context} for {instance_id}:{graph_id} from {peer}: fail to validate committee info on chain: {err_msg}" - ); - } else { - tracing::warn!( - "Ignore {context} for {instance_id} from {peer}: fail to validate committee info on chain: {err_msg}" - ); - } - true - } - _ => false, - } -} - -async fn ensure_self_or_valid_committee( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - graph_id: Option, - received_committee_pubkey: &PublicKey, - context: &str, -) -> Result { - if ctx.is_self_peer { - return Ok(true); - } - if let Err(e) = validate_committee( - ctx.goat_client, - &ctx.from_peer_id, - instance_id, - received_committee_pubkey, - ) - .await - { - if should_ignore_committee_error(&e, instance_id, graph_id, ctx, context) { - return Ok(false); - } - bail!(e); - } - Ok(true) -} - -async fn ensure_self_or_valid_committee_with_evm( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - graph_id: Option, - received_committee_pubkey: &PublicKey, - committee_evm_address: &alloy::primitives::Address, - context: &str, -) -> Result { - if ctx.is_self_peer { - return Ok(true); - } - if let Err(e) = validate_committee_with_evm_address( - ctx.goat_client, - &ctx.from_peer_id, - instance_id, - received_committee_pubkey, - committee_evm_address, - ) - .await - { - if should_ignore_committee_error(&e, instance_id, graph_id, ctx, context) { - return Ok(false); - } - bail!(e); - } - Ok(true) -} - -async fn refresh_and_compensate( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &BitvmGcGraph, - compensation_anchor_status: GraphStatus, -) -> Result<(GraphStatus, Option, bool)> { - let refresh = - refresh_graph(ctx.local_db, ctx.btc_client, ctx.goat_client, instance_id, graph_id, graph) - .await?; - let graph_status = refresh.status; - let sub_status = refresh.sub_status; - tracing::info!("Graph {graph_id} latest status: {graph_status}"); - if refresh.status_transition_accepted { - compensate_graph_events( - ctx.local_db, - ctx.btc_client, - instance_id, - graph_id, - graph, - refresh.scan.as_ref(), - compensation_anchor_status, - graph_status, - ) - .await?; - } - Ok((graph_status, sub_status, refresh.status_transition_accepted)) -} - -async fn refresh_newly_finalized_graph( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &BitvmGcGraph, -) -> Result<()> { - // Reconcile every verified GraphFinalize, including a duplicate. A prior - // attempt may have stored the finalized definition but failed before its - // first chain scan completed. - refresh_and_compensate(ctx, instance_id, graph_id, graph, GraphStatus::CommitteePresigned) - .await?; - Ok(()) -} - -async fn get_graph_for_refresh( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result { - let graph = get_graph(ctx.local_db, instance_id, graph_id) - .await? - .ok_or_else(|| anyhow!("Graph not found for {instance_id}:{graph_id}"))?; - BitvmGcGraph::from_simplified(&graph) -} - -async fn get_graph_for_refresh_or_defer( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - message: &GOATMessage, -) -> Result> { - let graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - message, - ) - .await? - { - Some(g) => g, - None => return Ok(None), - }; - Ok(Some(BitvmGcGraph::from_simplified(&graph)?)) -} - -async fn refresh_graph_status( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - message: Option<&GOATMessage>, - compensate_from_status: GraphStatus, -) -> Result)>> { - let graph = match message { - Some(message) => { - match get_graph_for_refresh_or_defer(ctx, instance_id, graph_id, message).await? { - Some(v) => v, - None => return Ok(None), - } - } - None => get_graph_for_refresh(ctx, instance_id, graph_id).await?, - }; - let (graph_status, sub_status, status_transition_accepted) = - refresh_and_compensate(ctx, instance_id, graph_id, &graph, compensate_from_status).await?; - if !status_transition_accepted { - tracing::warn!( - "Ignore graph action for {instance_id}:{graph_id}: chain scan status was rejected or graph is missing" - ); - return Ok(None); - } - Ok(Some((graph, graph_status, sub_status))) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_pegin_request_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - pegin_request_tx_hash: &str, - pegin_request_height: i64, -) -> Result<()> { - // triggered by BridgeInRequest event - // 1. bind the request metadata to the event this node indexed itself - let Some((pegin_request_tx_hash, pegin_request_height, pegin_timestamp)) = - resolve_pegin_request_metadata( - ctx.local_db, - instance_id, - pegin_request_tx_hash, - pegin_request_height, - ) - .await? - else { - return Ok(()); - }; - // 2. read & check the pegin request data - let (user_info, pegin_amount, answered_by) = - match read_pegin_request(ctx.btc_client, ctx.goat_client, instance_id).await { - Ok(v) => v, - Err(e) => { - if should_ignore_invalid_pegin_request(&e, instance_id) { - return Ok(()); - } - bail!(e) - } - }; - - // 3. save the pegin request data to local db - let stored = store_pegin_request( - ctx.btc_client, - ctx.local_db, - GenerateInstanceParams { - instance_id, - user_info, - pegin_amount, - pegin_request_tx_hash, - pegin_request_height, - pegin_timestamp, - }, - ) - .await?; - if !stored { - // The instance has already moved on, so answering now would be a - // duplicate on-chain call for a request this node handled long ago. - return Ok(()); - } - // 4. call Gateway.answerPeginRequest - // - // PeginRequest is gossiped and re-deliverable, and the pegin data stays - // Pending while answers accumulate, so nothing above notices a replay. - // Gateway.answerPeginRequest does not reject a repeat answer either - it - // overwrites the stored pubkey and re-emits CommitteeResponse - so without - // this every re-delivery spends another transaction. The contract keys - // answers on msg.sender, so compare against the signer that sends it. - if answered_by.contains(&ctx.goat_client.get_default_signer_address()) { - tracing::info!( - "Skip answerPeginRequest for {instance_id}: this committee already answered" - ); - return Ok(()); - } - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let instance_keypair = - load_or_create_committee_instance_keypair(&committee_master_key, instance_id)?; - let pubkey_for_instance = instance_keypair.public_key().into(); - ctx.goat_client.gateway_answer_pegin_request(&instance_id, &pubkey_for_instance).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_pegin_request_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - pegin_request_tx_hash: &str, - pegin_request_height: i64, -) -> Result<()> { - // triggered by BridgeInRequest event - // 1. bind the request metadata to the event this node indexed itself - let Some((pegin_request_tx_hash, pegin_request_height, pegin_timestamp)) = - resolve_pegin_request_metadata( - ctx.local_db, - instance_id, - pegin_request_tx_hash, - pegin_request_height, - ) - .await? - else { - return Ok(()); - }; - // 2. read & check the pegin request data - let (user_info, pegin_amount, _) = - match read_pegin_request(ctx.btc_client, ctx.goat_client, instance_id).await { - Ok(v) => v, - Err(e) => { - if should_ignore_invalid_pegin_request(&e, instance_id) { - return Ok(()); - } - bail!(e) - } - }; - // 3. save the pegin request data to local db - store_pegin_request( - ctx.btc_client, - ctx.local_db, - GenerateInstanceParams { - instance_id, - user_info, - pegin_amount, - pegin_request_tx_hash, - pegin_request_height, - pegin_timestamp, - }, - ) - .await?; - Ok(()) -} - -async fn defer_confirm_instance_until_previous_graph_presigned( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - successor_nonce: u64, - operator_pubkey: &PublicKey, -) -> Result { - if successor_nonce == 0 { - return Ok(false); - } - - let previous_nonce = successor_nonce - 1; - let retry_message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::ConfirmInstance(ConfirmInstance { instance_id }), - ); - let Some((previous_instance_id, previous_graph_id)) = - get_graph_id_by_nonce(ctx.local_db, previous_nonce, operator_pubkey).await? - else { - defer_or_enqueue_message( - ctx.local_db, - &retry_message, - 60, - MessageDeferReason::PreviousGraphPending, - "previous graph nonce is not available locally", - ) - .await?; - tracing::warn!( - "Defer ConfirmInstance for {instance_id}: previous graph with nonce {previous_nonce} is not available locally" - ); - return Ok(true); - }; - let Some(previous_graph) = - get_graph(ctx.local_db, previous_instance_id, previous_graph_id).await? - else { - if let Err(error) = try_send_sync_graph_request( - ctx.swarm, - ctx.goat_client, - previous_instance_id, - previous_graph_id, - ) - .await - { - tracing::warn!( - "Failed to send SyncGraphRequest for previous graph {previous_instance_id}:{previous_graph_id}: {error}" - ); - } - defer_or_enqueue_message( - ctx.local_db, - &retry_message, - 60, - MessageDeferReason::PreviousGraphPending, - "previous graph definition is not available locally", - ) - .await?; - tracing::info!( - "Defer ConfirmInstance for {instance_id}: waiting for previous graph raw data {previous_instance_id}:{previous_graph_id}" - ); - return Ok(true); - }; - if previous_graph.committee_pre_signed() { - return Ok(false); - } - - defer_or_enqueue_message( - ctx.local_db, - &retry_message, - 60, - MessageDeferReason::PreviousGraphPending, - "previous graph is waiting for committee pre-signatures", - ) - .await?; - let message_content = GOATMessageContent::CreateGraph(CreateGraph { - instance_id: previous_instance_id, - graph_id: previous_graph_id, - graph_nonce: previous_graph.parameters.graph_nonce, - graph: previous_graph, - }); - send_protocol_message( - ctx.swarm, - ctx.local_db, - GOATMessage::new(Actor::All, message_content), - StoredValue::Fresh, - ) - .await?; - tracing::info!( - "Defer ConfirmInstance for {instance_id}: re-broadcast previous CreateGraph {previous_instance_id}:{previous_graph_id} until it is committee pre-signed" - ); - Ok(true) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_confirm_instance_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, -) -> Result<()> { - // triggered by PeginDeposit tx - // 0. check if graph already created - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let local_operator_pubkey = operator_master_key.master_keypair().public_key().into(); - if let Some(graph) = get_graph_by_instance_id_and_operator_pubkey( - ctx.local_db, - instance_id, - &local_operator_pubkey, - ) - .await? - { - if defer_confirm_instance_until_previous_graph_presigned( - ctx, - instance_id, - graph.parameters.graph_nonce, - &local_operator_pubkey, - ) - .await? - { - return Ok(()); - } - let graph_id = graph.parameters.graph_id; - tracing::info!("Graph already created for {instance_id}, graph_id: {}", graph_id); - let message_content = GOATMessageContent::CreateGraph(CreateGraph { - instance_id, - graph_id, - graph_nonce: graph.parameters.graph_nonce, - graph, - }); - let msg = GOATMessage::new(Actor::All, message_content); - send_protocol_message(ctx.swarm, ctx.local_db, msg, StoredValue::Fresh).await?; - return Ok(()); - } - - let pending_graph_id = { - let mut storage = ctx.local_db.acquire().await?; - storage - .find_pending_graph_init_by_instance_and_operator_pubkey( - &instance_id, - &local_operator_pubkey.to_string(), - ) - .await? - .map(|pending| pending.graph_id) - }; - if let Some(graph_id) = pending_graph_id { - if let Some((next_graph_nonce, _)) = - get_current_prekickoff_tx(ctx.local_db, &local_operator_pubkey).await? - && defer_confirm_instance_until_previous_graph_presigned( - ctx, - instance_id, - next_graph_nonce, - &local_operator_pubkey, - ) - .await? - { - return Ok(()); - } - tracing::info!("Resume pending graph setup for {instance_id}, graph_id: {graph_id}"); - let message_content = GOATMessageContent::InitGraph(build_signed_init_graph( - instance_id, - graph_id, - &operator_master_key.master_keypair(), - )?); - enqueue_graph_setup_outbox_message( - ctx.local_db, - GOATMessage::new(Actor::Verifier, message_content), - None, - ) - .await?; - - return Ok(()); - } - - // 1. read & check parameters - let instance_params = match read_instance_info_from_goat(ctx.goat_client, instance_id).await { - Ok(v) => v, - Err(e) => { - if should_ignore_invalid_pegin_data(&e, instance_id) { - return Ok(()); - } - bail!(e) - } - }; - let pegin_deposit_txid = instance_params.build_pegin_tx()?.0.tx().compute_txid(); - if !tx_on_chain(ctx.btc_client, &pegin_deposit_txid).await? { - tracing::warn!( - "Ignore ConfirmInstance for {instance_id}: pegin deposit tx {pegin_deposit_txid} not found on chain" - ); - bail!("Invalid ConfirmInstance: pegin deposit tx {pegin_deposit_txid} not found on chain"); - } - - if let Some((next_graph_nonce, _)) = - get_current_prekickoff_tx(ctx.local_db, &local_operator_pubkey).await? - && defer_confirm_instance_until_previous_graph_presigned( - ctx, - instance_id, - next_graph_nonce, - &local_operator_pubkey, - ) - .await? - { - return Ok(()); - } - // after PeginPrepare is confirmed, broadcast InitGraph and let Verifiers generate GC. - - // 2. save the instance data to local db - store_instance_parameters(ctx.local_db, &instance_params).await?; - let graph_id = Uuid::new_v4(); - let mut storage = ctx.local_db.acquire().await?; - storage - .upsert_pending_graph_init(&instance_id, &local_operator_pubkey.to_string(), &graph_id) - .await?; - let message_content = GOATMessageContent::InitGraph(build_signed_init_graph( - instance_id, - graph_id, - &operator_master_key.master_keypair(), - )?); - enqueue_graph_setup_outbox_message( - ctx.local_db, - GOATMessage::new(Actor::Verifier, message_content), - None, - ) - .await?; - - Ok(()) -} - -// Generate garbled circuits and enqueue GenCircuits without blocking the swarm. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %message.instance_id, graph_id = %message.graph_id))] -async fn handle_init_graph_verifier(context: &HeavyTaskContext, message: InitGraph) -> Result<()> { - let InitGraph { instance_id, graph_id, operator_pubkey, operator_peer_id, .. } = &message; - if operator_peer_id != &context.from_peer_id.to_bytes() { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - from_peer_id = %context.from_peer_id, - "Ignore InitGraph whose signed operator peer id differs from its P2P source" - ); - return Ok(()); - } - if !verify_init_graph_signature(&message) { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - operator_pubkey = %operator_pubkey, - "Ignore InitGraph with invalid operator signature" - ); - return Ok(()); - } - if !matches!( - operator_stake_status(&context.goat_client, operator_pubkey).await.map_err(|error| { - retryable_dispatch_error( - RetryableDispatchReason::ExternalRpcUnavailable, - None, - format!("InitGraph operator stake lookup failed: {error:#}"), - ) - })?, - OperatorStakeStatus::Staked - ) { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - operator_pubkey = %operator_pubkey, - "Ignore InitGraph from an operator that does not meet stake requirements" - ); - return Ok(()); - } - - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let verifier_pubkey = verifier_master_key.master_keypair().public_key().into(); - - let saved_verifier_state = load_babe_setup_state(&context.local_db, *instance_id, *graph_id)? - .and_then(|state| state.verifier) - .filter(|state| state.verifier_pubkey == verifier_pubkey); - let verifier_state = if let Some(saved) = saved_verifier_state { - if saved.operator_pubkey != *operator_pubkey || saved.operator_peer_id != *operator_peer_id - { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - expected_operator_peer = %PeerId::from_bytes(&saved.operator_peer_id) - .map(|peer_id| peer_id.to_string()) - .unwrap_or_else(|_| "invalid".to_owned()), - from_peer_id = %context.from_peer_id, - "Ignore InitGraph with a different authenticated operator identity for an existing verifier setup" - ); - return Ok(()); - } - saved - } else { - get_babe_gc_asset_paths()?; - let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for BABE setup")?; - let static_input = derive_operator_static_input()?; - let (setup_package, private_state) = tokio::task::spawn_blocking(move || { - build_real_setup_package(BABE_N_CC, &vk, static_input) - }) - .await - .context("real BABE setup task failed")??; - tracing::info!("Verifier setup done."); - VerifierBabeSetupState { - verifier_pubkey, - operator_pubkey: *operator_pubkey, - operator_peer_id: operator_peer_id.clone(), - setup_package, - private_state, - finalized_indices: vec![], - soldering_proof_ready: None, - } - }; - - let setup_package = verifier_state.setup_package.clone(); - let operator_peer_id = PeerId::from_bytes(&verifier_state.operator_peer_id) - .map(|peer_id| peer_id.to_string()) - .context("decode operator peer id saved from InitGraph")?; - update_babe_setup_state(&context.local_db, *instance_id, *graph_id, |state| { - state.verifier = Some(verifier_state); - })?; - - let gen_circuits = GenCircuits { - instance_id: *instance_id, - graph_id: *graph_id, - verifier_pubkey, - setup_package, - }; - let message = GOATMessage::new(Actor::Operator, GOATMessageContent::GenCircuits(gen_circuits)); - let outbox_id = - enqueue_graph_setup_outbox_message(&context.local_db, message, Some(&operator_peer_id)) - .await?; - tracing::info!( - event = "verifier_gc_setup", - outcome = "enqueued", - outbox_id, - "enqueued GenCircuits for swarm publication" - ); - - Ok(()) -} - -// select a subset of GC and broadcast CutCircuits. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_gen_circuits_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - verifier_pubkey: &PublicKey, - setup_package: &CACSetupPackage, -) -> Result<()> { - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let local_operator_pubkey = operator_master_key.master_keypair().public_key().into(); - if !pending_graph_belongs_to_operator( - ctx.local_db, - instance_id, - graph_id, - &local_operator_pubkey, - ) - .await? - { - tracing::debug!( - "Ignore GenCircuits for {instance_id}:{graph_id}: no local pending graph session" - ); - return Ok(()); - } - - if setup_package.commits.len() != BABE_N_CC { - bail!( - "invalid GenCircuits setup package commitment count: expected {BABE_N_CC}, got {}", - setup_package.commits.len() - ); - } - - let verifier_peer_id = ctx.from_peer_id.to_bytes(); - if !ctx.goat_client.committee_mana_is_verifier(&verifier_peer_id).await.with_context(|| { - format!( - "failed to validate GenCircuits sender {} against the verifier registry", - ctx.from_peer_id - ) - })? { - tracing::warn!( - "Ignore GenCircuits for {instance_id}:{graph_id}: sender {} is not a registered verifier", - ctx.from_peer_id - ); - return Ok(()); - } - - let mut state = load_babe_setup_state(ctx.local_db, instance_id, graph_id)?.unwrap_or_default(); - let operator_state = state.operator.get_or_insert_with(|| OperatorBabeSetupState { - candidate_verifier_pubkeys: None, - candidates: vec![], - candidate_collection_started_at: None, - proof_collection_started_at: None, - selected_verifier_pubkeys: None, - asserted_operator_proof: None, - }); - - let was_frozen = operator_state.candidate_verifier_pubkeys.is_some(); - if let Some(existing) = operator_state - .candidates - .iter() - .find(|candidate| candidate.verifier_peer_id == verifier_peer_id) - { - if existing.verifier_pubkey != *verifier_pubkey { - tracing::warn!( - "Ignore GenCircuits for {instance_id}:{graph_id}: verifier peer {} already submitted a different public key", - ctx.from_peer_id - ); - return Ok(()); - } - if existing.setup_package != *setup_package { - bail!("conflicting GenCircuits setup package for verifier {verifier_pubkey}"); - } - } else if operator_state.candidates.iter().any(|candidate| { - XOnlyPublicKey::from(candidate.verifier_pubkey) == XOnlyPublicKey::from(*verifier_pubkey) - }) { - tracing::warn!( - "Ignore GenCircuits for {instance_id}:{graph_id}: verifier public key {verifier_pubkey} is already bound to another peer" - ); - return Ok(()); - } else if was_frozen { - tracing::debug!( - "Ignore GenCircuits for {instance_id}:{graph_id}: verifier membership is frozen" - ); - - return Ok(()); - } else { - if operator_state.candidates.len() - >= min_required_verifier().saturating_add(get_verifier_candidate_backup_count()) - { - tracing::debug!( - "Ignore GenCircuits for {instance_id}:{graph_id}: candidate collection reached its configured limit" - ); - return Ok(()); - } - operator_state.candidates.push(OperatorVerifierCandidate { - verifier_peer_id, - verifier_pubkey: *verifier_pubkey, - setup_package: setup_package.clone(), - candidate_index: None, - selected_circuit_indexes: vec![], - gc_data: None, - soldering_proof_ready: None, - }); - operator_state.candidate_collection_started_at.get_or_insert_with(current_time_secs); - } - - if operator_state.candidate_verifier_pubkeys.is_none() { - let started_at = - operator_state.candidate_collection_started_at.get_or_insert_with(current_time_secs); - let elapsed = current_time_secs() - *started_at; - let window = get_verifier_candidate_collection_window_secs(); - let reached_limit = operator_state.candidates.len() - >= min_required_verifier().saturating_add(get_verifier_candidate_backup_count()); - if operator_state.candidates.len() < min_required_verifier() - || (!reached_limit && elapsed < window) - { - let retry_after_secs = (window - elapsed).max(1); - let candidate_count = operator_state.candidates.len(); - save_babe_setup_state(ctx.local_db, instance_id, graph_id, &state)?; - return Err(retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(retry_after_secs), - format!( - "collecting verifier candidates: {}/{} received", - candidate_count, - min_required_verifier(), - ), - )); - } - } - - if operator_state.candidate_verifier_pubkeys.is_none() { - freeze_operator_candidates(operator_state)?; - } - // On every retry, enqueue any CutCircuits still owed by the frozen candidate set. - let owed = cut_circuits_owed(operator_state, current_time_secs()); - let sender_peer_id = get_peer_id(); - - save_babe_setup_state(ctx.local_db, instance_id, graph_id, &state)?; - ensure_cut_circuits_outbox(ctx.local_db, instance_id, graph_id, &owed, &sender_peer_id).await?; - Ok(()) -} - -/// Candidates still owed CutCircuits, excluding delivered proofs and closed selections. -fn cut_circuits_owed(state: &OperatorBabeSetupState, now: i64) -> Vec { - if state.candidate_verifier_pubkeys.is_none() || state.selected_verifier_pubkeys.is_some() { - return Vec::new(); - } - let delivered = state.candidates.iter().filter(|candidate| candidate.gc_data.is_some()).count(); - let window_over = state.proof_collection_started_at.is_some_and(|started_at| { - now - started_at >= get_verifier_candidate_collection_window_secs() - }); - if window_over && delivered >= min_required_verifier() { - return Vec::new(); - } - state - .candidates - .iter() - .filter(|candidate| { - candidate.gc_data.is_none() && candidate.soldering_proof_ready.is_none() - }) - .cloned() - .collect() -} - -/// Create missing CutCircuits outbox entries; leave all existing entries unchanged. -async fn ensure_cut_circuits_outbox( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - owed: &[OperatorVerifierCandidate], - sender_peer_id: &str, -) -> Result<()> { - for candidate in owed { - let message = GOATMessage::new( - Actor::Verifier, - GOATMessageContent::CutCircuits(CutCircuits { - instance_id, - graph_id, - verifier_pubkey: candidate.verifier_pubkey, - candidate_index: candidate - .candidate_index - .context("frozen candidate has no slot assigned")?, - selected_circuit_indexes: candidate.selected_circuit_indexes.clone(), - }), - ); - let ack_peer_id = - PeerId::from_bytes(&candidate.verifier_peer_id).map(|peer_id| peer_id.to_string()).ok(); - enqueue_graph_setup_outbox_message_from_peer( - local_db, - message, - ack_peer_id.as_deref(), - sender_peer_id, - ) - .await?; - } - Ok(()) -} - -// Generate proofs for the chosen GC and enqueue SolderingProofReady. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_cut_circuits_verifier( - context: &HeavyTaskContext, - instance_id: Uuid, - graph_id: Uuid, - verifier_pubkey: &PublicKey, - candidate_index: usize, - selected_circuit_indexes: &Vec, -) -> Result<()> { - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let local_verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); - if &local_verifier_pubkey != verifier_pubkey { - tracing::debug!( - "Ignore CutCircuits for {instance_id}:{graph_id}: target verifier pubkey does not match local verifier" - ); - - return Ok(()); - } - - let Some(mut verifier_state) = load_babe_setup_state(&context.local_db, instance_id, graph_id)? - .and_then(|state| state.verifier) - else { - tracing::warn!( - "Ignore CutCircuits for {instance_id}:{graph_id}: missing BABE verifier setup state" - ); - return Ok(()); - }; - - if verifier_state.verifier_pubkey != *verifier_pubkey { - tracing::warn!( - "Ignore CutCircuits for {instance_id}:{graph_id}: verifier pubkey does not match saved BABE setup state" - ); - - return Ok(()); - } - if verifier_state.operator_peer_id != context.from_peer_id.to_bytes() { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - from_peer_id = %context.from_peer_id, - "Ignore CutCircuits from a peer other than the operator that initiated verifier setup" - ); - return Ok(()); - } - // Reject a malformed cut before the expensive opening persists it: graph endorsement and - // challenge-assert both require exactly BABE_M_CC finalized instances. - if selected_circuit_indexes.len() != BABE_M_CC { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - selected = selected_circuit_indexes.len(), - expected = BABE_M_CC, - "Ignore CutCircuits with an invalid number of selected circuits" - ); - return Ok(()); - } - - if let Some(soldering_proof_ready) = verifier_state.soldering_proof_ready.clone() { - if soldering_proof_ready.candidate_index != candidate_index { - bail!( - "CutCircuits candidate index {candidate_index} conflicts with persisted slot {}", - soldering_proof_ready.candidate_index - ); - } - if verifier_state.finalized_indices != *selected_circuit_indexes { - bail!("CutCircuits finalized indices conflict with persisted selection"); - } - let message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::SolderingProofReady(soldering_proof_ready), - ); - let operator_peer_id = PeerId::from_bytes(&verifier_state.operator_peer_id) - .map(|peer_id| peer_id.to_string()) - .context("decode operator peer id saved from InitGraph")?; - let outbox_id = - enqueue_graph_setup_outbox_message(&context.local_db, message, Some(&operator_peer_id)) - .await?; - tracing::info!( - event = "verifier_soldering_proof", - outcome = "enqueued", - outbox_id, - "enqueued SolderingProofReady for swarm publication" - ); - - return Ok(()); - } - - let setup_package = verifier_state.setup_package.clone(); - get_babe_gc_asset_paths()?; - - let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for BABE opening")?; - let static_input = derive_operator_static_input()?; - let private_state = verifier_state.private_state.clone(); - let selected_indices = selected_circuit_indexes.clone(); - let package_for_opening = setup_package.clone(); - let soldering_builder = Arc::clone( - context - .soldering_builder - .as_ref() - .context("BABE soldering builder is not initialized for Verifier")?, - ); - let (opened, finalized, soldering) = tokio::task::spawn_blocking(move || { - open_real_setup_and_solder( - &soldering_builder, - &private_state, - &package_for_opening, - &selected_indices, - &vk, - static_input, - ) - }) - .await - .context("real BABE opening task failed")??; - - let soldering_proof_ready = save_soldering_proof_payload( - instance_id, - graph_id, - candidate_index, - &opened, - &finalized, - &soldering, - ) - .await?; - verifier_state.finalized_indices = selected_circuit_indexes.clone(); - verifier_state.soldering_proof_ready = Some(soldering_proof_ready.clone()); - - let operator_peer_id = PeerId::from_bytes(&verifier_state.operator_peer_id) - .map(|peer_id| peer_id.to_string()) - .context("decode operator peer id saved from InitGraph")?; - update_babe_setup_state(&context.local_db, instance_id, graph_id, |state| { - state.verifier = Some(verifier_state); - })?; - - let message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::SolderingProofReady(soldering_proof_ready), - ); - let outbox_id = - enqueue_graph_setup_outbox_message(&context.local_db, message, Some(&operator_peer_id)) - .await?; - tracing::info!( - event = "verifier_soldering_proof", - outcome = "enqueued", - outbox_id, - "enqueued SolderingProofReady for swarm publication" - ); - - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn handle_soldering_proof_ready_operator( - context: &HeavyTaskContext, - instance_id: Uuid, - graph_id: Uuid, - candidate_index: usize, - payload_hash: [u8; 32], - total_len: usize, -) -> Result<()> { - if total_len == 0 { - bail!("SolderingProofReady total_len must be greater than zero"); - } - let soldering_proof_ready = - SolderingProofReady { instance_id, graph_id, candidate_index, payload_hash, total_len }; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let local_operator_pubkey = operator_master_key.master_keypair().public_key().into(); - if !pending_graph_belongs_to_operator( - &context.local_db, - instance_id, - graph_id, - &local_operator_pubkey, - ) - .await? - { - tracing::debug!( - "Ignore SolderingProofReady for {instance_id}:{graph_id}: no local pending graph session" - ); - - return Ok(()); - } - - let state = - load_babe_setup_state(&context.local_db, instance_id, graph_id)?.ok_or_else(|| { - retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(30), - format!("missing BABE setup state for pending graph {graph_id}"), - ) - })?; - let operator_state = state.operator.as_ref().ok_or_else(|| { - retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(30), - format!("missing operator BABE setup state for pending graph {graph_id}"), - ) - })?; - let frozen = operator_state.candidate_verifier_pubkeys.as_ref().ok_or_else(|| { - retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(30), - "operator verifier membership is not frozen", - ) - })?; - let verifier_pubkey = frozen.get(candidate_index).ok_or_else(|| { - anyhow!("SolderingProofReady candidate index {candidate_index} out of range") - })?; - let candidate = operator_state - .candidates - .iter() - .find(|candidate| candidate.verifier_pubkey == *verifier_pubkey) - .ok_or_else(|| anyhow!("selected verifier candidate is missing"))?; - if candidate.candidate_index != Some(candidate_index) { - bail!("selected verifier candidate index does not match SolderingProofReady slot"); - } - if operator_state.selected_verifier_pubkeys.as_ref().is_some_and(|selected| { - !selected.iter().any(|selected_pubkey| selected_pubkey == verifier_pubkey) - }) { - tracing::info!( - instance_id = %instance_id, - graph_id = %graph_id, - candidate_index, - verifier_pubkey = %verifier_pubkey, - "Ignore late SolderingProofReady from verifier outside the sealed graph selection" - ); - return Ok(()); - } - let verifier_peer_id = context.from_peer_id.to_bytes(); - if candidate.verifier_peer_id != verifier_peer_id { - tracing::warn!( - "Ignore SolderingProofReady for {instance_id}:{graph_id}: sender {} does not own verifier candidate {candidate_index}", - context.from_peer_id - ); - return Ok(()); - } - - let store_base_path = get_soldering_proof_payload_store_path()?; - let payload_path = soldering_proof_payload_store_path( - &store_base_path, - instance_id, - graph_id, - candidate_index, - &payload_hash, - )?; - tracing::info!( - from_peer_id = %context.from_peer_id, - instance_id = %instance_id, - graph_id = %graph_id, - candidate_index, - total_len, - payload_hash = %soldering_payload_hash_hex(&payload_hash), - payload_path = %payload_path, - "received SolderingProofReady; reading payload from store" - ); - let payload = match read_soldering_proof_store_payload(&payload_path).await { - Ok(payload) => payload, - Err(err) => { - tracing::error!( - instance_id = %instance_id, - graph_id = %graph_id, - candidate_index, - payload_path = %payload_path, - error = %err, - "failed to read soldering proof payload from store" - ); - return Err(retryable_dispatch_error( - RetryableDispatchReason::PayloadNotReady, - Some(30), - format!("read soldering proof payload from store: {err}"), - )); - } - }; - tracing::info!( - instance_id = %instance_id, - graph_id = %graph_id, - candidate_index, - bytes = payload.len(), - total_len, - payload_hash = %soldering_payload_hash_hex(&payload_hash), - payload_path = %payload_path, - "read soldering proof payload from store, start processing" - ); - let result = - handle_soldering_proof_payload_operator(context, &soldering_proof_ready, &payload).await; - context.metrics_state.record_pegin_graph_setup(result.is_ok()); - result -} - -fn decode_soldering_proof_payload( - soldering_proof_ready: &SolderingProofReady, - payload: &[u8], -) -> Result { - let SolderingProofReady { instance_id, graph_id, candidate_index, payload_hash, total_len } = - soldering_proof_ready; - if payload.len() != *total_len { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - candidate_index, - actual_len = payload.len(), - total_len, - payload_hash = %soldering_payload_hash_hex(payload_hash), - "SolderingProof payload length mismatch" - ); - bail!( - "SolderingProof payload length {} does not match total_len {total_len}", - payload.len() - ); - } - let actual_hash = soldering_payload_hash(payload); - if actual_hash != *payload_hash { - tracing::warn!( - instance_id = %instance_id, - graph_id = %graph_id, - candidate_index, - expected_hash = %soldering_payload_hash_hex(payload_hash), - actual_hash = %soldering_payload_hash_hex(&actual_hash), - "SolderingProof payload hash mismatch" - ); - bail!("SolderingProof payload hash mismatch"); - } - bincode::deserialize(payload).context("deserialize compact soldering proof payload") -} - -pub(crate) async fn handle_soldering_proof_payload_operator( - context: &HeavyTaskContext, - soldering_proof_ready: &SolderingProofReady, - payload: &[u8], -) -> Result<()> { - let decode_started_at = Instant::now(); - tracing::info!( - event = "operator_soldering_proof", - stage = "payload_decode", - outcome = "started", - candidate_index = soldering_proof_ready.candidate_index, - payload_len = payload.len(), - "decoding soldering proof payload" - ); - let payload = decode_soldering_proof_payload(soldering_proof_ready, payload)?; - tracing::info!( - event = "operator_soldering_proof", - stage = "payload_decode", - outcome = "completed", - candidate_index = soldering_proof_ready.candidate_index, - elapsed_ms = decode_started_at.elapsed().as_millis(), - "decoded soldering proof payload" - ); - handle_compact_soldering_proof_operator(context, soldering_proof_ready, payload).await -} - -// verify Verifier SolderingProof, build Graph and broadcast CreateGraph. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %soldering_proof_ready.instance_id, graph_id = %soldering_proof_ready.graph_id))] -async fn handle_compact_soldering_proof_operator( - context: &HeavyTaskContext, - soldering_proof_ready: &SolderingProofReady, - payload: CompactSolderingProofPayload, -) -> Result<()> { - let instance_id = soldering_proof_ready.instance_id; - let graph_id = soldering_proof_ready.graph_id; - let candidate_index = soldering_proof_ready.candidate_index; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let local_operator_pubkey = operator_master_key.master_keypair().public_key().into(); - if !pending_graph_belongs_to_operator( - &context.local_db, - instance_id, - graph_id, - &local_operator_pubkey, - ) - .await? - { - tracing::debug!( - "Ignore SolderingProof for {instance_id}:{graph_id}: no local pending graph session" - ); - - return Ok(()); - } - - let mut state = - load_babe_setup_state(&context.local_db, instance_id, graph_id)?.ok_or_else(|| { - retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(30), - format!("missing BABE setup state for pending graph {graph_id}"), - ) - })?; - let operator_state = state.operator.as_mut().ok_or_else(|| { - retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(30), - format!("missing operator BABE setup state for pending graph {graph_id}"), - ) - })?; - let frozen = operator_state.candidate_verifier_pubkeys.as_ref().ok_or_else(|| { - retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some(30), - "operator verifier membership is not frozen", - ) - })?; - let verifier_pubkey = *frozen - .get(candidate_index) - .ok_or_else(|| anyhow!("SolderingProof candidate index {candidate_index} out of range"))?; - - let candidate = operator_state - .candidates - .iter() - .find(|candidate| candidate.verifier_pubkey == verifier_pubkey) - .ok_or_else(|| anyhow!("selected verifier candidate is missing"))?; - if candidate.candidate_index != Some(candidate_index) { - bail!("selected verifier candidate index does not match SolderingProof slot"); - } - let setup_package = candidate.setup_package.clone(); - let claimed_finalized_indices = candidate.selected_circuit_indexes.clone(); - let expand_started_at = Instant::now(); - tracing::info!( - event = "operator_soldering_proof", - stage = "payload_expand", - outcome = "started", - candidate_index, - "expanding compact soldering proof" - ); - let (opened, finalized, soldering) = expand_compact_soldering_proof_payload(payload) - .context("expand compact soldering proof payload")?; - tracing::info!( - event = "operator_soldering_proof", - stage = "payload_expand", - outcome = "completed", - candidate_index, - opened_instances = opened.len(), - finalized_instances = finalized.len(), - elapsed_ms = expand_started_at.elapsed().as_millis(), - "expanded compact soldering proof" - ); - - let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for BABE validation")?; - let static_input = derive_operator_static_input()?; - let package_for_validation = setup_package.clone(); - let opened_for_validation = opened.clone(); - let finalized_for_validation = finalized.clone(); - let soldering_for_validation = soldering.clone(); - let soldering_builder = Arc::clone( - context - .soldering_builder - .as_ref() - .context("BABE soldering builder is not initialized for Operator")?, - ); - - let verification_started_at = Instant::now(); - tracing::info!( - event = "operator_soldering_proof", - stage = "setup_verify", - outcome = "started", - candidate_index, - opened_instances = opened.len(), - finalized_instances = finalized.len(), - "verifying verifier soldering proof" - ); - tokio::task::spawn_blocking(move || { - verify_real_setup( - &soldering_builder, - &package_for_validation, - &opened_for_validation, - &finalized_for_validation, - &soldering_for_validation, - &vk, - &claimed_finalized_indices, - static_input, - ) - }) - .await - .context("real BABE setup verification task failed")??; - tracing::info!( - event = "operator_soldering_proof", - stage = "setup_verify", - outcome = "completed", - candidate_index, - elapsed_ms = verification_started_at.elapsed().as_millis(), - "verified verifier soldering proof" - ); - - tracing::info!( - event = "operator_graph_creation", - outcome = "soldering_verified", - candidate_index, - verifier_pubkey = %verifier_pubkey, - finalized_instances = finalized.len(), - payload_hash = %hex::encode(soldering_proof_ready.payload_hash), - "verified verifier soldering proof" - ); - - if finalized.len() != BABE_M_CC { - bail!("each verifier must contribute exactly {BABE_M_CC} finalized BABE instances"); - } - let epk = &setup_package.commits[finalized[0].index].epk; - let gc_data_started_at = Instant::now(); - tracing::info!( - event = "operator_soldering_proof", - stage = "gc_data_extract", - outcome = "started", - candidate_index, - "building BABE prover state and extracting GC data" - ); - let prover_state = build_babe_prover_state(&setup_package, finalized, soldering)?; - let gc_data = extract_gc_circuit_data(verifier_pubkey, epk, &prover_state.h_msgs)?; - tracing::info!( - event = "operator_soldering_proof", - stage = "gc_data_extract", - outcome = "completed", - candidate_index, - elapsed_ms = gc_data_started_at.elapsed().as_millis(), - "extracted GC data from soldering proof" - ); - record_candidate_gc_data( - operator_state, - verifier_pubkey, - candidate_index, - &setup_package, - gc_data, - &prover_state, - soldering_proof_ready.clone(), - )?; - let completed_slots = - operator_state.candidates.iter().filter(|candidate| candidate.gc_data.is_some()).count(); - let required_slots = min_required_verifier(); - let proof_window_started_at = - operator_state.proof_collection_started_at.get_or_insert_with(current_time_secs); - let elapsed = current_time_secs() - *proof_window_started_at; - let collection_window = get_verifier_candidate_collection_window_secs(); - if completed_slots < required_slots || elapsed < collection_window { - if let Err(error) = save_babe_setup_state(&context.local_db, instance_id, graph_id, &state) - { - tracing::error!( - event = "operator_graph_creation", - outcome = "failed", - stage = "babe_state_persist", - candidate_index, - error = %error, - "failed to persist incomplete operator BABE setup state" - ); - return Err(error).context("persist incomplete operator BABE setup state"); - } - tracing::info!( - event = "operator_graph_creation", - outcome = "waiting_for_soldering", - candidate_index, - completed_slots, - required_slots, - retry_after_secs = (collection_window - elapsed).max(1), - "persisted verified soldering proof; waiting for the proof selection window" - ); - return Err(retryable_dispatch_error( - RetryableDispatchReason::DependencyPending, - Some((collection_window - elapsed).max(1)), - format!("collecting verified soldering proofs: {completed_slots}/{required_slots}"), - )); - } - - let selected_verifier_pubkeys = - if let Some(selected) = &operator_state.selected_verifier_pubkeys { - selected.clone() - } else { - let mut selected = operator_state - .candidates - .iter() - .filter(|candidate| candidate.gc_data.is_some()) - .map(|candidate| candidate.verifier_pubkey) - .collect::>(); - selected.sort_by_key(|candidate| candidate.to_bytes()); - selected.truncate(required_slots); - selected - }; - seal_selected_verifiers(operator_state, selected_verifier_pubkeys)?; - let bitvm_gc_circuit_datas = selected_gc_data(operator_state)?; - let selected_verifier_count = operator_state - .selected_verifier_pubkeys - .as_ref() - .context("selected verifier set is missing after proof collection")? - .len(); - let selected_verifier_peer_ids = (0..selected_verifier_count) - .map(|verifier_index| { - let candidate = selected_candidate_for_graph_index(operator_state, verifier_index)?; - PeerId::from_bytes(&candidate.verifier_peer_id) - .context("decode selected verifier peer id for CreateGraph ACK") - .map(|peer_id| peer_id.to_string()) - }) - .collect::>>()?; - let mut obsolete_setup_outbox_ids = vec![format!("init-graph:{graph_id}")]; - obsolete_setup_outbox_ids.extend(operator_state.candidates.iter().map(|candidate| { - format!("cut-circuits:{instance_id}:{graph_id}:{}", candidate.verifier_pubkey) - })); - if let Err(error) = save_babe_setup_state(&context.local_db, instance_id, graph_id, &state) { - tracing::error!( - event = "operator_graph_creation", - outcome = "failed", - stage = "babe_state_persist", - candidate_index, - error = %error, - "failed to persist complete operator BABE setup state" - ); - return Err(error).context("persist complete operator BABE setup state"); - } - tracing::info!( - event = "operator_graph_creation", - outcome = "all_soldering_collected", - verifier_slots = bitvm_gc_circuit_datas.len(), - "all verifier soldering proofs are ready to build the graph" - ); - - // Reuse an existing graph for this ID and resume CreateGraph delivery. - let (graph_nonce, graph, definition_hash, already_finalized) = match get_graph( - &context.local_db, - instance_id, - graph_id, - ) - .await? - { - Some(stored) => { - let graph_nonce = stored.parameters.graph_nonce; - let definition_hash = hex::encode(stored.parameters_hash()?); - let already_finalized = BitvmGcGraph::from_simplified(&stored)?.committee_pre_signed(); - tracing::info!( - event = "operator_graph_creation", - outcome = "resumed", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - already_finalized, - "graph was stored by an earlier run; resuming at the CreateGraph outbox" - ); - (graph_nonce, stored, definition_hash, already_finalized) - } - None => { - let instance_params = get_instance_parameters(&context.local_db, instance_id) - .await? - .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; - - let (graph_nonce, cur_prekickoff_txn) = - match get_current_prekickoff_tx(&context.local_db, &local_operator_pubkey).await? { - Some((graph_nonce, prekickoff_tx)) => (graph_nonce, prekickoff_tx), - None => ( - 0, - build_genesis_prekickoff_tx(&context.btc_client, &context.goat_client) - .await?, - ), - }; - let prekickoff_params = - build_prekickoff_params(&context.btc_client, graph_nonce, cur_prekickoff_txn) - .await?; - - let graph_build_started_at = Instant::now(); - tracing::info!( - event = "operator_soldering_proof", - stage = "graph_build", - outcome = "started", - verifier_slots = bitvm_gc_circuit_datas.len(), - graph_nonce, - "building graph parameters from verified soldering proofs" - ); - let mut graph_params = build_graph_params( - &context.local_db, - &context.goat_client, - instance_params, - prekickoff_params, - bitvm_gc_circuit_datas, - graph_nonce, - graph_id, - ) - .await?; - - let challenge_init_txid = generate_bitvm_graph(graph_params.clone())? - .watchtower_challenge_init - .tx() - .compute_txid() - .to_byte_array(); - graph_params.pubin_disprove_constant = get_guest_constant_value( - graph_id, - challenge_init_txid, - &graph_params.watchtower_pubkeys, - )?; - let mut graph = generate_bitvm_graph(graph_params)?; - anyhow::ensure!( - graph.watchtower_challenge_init.tx().compute_txid().to_byte_array() - == challenge_init_txid, - "Operator constant unexpectedly changes the watchtower challenge init transaction" - ); - operator_pre_sign(operator_master_key.master_keypair(), &mut graph)?; - - let graph = graph.to_simplified()?; - tracing::info!( - event = "operator_soldering_proof", - stage = "graph_build", - outcome = "completed", - graph_nonce, - elapsed_ms = graph_build_started_at.elapsed().as_millis(), - "built operator-pre-signed graph" - ); - let definition_hash = hex::encode(graph.parameters_hash()?); - tracing::info!( - event = "operator_graph_creation", - outcome = "started", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - "storing operator-pre-signed graph definition" - ); - if let Err(error) = store_operator_presigned_graph(&context.local_db, &graph).await { - tracing::error!( - event = "operator_graph_creation", - outcome = "failed", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - error = %error, - "failed to store operator-pre-signed graph definition" - ); - return Err(error).context("store operator-pre-signed graph definition"); - } - tracing::info!( - event = "operator_graph_creation", - outcome = "committed", - stage = "definition_store", - graph_nonce, - definition_hash = %definition_hash, - "stored operator-pre-signed graph definition" - ); - (graph_nonce, graph, definition_hash, false) - } - }; - - let message = GOATMessage::new( - Actor::All, - GOATMessageContent::CreateGraph(CreateGraph { instance_id, graph_id, graph_nonce, graph }), - ); - let mut outbox_ids = Vec::new(); - if !already_finalized { - for ack_peer_id in &selected_verifier_peer_ids { - outbox_ids.push( - enqueue_graph_setup_outbox_message( - &context.local_db, - message.clone(), - Some(ack_peer_id), - ) - .await?, - ); - } - } - let mut storage = context.local_db.acquire().await?; - // The per-verifier outbox rows retry until each selected verifier ACKs. - // A finalized graph was already announced; only the setup cleanup is left. - let mut cancelled_setup_messages = 0; - for setup_outbox_id in obsolete_setup_outbox_ids { - cancelled_setup_messages += - storage.cancel_p2p_outbox_message(&setup_outbox_id).await? as u64; - } - drop(storage); - tracing::info!( - event = "operator_graph_creation", - outcome = "enqueued", - stage = "create_graph_outbox", - graph_nonce, - definition_hash = %definition_hash, - outbox_ids = ?outbox_ids, - cancelled_setup_messages, - "enqueued CreateGraph for selected verifier delivery" - ); - - // The outbox is durable before this cleanup. A process crash before - // insertion leaves the session for inbox recovery; a crash after insertion - // leaves the outbox for swarm publication. - let mut storage = context.local_db.acquire().await?; - let deleted_pending_sessions = storage - .delete_pending_graph_init(&instance_id, &local_operator_pubkey.to_string()) - .await - .context("delete pending graph session after CreateGraph outbox enqueue")?; - tracing::info!( - event = "operator_graph_creation", - outcome = "completed", - stage = "pending_session_delete", - graph_nonce, - definition_hash = %definition_hash, - deleted_pending_sessions, - "deleted pending graph session after CreateGraph outbox enqueue" - ); - - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_confirm_instance_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, -) -> Result<()> { - // triggered by PeginDeposit tx - // 1. read & check parameters - let instance_params = match read_instance_info_from_goat(ctx.goat_client, instance_id).await { - Ok(v) => v, - Err(e) => { - if should_ignore_invalid_pegin_data(&e, instance_id) { - return Ok(()); - } - bail!(e) - } - }; - let pegin_deposit_txid = instance_params.build_pegin_tx()?.0.tx().compute_txid(); - if !tx_on_chain(ctx.btc_client, &pegin_deposit_txid).await? { - tracing::warn!( - "Ignore ConfirmInstance for {instance_id}: pegin deposit tx {pegin_deposit_txid} not found on chain" - ); - return Ok(()); - } - // 2. save the instance data to local db - store_instance_parameters(ctx.local_db, &instance_params).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_create_graph_verifier( - context: &HeavyTaskContext, - instance_id: Uuid, - graph_id: Uuid, - graph_nonce: u64, - graph: &SimplifiedBitvmGcGraph, -) -> Result<()> { - if !message_identity_matches( - "CreateGraph", - instance_id, - graph_id, - Some(graph_nonce), - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id, - graph.parameters.graph_nonce, - ) { - return Ok(()); - } - - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let local_verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); - let Some(verifier_index) = - find_verifier_index_by_pubkey(&graph.parameters.gc_data, &local_verifier_pubkey)? - else { - return Ok(()); - }; - - let full_graph = BitvmGcGraph::from_simplified(graph)?; - validate_verifier_slot(&full_graph, verifier_index)?; - verify_graph_operator_pre_signatures(&full_graph) - .context("verify operator pre-signatures before endorsing graph parameters")?; - let Some(verifier_state) = load_babe_setup_state(&context.local_db, instance_id, graph_id)? - .and_then(|state| state.verifier) - else { - tracing::warn!( - "Ignore CreateGraph for {instance_id}:{graph_id}: missing local BABE verifier state" - ); - return Ok(()); - }; - if verifier_state.verifier_pubkey != local_verifier_pubkey { - tracing::warn!( - "Ignore CreateGraph for {instance_id}:{graph_id}: local BABE verifier pubkey mismatch" - ); - return Ok(()); - } - let Some(soldering_proof_ready) = verifier_state.soldering_proof_ready.clone() else { - tracing::warn!( - "Ignore CreateGraph for {instance_id}:{graph_id}: local verifier has no soldering proof" - ); - return Ok(()); - }; - let store_base_path = get_soldering_proof_payload_store_path()?; - let payload_path = soldering_proof_payload_store_path( - &store_base_path, - instance_id, - graph_id, - soldering_proof_ready.candidate_index, - &soldering_proof_ready.payload_hash, - )?; - let payload = read_soldering_proof_store_payload(&payload_path) - .await - .context("read local soldering proof payload for graph params endorsement")?; - let payload = decode_soldering_proof_payload(&soldering_proof_ready, &payload)?; - let (_opened, finalized, soldering) = expand_compact_soldering_proof_payload(payload) - .context("expand local soldering proof payload for graph params endorsement")?; - if finalized.len() != BABE_M_CC { - bail!("local verifier finalized BABE instance count mismatch"); - } - let epk = &verifier_state.setup_package.commits[finalized[0].index].epk; - let prover_state = - build_babe_prover_state(&verifier_state.setup_package, finalized, soldering)?; - let expected_gc_data = - extract_gc_circuit_data(local_verifier_pubkey, epk, &prover_state.h_msgs)?; - if graph.parameters.gc_data[verifier_index] != expected_gc_data { - tracing::warn!( - "Ignore CreateGraph for {instance_id}:{graph_id}: graph GC data for local verifier does not match local soldering proof" - ); - return Ok(()); - } - - let canonical_graph_params_hash = graph.canonical_graph_params_hash()?; - let signature = sign_verifier_graph_params(verifier_master_key.master_keypair(), graph)?; - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement { - instance_id, - graph_id, - verifier_pubkey: local_verifier_pubkey, - verifier_index, - canonical_graph_params_hash, - signature, - }), - ); - let serialized = message.serialize_message().await?; - let endorsement_outbox_id = - format!("verifier-graph-params-endorsement:{graph_id}:{verifier_index}"); - let mut storage = context.local_db.acquire().await?; - storage - .enqueue_p2p_outbox_message( - &endorsement_outbox_id, - message.content.event_type(), - &serialized, - ) - .await?; - let cancelled_gen_circuits = storage - .cancel_p2p_outbox_message(&format!( - "gen-circuits:{instance_id}:{graph_id}:{local_verifier_pubkey}" - )) - .await?; - let cancelled_soldering_proof = storage - .cancel_p2p_outbox_message(&format!( - "soldering-proof-ready:{graph_id}:{}:{}", - soldering_proof_ready.candidate_index, - hex::encode(soldering_proof_ready.payload_hash), - )) - .await?; - - tracing::info!( - event = "verifier_graph_validation", - outcome = "endorsement_enqueued", - instance_id = %instance_id, - graph_id = %graph_id, - verifier_index, - endorsement_outbox_id, - cancelled_gen_circuits, - cancelled_soldering_proof, - "validated CreateGraph and enqueued verifier graph params endorsement" - ); - Ok(()) -} - -async fn try_start_graph_committee_setup( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &SimplifiedBitvmGcGraph, -) -> Result<()> { - let verifier_endorsements = - get_verifier_graph_params_endorsements_for_graph(ctx.local_db, instance_id, graph_id) - .await?; - if verifier_endorsements.len() < graph.parameters.gc_data.len() { - tracing::info!( - "Defer CreateGraph for {instance_id}:{graph_id}: waiting for verifier graph params endorsements ({}/{})", - verifier_endorsements.len(), - graph.parameters.gc_data.len() - ); - return Ok(()); - } - let validation = validate_init_graph( - ctx.local_db, - ctx.btc_client, - ctx.goat_client, - graph, - &verifier_endorsements, - ) - .await; - ctx.metrics_state.record_graph_validation(validation.is_ok()); - if let Err(e) = validation { - if should_ignore_invalid_graph(&e, instance_id, graph_id, "CreateGraph", None) { - return Ok(()); - } - bail!(e) - }; - - // generate Musig2 nonces & broadcast NonceGeneration - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; - let full_graph = BitvmGcGraph::from_simplified(graph)?; - let (pub_nonces, _, nonce_sigs) = - committee_master_key.nonces_for_graph_job_with_keypair(&full_graph, instance_keypair)?; - let local_committee_pubkey = instance_keypair.public_key().into(); - if get_committee_partial_sigs_for_graph_member( - ctx.local_db, - instance_id, - graph_id, - &local_committee_pubkey, - ) - .await? - .is_some() - { - tracing::warn!( - "Ignore CreateGraph for {instance_id}:{graph_id}: local committee partial signatures already exist" - ); - return Ok(()); - } - let message_content = GOATMessageContent::NonceGeneration(NonceGeneration { - instance_id, - graph_id, - committee_pubkey: local_committee_pubkey, - pub_nonces: pub_nonces.clone(), - nonce_sigs, - }); - send_protocol_message( - ctx.swarm, - ctx.local_db, - GOATMessage::new(Actor::All, message_content), - StoredValue::Fresh, - ) - .await?; - store_committee_pub_nonces_for_graph( - ctx.local_db, - instance_id, - graph_id, - local_committee_pubkey, - pub_nonces, - ) - .await?; - maybe_vote_and_presign_graph(ctx, instance_id, graph_id, graph).await -} - -async fn graph_nonce_consensus_context( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &SimplifiedBitvmGcGraph, -) -> Result, CommitteeAggNonces, [u8; 32])>> { - let committee_pubkeys = ctx.goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - let pub_nonces_unchecked = - get_committee_pub_nonces_for_graph(ctx.local_db, instance_id, graph_id).await?; - if pub_nonces_unchecked.len() < committee_pubkeys.len() { - return Ok(None); - } - - let full_graph = BitvmGcGraph::from_simplified(graph)?; - let verifier_num = full_graph.parameters.gc_data.len(); - let watchtower_num = full_graph.parameters.watchtower_pubkeys.len(); - let mut checked_pub_nonces = Vec::with_capacity(pub_nonces_unchecked.len()); - for (pubkey, pub_nonces) in pub_nonces_unchecked { - pub_nonces.validate_length(watchtower_num, verifier_num).map_err(|error| { - anyhow!("committee public nonces from {pubkey} have invalid length: {error}") - })?; - checked_pub_nonces.push((pubkey, pub_nonces)); - } - let ordered_pub_nonces = order_committee_values( - &committee_pubkeys, - checked_pub_nonces, - "graph committee pub nonces", - )?; - let agg_nonces = nonces_aggregation(&ordered_pub_nonces)?; - let consensus_hash = - committee_nonce_consensus_hash(&full_graph, &committee_pubkeys, &ordered_pub_nonces)?; - Ok(Some((full_graph, committee_pubkeys, agg_nonces, consensus_hash))) -} - -async fn maybe_vote_and_presign_graph( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &SimplifiedBitvmGcGraph, -) -> Result<()> { - let Some((mut full_graph, committee_pubkeys, agg_nonces, consensus_hash)) = - graph_nonce_consensus_context(ctx, instance_id, graph_id, graph).await? - else { - return Ok(()); - }; - - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; - let local_committee_pubkey = instance_keypair.public_key().into(); - if let Some(stored_partial_sigs) = get_committee_partial_sigs_for_graph_member( - ctx.local_db, - instance_id, - graph_id, - &local_committee_pubkey, - ) - .await? - { - // Publish stored partial signatures without signing again. - publish_stored( - ctx, - &format!("committee-presign:{graph_id}"), - Actor::All, - GOATMessageContent::CommitteePresign(CommitteePresign { - instance_id, - graph_id, - committee_pubkey: local_committee_pubkey, - committee_partial_sigs: stored_partial_sigs, - agg_nonces, - }), - StoredValue::Found, - ) - .await?; - return endorse_graph_if_presigned(ctx, instance_id, graph_id, &full_graph).await; - } - - let consensus_votes = - get_committee_agg_nonce_consensus_for_graph(ctx.local_db, instance_id, graph_id).await?; - if let Some((_, stored_hash, _)) = consensus_votes - .iter() - .find(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) - && *stored_hash != consensus_hash - { - bail!(SpecialError::InvalidGraph(format!( - "local committee agg nonce consensus differs for graph {graph_id}" - ))); - } - // Recover delivery of the stored vote. - let local_vote = consensus_votes - .iter() - .find(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) - .map(|(_, _, signature)| *signature); - let (signature, origin) = match local_vote { - Some(signature) => (signature, StoredValue::Found), - None => { - let signature = SECP256K1 - .sign_schnorr(&SecpMessage::from_digest(consensus_hash), &instance_keypair); - store_committee_agg_nonce_consensus_for_graph( - ctx.local_db, - instance_id, - graph_id, - local_committee_pubkey, - consensus_hash, - signature, - ) - .await?; - (signature, StoredValue::Fresh) - } - }; - publish_stored( - ctx, - &format!("agg-nonce-consensus:{graph_id}"), - Actor::Committee, - GOATMessageContent::AggNonceConsensus(AggNonceConsensus { - instance_id, - graph_id, - committee_pubkey: local_committee_pubkey, - consensus_hash, - signature, - }), - origin, - ) - .await?; - - let consensus_votes = - get_committee_agg_nonce_consensus_for_graph(ctx.local_db, instance_id, graph_id).await?; - if consensus_votes.len() < committee_pubkeys.len() { - tracing::info!( - "Defer committee pre-sign for {instance_id}:{graph_id}: waiting for agg nonce consensus ({}/{})", - consensus_votes.len(), - committee_pubkeys.len(), - ); - return Ok(()); - } - let ordered_votes = order_committee_values( - &committee_pubkeys, - consensus_votes - .into_iter() - .map(|(committee_pubkey, consensus_hash, signature)| { - (committee_pubkey, (consensus_hash, signature)) - }) - .collect(), - "graph committee agg nonce consensus", - )?; - for (committee_pubkey, (received_hash, signature)) in - committee_pubkeys.iter().zip(ordered_votes) - { - if received_hash != consensus_hash { - bail!(SpecialError::InvalidGraph(format!( - "committee agg nonce consensus differs for graph {graph_id} and committee {committee_pubkey}" - ))); - } - SECP256K1 - .verify_schnorr( - &signature, - &SecpMessage::from_digest(consensus_hash), - &XOnlyPublicKey::from(*committee_pubkey), - ) - .map_err(|error| { - SpecialError::InvalidGraph(format!( - "invalid agg nonce consensus signature for graph {graph_id} and committee {committee_pubkey}: {error}" - )) - })?; - } - - // Recheck the persisted bindings at the actual signing boundary, including recovery. - let verifier_endorsements = - get_verifier_graph_params_endorsements_for_graph(ctx.local_db, instance_id, graph_id) - .await?; - validate_verifier_graph_params_endorsements(ctx.goat_client, graph, &verifier_endorsements) - .await?; - let (_, sec_nonces, _) = - committee_master_key.nonces_for_graph_job_with_keypair(&full_graph, instance_keypair)?; - let committee_partial_sigs = - committee_pre_sign(instance_keypair, sec_nonces, agg_nonces.clone(), &mut full_graph)?; - store_committee_partial_sigs_for_graph( - ctx.local_db, - instance_id, - graph_id, - local_committee_pubkey, - committee_partial_sigs.clone(), - ) - .await?; - publish_stored( - ctx, - &format!("committee-presign:{graph_id}"), - Actor::All, - GOATMessageContent::CommitteePresign(CommitteePresign { - instance_id, - graph_id, - committee_pubkey: local_committee_pubkey, - committee_partial_sigs, - agg_nonces, - }), - StoredValue::Fresh, - ) - .await?; - endorse_graph_if_presigned(ctx, instance_id, graph_id, &full_graph).await -} - -async fn endorse_graph_if_presigned( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &BitvmGcGraph, -) -> Result<()> { - let committee_pubkeys = ctx.goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - let committee_partial_sigs = - get_committee_partial_sigs_for_graph(ctx.local_db, instance_id, graph_id).await?; - if committee_partial_sigs.len() != committee_pubkeys.len() { - return Ok(()); - } - // Check the recovery gate before signing the endorsement. - let gate_key = format!("endorse-graph:{graph_id}"); - if crate::p2p_admission::protocol_republish_gate() - .is_cooling(&gate_key, std::time::Instant::now()) - { - return Ok(()); - } - - let committee_sig_for_graph = endorse_graph(ctx.goat_client, graph).await?; - let committee_sig_for_params = endorse_graph_params(graph).await?; - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; - let local_committee_pubkey = instance_keypair.public_key().into(); - let committee_evm_address = get_node_goat_address() - .ok_or_else(|| anyhow!("failed to get node goat address".to_string()))?; - let message_content = GOATMessageContent::EndorseGraph(EndorseGraph { - instance_id, - graph_id, - committee_pubkey: local_committee_pubkey, - committee_sig_for_graph: committee_sig_for_graph.as_bytes().to_vec(), - committee_sig_for_params: committee_sig_for_params.as_bytes().to_vec(), - committee_evm_address, - }); - // `Found`: whether this is the first endorsement or a repeat is exactly what - // the outbox row records, and a repeat must not be published past it. - publish_stored(ctx, &gate_key, Actor::All, message_content, StoredValue::Found).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_create_graph_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph_nonce: u64, - graph: &SimplifiedBitvmGcGraph, - content: &GOATMessageContent, -) -> Result<()> { - if !message_identity_matches( - "CreateGraph", - instance_id, - graph_id, - Some(graph_nonce), - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id, - graph.parameters.graph_nonce, - ) { - return Ok(()); - } - - // received from Operator - if graph.parameters.graph_nonce > 0 { - let previous_nonce = graph.parameters.graph_nonce - 1; - match get_graph_id_by_nonce(ctx.local_db, previous_nonce, &graph.parameters.operator_pubkey) - .await? - { - Some((previous_instance_id, previous_graph_id)) => { - let Some(previous_graph) = - get_graph(ctx.local_db, previous_instance_id, previous_graph_id).await? - else { - if let Err(e) = try_send_sync_graph_request( - ctx.swarm, - ctx.goat_client, - previous_instance_id, - previous_graph_id, - ) - .await - { - tracing::warn!( - "Failed to send SyncGraphRequest for previous graph {previous_instance_id}:{previous_graph_id}: {e}" - ); - } - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - 60, - MessageDeferReason::PreviousGraphPending, - "previous graph definition is not available locally", - ) - .await?; - tracing::info!( - "Defer CreateGraph for {instance_id}:{graph_id}: waiting for previous graph raw data {previous_instance_id}:{previous_graph_id}" - ); - return Ok(()); - }; - if !previous_graph.committee_pre_signed() { - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - 60, - MessageDeferReason::PreviousGraphPending, - "previous graph is waiting for committee pre-signatures", - ) - .await?; - tracing::info!( - "Defer CreateGraph for {instance_id}:{graph_id}: waiting for previous graph {previous_instance_id}:{previous_graph_id} to be committee pre-signed" - ); - return Ok(()); - } - } - None => { - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - 60, - MessageDeferReason::PreviousGraphPending, - "previous graph nonce is not available locally", - ) - .await?; - tracing::info!( - "Defer CreateGraph for {instance_id}:{graph_id}: previous graph with nonce {previous_nonce} is not available locally" - ); - return Ok(()); - } - } - } - - // 1. check graph data & operator stake without verifier params endorsements; those may arrive later. - let validation = - validate_init_graph_base(ctx.local_db, ctx.btc_client, ctx.goat_client, graph).await; - ctx.metrics_state.record_graph_validation(validation.is_ok()); - if let Err(e) = validation { - if should_ignore_invalid_graph(&e, instance_id, graph_id, "CreateGraph", None) { - return Ok(()); - } - bail!(e) - }; - // 2. save the graph data to local db - store_operator_presigned_graph(ctx.local_db, graph).await?; - // 3. start committee setup once verifier params endorsements are complete - try_start_graph_committee_setup(ctx, instance_id, graph_id, graph).await -} - -#[allow(clippy::too_many_arguments)] -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_verifier_graph_params_endorsement_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - verifier_pubkey: &PublicKey, - verifier_index: usize, - canonical_graph_params_hash: [u8; 32], - signature: &secp256k1::schnorr::Signature, - content: &GOATMessageContent, -) -> Result<()> { - if !ctx.goat_client.committee_mana_is_verifier(&ctx.from_peer_id.to_bytes()).await? { - tracing::warn!( - "Ignore VerifierGraphParamsEndorsement for {instance_id}:{graph_id}: sender {} is not a registered verifier", - ctx.from_peer_id - ); - return Ok(()); - } - - let message = make_message(ctx, content); - let graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - &message, - ) - .await? - { - Some(graph) => graph, - None => return Ok(()), - }; - let expected_hash = graph.canonical_graph_params_hash()?; - if canonical_graph_params_hash != expected_hash { - tracing::warn!( - "Ignore VerifierGraphParamsEndorsement for {instance_id}:{graph_id} from {verifier_pubkey}: canonical graph params hash mismatch" - ); - return Ok(()); - } - if verifier_index >= graph.parameters.gc_data.len() { - tracing::warn!( - "Ignore VerifierGraphParamsEndorsement for {instance_id}:{graph_id} from {verifier_pubkey}: verifier index {verifier_index} out of range" - ); - return Ok(()); - } - if graph.parameters.gc_data[verifier_index].verifier_pubkey != *verifier_pubkey { - tracing::warn!( - "Ignore VerifierGraphParamsEndorsement for {instance_id}:{graph_id}: verifier pubkey does not own slot {verifier_index}" - ); - return Ok(()); - } - if !verify_verifier_graph_params_endorsement(verifier_pubkey, &graph, signature)? { - tracing::warn!( - "Ignore VerifierGraphParamsEndorsement for {instance_id}:{graph_id} from {verifier_pubkey}: invalid signature" - ); - return Ok(()); - } - - store_verifier_graph_params_endorsement( - ctx.local_db, - instance_id, - graph_id, - *verifier_pubkey, - verifier_index, - &ctx.from_peer_id.to_string(), - *signature, - ) - .await?; - try_start_graph_committee_setup(ctx, instance_id, graph_id, &graph).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_nonce_generation_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - pub_nonces: &CommitteePubNonces, - nonce_sigs: &CommitteeNonceSignatures, - content: &GOATMessageContent, -) -> Result<()> { - if graph_signing_round_is_over(ctx.local_db, instance_id, graph_id).await? { - tracing::debug!( - "Ignore NonceGeneration for {instance_id}:{graph_id}: signing round is over" - ); - return Ok(()); - } - // received from Committee members - if !ensure_self_or_valid_committee( - ctx, - instance_id, - Some(graph_id), - received_committee_pubkey, - "NonceGeneration", - ) - .await? - { - return Ok(()); - } - // 1. check pub_nonces & nonce signatures - let message = make_message(ctx, content); - let graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - &message, - ) - .await? - { - Some(g) => g, - None => return Ok(()), - }; - let watchtower_num = graph.parameters.watchtower_pubkeys.len(); - let verifier_num = graph.parameters.gc_data.len(); - let committee_xonly_pubkey = XOnlyPublicKey::from(*received_committee_pubkey); - if !verify_nonce_signatures( - &committee_xonly_pubkey, - pub_nonces, - nonce_sigs, - watchtower_num, - verifier_num, - )? { - tracing::warn!( - "Ignore NonceGeneration for {instance_id}:{graph_id} from {}: invalid pub_nonces or nonce_sigs", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - // 2. Save immutable pub_nonces for this committee member and graph. - store_committee_pub_nonces_for_graph( - ctx.local_db, - instance_id, - graph_id, - *received_committee_pubkey, - pub_nonces.clone(), - ) - .await?; - maybe_vote_and_presign_graph(ctx, instance_id, graph_id, &graph).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_agg_nonce_consensus_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - received_consensus_hash: [u8; 32], - signature: &secp256k1::schnorr::Signature, - content: &GOATMessageContent, -) -> Result<()> { - if graph_signing_round_is_over(ctx.local_db, instance_id, graph_id).await? { - tracing::debug!( - "Ignore AggNonceConsensus for {instance_id}:{graph_id}: signing round is over" - ); - return Ok(()); - } - if !ensure_self_or_valid_committee( - ctx, - instance_id, - Some(graph_id), - received_committee_pubkey, - "AggNonceConsensus", - ) - .await? - { - return Ok(()); - } - - let message = make_message(ctx, content); - let graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - &message, - ) - .await? - { - Some(graph) => graph, - None => return Ok(()), - }; - let Some((_, _, _, expected_consensus_hash)) = - graph_nonce_consensus_context(ctx, instance_id, graph_id, &graph).await? - else { - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::CommitteeNoncesPending, - "waiting for committee public nonces before validating agg nonce consensus", - ) - .await?; - tracing::info!( - "Defer AggNonceConsensus for {instance_id}:{graph_id}: waiting for committee pub nonces" - ); - return Ok(()); - }; - if received_consensus_hash != expected_consensus_hash { - tracing::warn!( - "Ignore AggNonceConsensus for {instance_id}:{graph_id} from {}: consensus hash mismatch", - received_committee_pubkey, - ); - return Ok(()); - } - if SECP256K1 - .verify_schnorr( - signature, - &SecpMessage::from_digest(expected_consensus_hash), - &XOnlyPublicKey::from(*received_committee_pubkey), - ) - .is_err() - { - tracing::warn!( - "Ignore AggNonceConsensus for {instance_id}:{graph_id} from {}: invalid signature", - received_committee_pubkey, - ); - return Ok(()); - } - store_committee_agg_nonce_consensus_for_graph( - ctx.local_db, - instance_id, - graph_id, - *received_committee_pubkey, - received_consensus_hash, - *signature, - ) - .await?; - maybe_vote_and_presign_graph(ctx, instance_id, graph_id, &graph).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_nonce_generation_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - pub_nonces: &CommitteePubNonces, - nonce_sigs: &CommitteeNonceSignatures, -) -> Result<()> { - // received from Committee members - if !ensure_self_or_valid_committee( - ctx, - instance_id, - Some(graph_id), - received_committee_pubkey, - "NonceGeneration", - ) - .await? - { - return Ok(()); - } - let graph = match get_graph(ctx.local_db, instance_id, graph_id).await? { - Some(g) => g, - None => { - tracing::warn!( - "Ignore NonceGeneration for {instance_id}:{graph_id} from {}: graph not found, maybe belongs to another Operator", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - }; - let verifier_num = graph.parameters.gc_data.len(); - let watchtower_num = graph.parameters.watchtower_pubkeys.len(); - // 1. check pub_nonces & nonce signatures - let committee_xonly_pubkey = XOnlyPublicKey::from(*received_committee_pubkey); - if !verify_nonce_signatures( - &committee_xonly_pubkey, - pub_nonces, - nonce_sigs, - watchtower_num, - verifier_num, - )? { - tracing::warn!( - "Ignore NonceGeneration for {instance_id}:{graph_id} from {}: invalid pub_nonces or nonce_sigs", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - if let Err(e) = pub_nonces.validate_length(watchtower_num, verifier_num) { - tracing::warn!( - "Ignore NonceGeneration for {instance_id}:{graph_id} from {}: invalid pub_nonces length: {e}", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - // 2. Save immutable pub_nonces for this committee member and graph. - store_committee_pub_nonces_for_graph( - ctx.local_db, - instance_id, - graph_id, - *received_committee_pubkey, - pub_nonces.clone(), - ) - .await?; - // 3. if received enough endorsement signatures, mark the graph as endorsed, send the graph to local db, broadcast GraphFinalize - // Operator may receive EndorseGraph, CommitteePresign or NonceGeneration messages in any order - // So we need to check if we have collected enough endorsements, pub_nonces and partial_sigs every time we receive them - if let Some((finalized_graph, _)) = try_finalize_graph( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - Some(&graph), - true, - ) - .await? - { - refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; - } - Ok(()) -} - -async fn validate_committee_presign_for_graph( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - committee_partial_sigs: &CommitteePartialSignatures, - received_agg_nonces: &CommitteeAggNonces, - content: &GOATMessageContent, -) -> Result> { - let message = make_message(ctx, content); - let graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - &message, - ) - .await? - { - Some(g) => g, - None => return Ok(None), - }; - let graph = BitvmGcGraph::from_simplified(&graph)?; - let committee_pubkeys = ctx.goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - let pub_nonces_unchecked = - get_committee_pub_nonces_for_graph(ctx.local_db, instance_id, graph_id).await?; - if pub_nonces_unchecked.len() != committee_pubkeys.len() { - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::CommitteeNoncesPending, - "waiting for committee public nonces", - ) - .await?; - tracing::info!( - "Defer CommitteePresign for {instance_id}:{graph_id}: waiting for committee pub nonces" - ); - return Ok(None); - } - let received_pub_nonces = match pub_nonces_unchecked - .iter() - .find(|(pubkey, _)| pubkey == received_committee_pubkey) - .map(|(_, pub_nonces)| pub_nonces.clone()) - { - Some(pub_nonces) => pub_nonces, - None => { - tracing::warn!( - "Ignore CommitteePresign for {instance_id}:{graph_id} from {}: missing signer pub nonces", - received_committee_pubkey - ); - return Ok(None); - } - }; - let pub_nonces = match order_committee_values( - &committee_pubkeys, - pub_nonces_unchecked, - "graph committee pub nonces", - ) { - Ok(pub_nonces) => pub_nonces, - Err(e) => { - tracing::warn!( - "Ignore CommitteePresign for {instance_id}:{graph_id} from {}: invalid pub nonce set: {e}", - received_committee_pubkey - ); - return Ok(None); - } - }; - let agg_nonces = nonces_aggregation(&pub_nonces)?; - if &agg_nonces != received_agg_nonces { - tracing::warn!( - "Ignore CommitteePresign for {instance_id}:{graph_id} from {}: agg_nonces mismatch", - received_committee_pubkey - ); - return Ok(None); - } - if let Err(e) = verify_graph_committee_partial_sigs( - &graph, - &committee_pubkeys, - received_committee_pubkey, - &received_pub_nonces, - &agg_nonces, - committee_partial_sigs, - ) { - tracing::warn!( - "Ignore CommitteePresign for {instance_id}:{graph_id} from {}: invalid partial signatures: {e}", - received_committee_pubkey - ); - return Ok(None); - } - Ok(Some(graph)) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_committee_presign_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - committee_partial_sigs: &CommitteePartialSignatures, - _agg_nonces: &CommitteeAggNonces, - content: &GOATMessageContent, -) -> Result<()> { - if graph_signing_round_is_over(ctx.local_db, instance_id, graph_id).await? { - tracing::debug!( - "Ignore CommitteePresign for {instance_id}:{graph_id}: signing round is over" - ); - return Ok(()); - } - // received from Committee members - if !ensure_self_or_valid_committee( - ctx, - instance_id, - Some(graph_id), - received_committee_pubkey, - "CommitteePresign", - ) - .await? - { - return Ok(()); - } - let graph = match validate_committee_presign_for_graph( - ctx, - instance_id, - graph_id, - received_committee_pubkey, - committee_partial_sigs, - _agg_nonces, - content, - ) - .await? - { - Some(graph) => graph, - None => return Ok(()), - }; - // 1. save the validated committee partial sigs to local db - store_committee_partial_sigs_for_graph( - ctx.local_db, - instance_id, - graph_id, - *received_committee_pubkey, - committee_partial_sigs.clone(), - ) - .await?; - // 2. This also covers the case where the local partial signature was the last one and is - // not delivered back by gossip. - endorse_graph_if_presigned(ctx, instance_id, graph_id, &graph).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_committee_presign_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - committee_partial_sigs: &CommitteePartialSignatures, - _agg_nonces: &CommitteeAggNonces, - content: &GOATMessageContent, -) -> Result<()> { - // received from Committee members - if !ensure_self_or_valid_committee( - ctx, - instance_id, - Some(graph_id), - received_committee_pubkey, - "CommitteePresign", - ) - .await? - { - return Ok(()); - } - if validate_committee_presign_for_graph( - ctx, - instance_id, - graph_id, - received_committee_pubkey, - committee_partial_sigs, - _agg_nonces, - content, - ) - .await? - .is_none() - { - return Ok(()); - } - // 1. save the validated committee partial sigs to local db - store_committee_partial_sigs_for_graph( - ctx.local_db, - instance_id, - graph_id, - *received_committee_pubkey, - committee_partial_sigs.clone(), - ) - .await?; - // 3. if received enough endorsement signatures, mark the graph as endorsed, send the graph to local database, broadcast GraphFinalize - // Operator may receive EndorseGraph, CommitteePresign or NonceGeneration messages in any order - // So we need to check if we have collected enough endorsements, pub_nonces and partial_sigs every time we receive them - if let Some((finalized_graph, _)) = try_finalize_graph( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - None, - true, - ) - .await? - { - refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_endorse_graph_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - received_committee_pubkey: &PublicKey, - committee_sig_for_graph: &[u8], - committee_sig_for_params: &[u8], - committee_evm_address: &alloy::primitives::Address, -) -> Result<()> { - // received from Committee members - if !ensure_self_or_valid_committee_with_evm( - ctx, - instance_id, - Some(graph_id), - received_committee_pubkey, - committee_evm_address, - "EndorseGraph", - ) - .await? - { - return Ok(()); - } - // 1. check endorsement signature - let graph = match get_graph(ctx.local_db, instance_id, graph_id).await? { - Some(g) => g, - None => { - tracing::warn!( - "Ignore EndorseGraph for {instance_id}:{graph_id} from {}: graph not found, maybe belongs to another Operator", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - }; - let full_graph = BitvmGcGraph::from_simplified(&graph)?; - if let Err(e) = verify_graph_endorsement( - ctx.goat_client, - committee_evm_address, - &full_graph, - committee_sig_for_graph, - ) - .await - { - tracing::warn!( - "Ignore EndorseGraph for {instance_id}:{graph_id} from {}: invalid endorsement signature: {e}", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - match verify_graph_params_endorsement( - committee_evm_address, - &full_graph, - committee_sig_for_params, - ) { - Ok(true) => {} - Ok(false) => { - tracing::warn!( - "Ignore EndorseGraph for {instance_id}:{graph_id} from {}: invalid params endorsement signature", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - Err(e) => { - tracing::warn!( - "Ignore EndorseGraph for {instance_id}:{graph_id} from {}: invalid params endorsement signature: {e}", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - } - // 2. save the endorsement signature to local db - store_committee_endorsement_for_graph( - ctx.local_db, - instance_id, - graph_id, - *received_committee_pubkey, - *committee_evm_address, - committee_sig_for_graph.to_owned(), - committee_sig_for_params.to_owned(), - ) - .await?; - // 3. if received enough endorsement signatures, mark the graph as endorsed, send the graph to local database, broadcast GraphFinalize - // Operator may receive EndorseGraph, CommitteePresign or NonceGeneration messages in any order - // So we need to check if we have collected enough endorsements, pub_nonces and partial_sigs every time we receive them - if let Some((finalized_graph, _)) = try_finalize_graph( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - Some(&graph), - true, - ) - .await? - { - refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_graph_finalize_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph_nonce: u64, - graph: &SimplifiedBitvmGcGraph, - endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], - params_endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], -) -> Result<()> { - if !message_identity_matches( - "GraphFinalize", - instance_id, - graph_id, - Some(graph_nonce), - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id, - graph.parameters.graph_nonce, - ) { - return Ok(()); - } - - // received from Operator - // 1. check graph data - let validation = validate_finalized_graph( - ctx.btc_client, - ctx.goat_client, - graph, - endorse_sigs, - params_endorse_sigs, - ) - .await; - ctx.metrics_state.record_graph_validation(validation.is_ok()); - if let Err(e) = validation { - if should_ignore_invalid_graph( - &e, - instance_id, - graph_id, - "GraphFinalize", - Some(&ctx.from_peer_id), - ) { - return Ok(()); - } - bail!(e) - } - // 2. Store a finalized graph only when it upgrades the local graph. - let _ = store_finalized_graph_if_needed(ctx.local_db, graph).await?; - store_committee_endorsements_for_graph( - ctx.local_db, - instance_id, - graph_id, - endorse_sigs.to_owned(), - params_endorse_sigs.to_owned(), - ) - .await?; - // After storing, mark the graph as finalized for the instance threshold. - mark_graph_as_endorsed(ctx.local_db, instance_id, graph_id).await?; - try_transition_instance_to_presigned(ctx.local_db, instance_id).await?; - let finalized_graph = BitvmGcGraph::from_simplified(graph)?; - refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; - // 3. if endorsed graph count >= threshold, generate & broadcast PeginConfirmNonce - if has_required_presigned_graphs(ctx.local_db, instance_id).await? { - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; - let local_committee_pubkey = instance_keypair.public_key().into(); - let stored_pub_nonce = get_committee_pub_nonce_for_instance( - ctx.local_db, - instance_id, - &local_committee_pubkey, - ) - .await?; - if stored_pub_nonce.is_none() { - if get_committee_partial_sig_for_instance( - ctx.local_db, - instance_id, - &local_committee_pubkey, - ) - .await? - .is_some() - { - tracing::warn!( - "Skip PeginConfirm nonce for {instance_id}: local committee partial signature already exists" - ); - } else { - let (_, pub_nonce, nonce_sig) = committee_master_key - .nonce_for_instance_job_with_keypair( - &graph.parameters.instance_parameters, - instance_keypair, - )?; - let message_content = GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { - instance_id, - committee_pubkey: local_committee_pubkey, - pub_nonce: pub_nonce.clone(), - nonce_sig, - }); - store_committee_pub_nonce_for_instance( - ctx.local_db, - instance_id, - local_committee_pubkey, - pub_nonce, - ) - .await?; - send_protocol_message( - ctx.swarm, - ctx.local_db, - GOATMessage::new(Actor::Committee, message_content), - StoredValue::Fresh, - ) - .await?; - } - } - if let Some(pub_nonce) = stored_pub_nonce { - // Recover the stored nonce proof after Presigned without recomputing MuSig signatures. - let (_, expected_nonce, nonce_sig) = committee_master_key - .nonce_for_instance_job_with_keypair( - &graph.parameters.instance_parameters, - instance_keypair, - )?; - if expected_nonce != pub_nonce { - bail!("stored pegin public nonce differs from deterministic nonce"); - } - publish_stored( - ctx, - &format!("pegin-nonce:{instance_id}"), - Actor::Committee, - GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { - instance_id, - committee_pubkey: local_committee_pubkey, - pub_nonce, - nonce_sig, - }), - StoredValue::Found, - ) - .await?; - } - maybe_vote_and_sign_pegin_confirm(ctx, instance_id).await?; - } - // 4. (Relayer) try to call Gateway.postGraphData - // GraphFinalize may come after PostReady, so we need to check it here - if is_relayer() { - let pegin_data = ctx.goat_client.gateway_get_pegin_data(&instance_id).await?; - if pegin_data.status != PeginStatus::Withdrawable { - // pegin not posted yet - return Ok(()); - } - let graph_data = ctx.goat_client.gateway_get_graph_data(&graph_id).await?; - if graph_data.operator_pubkey != [0u8; 32] { - // already posted - return Ok(()); - } - let graph = BitvmGcGraph::from_simplified(graph)?; - let graph_data = build_graph_data(&graph)?; - let endorse_sigs = endorse_sigs.iter().map(|(_, _, sig)| sig.clone()).collect::>(); - ctx.goat_client - .gateway_post_graph_data(&instance_id, &graph_id, &graph_data, &endorse_sigs) - .await?; - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_graph_finalize_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph_nonce: u64, - graph: &SimplifiedBitvmGcGraph, - endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], - params_endorse_sigs: &[(PublicKey, alloy::primitives::Address, Vec)], -) -> Result<()> { - if !message_identity_matches( - "GraphFinalize", - instance_id, - graph_id, - Some(graph_nonce), - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id, - graph.parameters.graph_nonce, - ) { - return Ok(()); - } - - // received from Operator - // 1. check graph data - let validation = validate_finalized_graph( - ctx.btc_client, - ctx.goat_client, - graph, - endorse_sigs, - params_endorse_sigs, - ) - .await; - ctx.metrics_state.record_graph_validation(validation.is_ok()); - if let Err(e) = validation { - if should_ignore_invalid_graph( - &e, - instance_id, - graph_id, - "GraphFinalize", - Some(&ctx.from_peer_id), - ) { - return Ok(()); - } - bail!(e) - } - // 2. Store a finalized graph only when it upgrades the local graph. - let _ = store_finalized_graph_if_needed(ctx.local_db, graph).await?; - store_committee_endorsements_for_graph( - ctx.local_db, - instance_id, - graph_id, - endorse_sigs.to_owned(), - params_endorse_sigs.to_owned(), - ) - .await?; - mark_graph_as_endorsed(ctx.local_db, instance_id, graph_id).await?; - try_transition_instance_to_presigned(ctx.local_db, instance_id).await?; - let finalized_graph = BitvmGcGraph::from_simplified(graph)?; - refresh_newly_finalized_graph(ctx, instance_id, graph_id, &finalized_graph).await?; - Ok(()) -} - -async fn pegin_confirm_nonce_consensus_context( - ctx: &HandlerContext<'_>, - instance_id: Uuid, -) -> Result< - Option<( - BitvmGcInstanceParameters, - Vec, - Vec, - musig2::AggNonce, - [u8; 32], - )>, -> { - let instance_parameters = get_instance_parameters(ctx.local_db, instance_id) - .await? - .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; - let committee_pubkeys = ctx.goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - let pub_nonces = get_committee_pub_nonces_for_instance(ctx.local_db, instance_id).await?; - if pub_nonces.len() < committee_pubkeys.len() { - return Ok(None); - } - let ordered_pub_nonces = - order_committee_values(&committee_pubkeys, pub_nonces, "pegin committee pub nonces")?; - let agg_nonce = nonce_aggregation(&ordered_pub_nonces); - let consensus_hash = pegin_confirm_nonce_consensus_hash( - &instance_parameters, - &committee_pubkeys, - &ordered_pub_nonces, - )?; - Ok(Some(( - instance_parameters, - committee_pubkeys, - ordered_pub_nonces, - agg_nonce, - consensus_hash, - ))) -} - -async fn has_complete_pegin_confirm_nonce_consensus( - ctx: &HandlerContext<'_>, - instance_id: Uuid, - committee_pubkeys: &[PublicKey], - consensus_hash: [u8; 32], -) -> Result { - let consensus_votes = - get_committee_agg_nonce_consensus_for_instance(ctx.local_db, instance_id).await?; - if consensus_votes.len() != committee_pubkeys.len() { - return Ok(false); - } - let ordered_votes = order_committee_values( - committee_pubkeys, - consensus_votes - .into_iter() - .map(|(committee_pubkey, vote_hash, signature)| { - (committee_pubkey, (vote_hash, signature)) - }) - .collect(), - "PeginConfirm agg nonce consensus", - )?; - for (committee_pubkey, (received_hash, signature)) in - committee_pubkeys.iter().zip(ordered_votes) - { - ensure!( - received_hash == consensus_hash, - "committee PeginConfirm nonce consensus differs for instance {instance_id} and committee {committee_pubkey}" - ); - SECP256K1 - .verify_schnorr( - &signature, - &SecpMessage::from_digest(consensus_hash), - &XOnlyPublicKey::from(*committee_pubkey), - ) - .map_err(|error| { - SpecialError::InvalidPeginData(format!( - "invalid PeginConfirm nonce consensus signature for instance {instance_id} and committee {committee_pubkey}: {error}" - )) - })?; - } - Ok(true) -} - -async fn broadcast_pegin_confirm_if_presigned( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - instance_parameters: &BitvmGcInstanceParameters, - committee_pubkeys: &[PublicKey], -) -> Result<()> { - if !is_relayer() { - return Ok(()); - } - let pub_nonces = get_committee_pub_nonces_for_instance(ctx.local_db, instance_id).await?; - let partial_sigs = get_committee_partial_sigs_for_instance(ctx.local_db, instance_id).await?; - if pub_nonces.len() != committee_pubkeys.len() || partial_sigs.len() != committee_pubkeys.len() - { - return Ok(()); - } - let pub_nonces = - order_committee_values(committee_pubkeys, pub_nonces, "pegin committee pub nonces")?; - let partial_sigs = - order_committee_values(committee_pubkeys, partial_sigs, "pegin committee partial sigs")?; - let mut pegin_confirm = instance_parameters.build_pegin_tx()?.1; - let agg_nonce = nonce_aggregation(&pub_nonces); - let context = instance_parameters.get_base_context(); - let full_sig = pegin_confirm - .aggregate_input_0_musig2_signatures(&context, partial_sigs, &agg_nonce) - .map_err(|error| { - anyhow!("Failed to aggregate PeginConfirm signatures for {instance_id}: {error}") - })?; - let connector_z = instance_parameters.connector_z(); - pegin_confirm.push_input_0_signature(&connector_z, full_sig); - let result = broadcast_tx(ctx.btc_client, pegin_confirm.tx()).await; - ctx.metrics_state.record_pegin_confirm(result.is_ok()); - result?; - Ok(()) -} - -async fn maybe_vote_and_sign_pegin_confirm( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, -) -> Result<()> { - let Some((instance_parameters, committee_pubkeys, _pub_nonces, agg_nonce, consensus_hash)) = - pegin_confirm_nonce_consensus_context(ctx, instance_id).await? - else { - return Ok(()); - }; - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; - let local_committee_pubkey = instance_keypair.public_key().into(); - if let Some(stored_partial_sig) = - get_committee_partial_sig_for_instance(ctx.local_db, instance_id, &local_committee_pubkey) - .await? - { - // Recreate a missing endorsement while preserving the stored partial signature. - let stored_endorse_sig = get_committee_endorse_sigs_for_pegin(ctx.local_db, instance_id) - .await? - .into_iter() - .find(|(committee_pubkey, _)| *committee_pubkey == local_committee_pubkey) - .map(|(_, endorse_sig)| endorse_sig); - let endorse_sig = match stored_endorse_sig { - Some(endorse_sig) => endorse_sig, - None => { - let pegin_txid = instance_parameters.build_pegin_tx()?.1.tx().compute_txid(); - let endorse_sig = endorse_pegin(ctx.goat_client, instance_id, &pegin_txid) - .await? - .as_bytes() - .to_vec(); - store_committee_endorse_sig_for_pegin( - ctx.local_db, - instance_id, - local_committee_pubkey, - endorse_sig.clone(), - ) - .await?; - endorse_sig - } - }; - publish_stored( - ctx, - &format!("pegin-confirm-partial-sig:{instance_id}"), - Actor::Committee, - GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { - instance_id, - committee_pubkey: local_committee_pubkey, - partial_sig: stored_partial_sig, - endorse_sig, - }), - StoredValue::Found, - ) - .await?; - return broadcast_pegin_confirm_if_presigned( - ctx, - instance_id, - &instance_parameters, - &committee_pubkeys, - ) - .await; - } - - let consensus_votes = - get_committee_agg_nonce_consensus_for_instance(ctx.local_db, instance_id).await?; - if let Some((_, stored_hash, _)) = consensus_votes - .iter() - .find(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) - && *stored_hash != consensus_hash - { - bail!(SpecialError::InvalidPeginData(format!( - "local PeginConfirm nonce consensus differs for instance {instance_id}" - ))); - } - // Stored, then published: see the same step for graphs in - // `maybe_vote_and_presign_graph`. - let local_vote = consensus_votes - .iter() - .find(|(committee_pubkey, _, _)| *committee_pubkey == local_committee_pubkey) - .map(|(_, _, signature)| *signature); - let (signature, origin) = match local_vote { - Some(signature) => (signature, StoredValue::Found), - None => { - let signature = SECP256K1 - .sign_schnorr(&SecpMessage::from_digest(consensus_hash), &instance_keypair); - store_committee_agg_nonce_consensus_for_instance( - ctx.local_db, - instance_id, - local_committee_pubkey, - consensus_hash, - signature, - ) - .await?; - (signature, StoredValue::Fresh) - } - }; - publish_stored( - ctx, - &format!("pegin-confirm-nonce-consensus:{instance_id}"), - Actor::Committee, - GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { - instance_id, - committee_pubkey: local_committee_pubkey, - consensus_hash, - signature, - }), - origin, - ) - .await?; - if !has_complete_pegin_confirm_nonce_consensus( - ctx, - instance_id, - &committee_pubkeys, - consensus_hash, - ) - .await? - { - tracing::info!( - "Defer PeginConfirm partial signature for {instance_id}: waiting for agg nonce consensus" - ); - return Ok(()); - } - - let (sec_nonce, _, _) = committee_master_key - .nonce_for_instance_job_with_keypair(&instance_parameters, instance_keypair)?; - let mut pegin_confirm = instance_parameters.build_pegin_tx()?.1; - let committee_context = instance_parameters.get_committee_context(instance_keypair)?; - let partial_sig = pegin_confirm - .sign_input_0_musig2(&committee_context, &sec_nonce, &agg_nonce) - .map_err(|error| anyhow!("Failed to sign PeginConfirm for {instance_id}: {error}"))?; - let endorse_sig = - endorse_pegin(ctx.goat_client, instance_id, &pegin_confirm.tx().compute_txid()).await?; - store_committee_partial_sig_for_instance( - ctx.local_db, - instance_id, - local_committee_pubkey, - partial_sig, - ) - .await?; - store_committee_endorse_sig_for_pegin( - ctx.local_db, - instance_id, - local_committee_pubkey, - endorse_sig.as_bytes().to_vec(), - ) - .await?; - publish_stored( - ctx, - &format!("pegin-confirm-partial-sig:{instance_id}"), - Actor::Committee, - GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { - instance_id, - committee_pubkey: local_committee_pubkey, - partial_sig, - endorse_sig: endorse_sig.as_bytes().to_vec(), - }), - StoredValue::Fresh, - ) - .await?; - broadcast_pegin_confirm_if_presigned(ctx, instance_id, &instance_parameters, &committee_pubkeys) - .await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_pegin_confirm_nonce_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - received_committee_pubkey: &PublicKey, - pub_nonce: &musig2::PubNonce, - nonce_sig: &secp256k1::schnorr::Signature, - content: &GOATMessageContent, -) -> Result<()> { - // received from Committee members - if !ensure_self_or_valid_committee( - ctx, - instance_id, - None, - received_committee_pubkey, - "PeginConfirmNonce", - ) - .await? - { - return Ok(()); - } - // 1. check pub_nonce - if !verify_public_nonce(nonce_sig, pub_nonce, &XOnlyPublicKey::from(*received_committee_pubkey)) - { - tracing::warn!( - "Ignore PeginConfirmNonce for {instance_id} from {}: invalid pub_nonce or nonce_sig", - received_committee_pubkey.to_string() - ); - return Ok(()); - } - // 2. save the pub_nonce to local db - store_committee_pub_nonce_for_instance( - ctx.local_db, - instance_id, - *received_committee_pubkey, - pub_nonce.clone(), - ) - .await?; - redeliver_own_pegin_confirm_value( - ctx, - instance_id, - &format!("pegin-nonce:{instance_id}"), - received_committee_pubkey, - content, - ) - .await?; - // 3. Agree on the full nonce transcript before generating a partial signature. - maybe_vote_and_sign_pegin_confirm(ctx, instance_id).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_pegin_confirm_nonce_consensus_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - received_committee_pubkey: &PublicKey, - received_consensus_hash: [u8; 32], - signature: &secp256k1::schnorr::Signature, - content: &GOATMessageContent, -) -> Result<()> { - if !ensure_self_or_valid_committee( - ctx, - instance_id, - None, - received_committee_pubkey, - "PeginConfirmNonceConsensus", - ) - .await? - { - return Ok(()); - } - let message = make_message(ctx, content); - let Some((_, _, _, _, expected_consensus_hash)) = - pegin_confirm_nonce_consensus_context(ctx, instance_id).await? - else { - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::CommitteeNoncesPending, - "waiting for committee public nonces before validating PeginConfirm nonce consensus", - ) - .await?; - tracing::info!( - "Defer PeginConfirmNonceConsensus for {instance_id}: waiting for committee pub nonces" - ); - return Ok(()); - }; - if received_consensus_hash != expected_consensus_hash { - tracing::warn!( - "Ignore PeginConfirmNonceConsensus for {instance_id} from {}: consensus hash mismatch", - received_committee_pubkey - ); - return Ok(()); - } - if SECP256K1 - .verify_schnorr( - signature, - &SecpMessage::from_digest(expected_consensus_hash), - &XOnlyPublicKey::from(*received_committee_pubkey), - ) - .is_err() - { - tracing::warn!( - "Ignore PeginConfirmNonceConsensus for {instance_id} from {}: invalid signature", - received_committee_pubkey - ); - return Ok(()); - } - store_committee_agg_nonce_consensus_for_instance( - ctx.local_db, - instance_id, - *received_committee_pubkey, - received_consensus_hash, - *signature, - ) - .await?; - // A locally queued copy is the recovery monitor re-delivering this node's own - // stored vote, as the nonce and partial-signature handlers do for theirs. - redeliver_own_pegin_confirm_value( - ctx, - instance_id, - &format!("pegin-confirm-nonce-consensus:{instance_id}"), - received_committee_pubkey, - content, - ) - .await?; - maybe_vote_and_sign_pegin_confirm(ctx, instance_id).await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_pegin_confirm_partial_sig_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - received_committee_pubkey: &PublicKey, - partial_sig: &musig2::PartialSignature, - endorse_sig: &[u8], - content: &GOATMessageContent, -) -> Result<()> { - // received from Committee members - if !ensure_self_or_valid_committee( - ctx, - instance_id, - None, - received_committee_pubkey, - "PeginConfirmPartialSig", - ) - .await? - { - return Ok(()); - } - let message = make_message(ctx, content); - let committee_pubkeys = ctx.goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - let pub_nonces_unchecked = - get_committee_pub_nonces_for_instance(ctx.local_db, instance_id).await?; - if pub_nonces_unchecked.len() != committee_pubkeys.len() { - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::CommitteeNoncesPending, - "waiting for committee public nonces", - ) - .await?; - tracing::info!( - "Defer PeginConfirmPartialSig for {instance_id}: waiting for committee pub nonces" - ); - return Ok(()); - } - let received_pub_nonce = match pub_nonces_unchecked - .iter() - .find(|(pubkey, _)| pubkey == received_committee_pubkey) - .map(|(_, pub_nonce)| pub_nonce.clone()) - { - Some(pub_nonce) => pub_nonce, - None => { - tracing::warn!( - "Ignore PeginConfirmPartialSig for {instance_id} from {}: missing signer pub nonce", - received_committee_pubkey - ); - return Ok(()); - } - }; - let pub_nonces = match order_committee_values( - &committee_pubkeys, - pub_nonces_unchecked, - "pegin committee pub nonces", - ) { - Ok(pub_nonces) => pub_nonces, - Err(e) => { - tracing::warn!( - "Ignore PeginConfirmPartialSig for {instance_id} from {}: invalid pub nonce set: {e}", - received_committee_pubkey - ); - return Ok(()); - } - }; - let agg_nonce = nonce_aggregation(&pub_nonces); - let instance_params = get_instance_parameters(ctx.local_db, instance_id) - .await? - .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; - let consensus_hash = - pegin_confirm_nonce_consensus_hash(&instance_params, &committee_pubkeys, &pub_nonces)?; - if !has_complete_pegin_confirm_nonce_consensus( - ctx, - instance_id, - &committee_pubkeys, - consensus_hash, - ) - .await? - { - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::CommitteeNonceConsensusPending, - "waiting for committee PeginConfirm agg nonce consensus", - ) - .await?; - tracing::info!( - "Defer PeginConfirmPartialSig for {instance_id}: waiting for agg nonce consensus" - ); - return Ok(()); - } - if let Err(e) = verify_pegin_confirm_partial_sig( - &instance_params, - &committee_pubkeys, - received_committee_pubkey, - &received_pub_nonce, - &agg_nonce, - *partial_sig, - ) { - tracing::warn!( - "Ignore PeginConfirmPartialSig for {instance_id} from {}: invalid partial signature: {e}", - received_committee_pubkey - ); - return Ok(()); - } - let pegin_confirm = instance_params.build_pegin_tx()?.1; - let pegin_txid = pegin_confirm.tx().compute_txid(); - match verify_pegin_endorsement( - ctx.goat_client, - instance_id, - received_committee_pubkey, - &pegin_txid, - endorse_sig, - ) - .await - { - Ok(true) => {} - Ok(false) => { - tracing::warn!( - "Ignore PeginConfirmPartialSig for {instance_id} from {}: invalid endorsement signature", - received_committee_pubkey - ); - return Ok(()); - } - Err(e) => { - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::ValidationRetry, - &format!("failed to verify committee endorsement signature: {e}"), - ) - .await?; - tracing::warn!( - "Retry PeginConfirmPartialSig later for {instance_id} from {}: failed to verify endorsement signature: {e}", - received_committee_pubkey - ); - return Ok(()); - } - } - // 1. save the validated partial signature & endorsement signature to local db - store_committee_partial_sig_for_instance( - ctx.local_db, - instance_id, - *received_committee_pubkey, - *partial_sig, - ) - .await?; - store_committee_endorse_sig_for_pegin( - ctx.local_db, - instance_id, - *received_committee_pubkey, - endorse_sig.to_owned(), - ) - .await?; - redeliver_own_pegin_confirm_value( - ctx, - instance_id, - &format!("pegin-confirm-partial-sig:{instance_id}"), - received_committee_pubkey, - content, - ) - .await?; - broadcast_pegin_confirm_if_presigned(ctx, instance_id, &instance_params, &committee_pubkeys) - .await -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id))] -async fn handle_post_ready(ctx: &mut HandlerContext<'_>, instance_id: Uuid) -> Result<()> { - // triggered by PeginConfirm tx - if !is_relayer() { - return Ok(()); - } - // 1. (Relayer)call Gateway.postPeginData on GoatChain - let committee_pubkeys = ctx.goat_client.gateway_get_committee_pubkeys(&instance_id).await?; - let pegin_data = ctx.goat_client.gateway_get_pegin_data(&instance_id).await?; - if pegin_data.status == PeginStatus::None { - tracing::warn!("Ignore PostReady for {instance_id}: not a pending pegin request"); - return Ok(()); - } else if pegin_data.status == PeginStatus::Pending { - let instance_params = get_instance_parameters(ctx.local_db, instance_id) - .await? - .ok_or_else(|| anyhow!("Instance parameters not found for {instance_id}"))?; - let pegin_confirm = instance_params.build_pegin_tx()?.1; - let pegin_txid = pegin_confirm.tx().compute_txid(); - let pegin_tx = match ctx.btc_client.get_tx(&pegin_txid).await? { - Some(tx) => tx, - None => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = GOATMessage::new( - ctx.actor.clone(), - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinTransactionPending, - "pegin-confirm transaction is not available from the Bitcoin backend", - ) - .await?; - tracing::warn!( - "Retry postPeginData later for {instance_id}: Pegin-Confirm transaction not found on Bitcoin: {pegin_txid}" - ); - return Ok(()); - } - }; - let endorse_sigs = get_committee_endorse_sigs_for_pegin(ctx.local_db, instance_id) - .await? - .into_iter() - .map(|(_, es)| es) - .collect::>(); - if endorse_sigs.len() != committee_pubkeys.len() { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = GOATMessage::new( - ctx.actor.clone(), - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::CommitteeEndorsementsPending, - "waiting for all committee endorsements of the pegin-confirm transaction", - ) - .await?; - tracing::warn!( - "Retry postPeginData later for {instance_id}: not enough endorse sigs for pegin confirm tx: {}", - endorse_sigs.len() - ); - return Ok(()); - } - let pegin_height = match ctx.btc_client.get_tx_status(&pegin_txid).await?.block_height { - Some(height) => height as u64, - None => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = GOATMessage::new( - ctx.actor.clone(), - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "pegin-confirm transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry postPeginData later for {instance_id}: pegin confirm tx not confirmed on btc yet" - ); - return Ok(()); - } - }; - let goat_confirmed_height = ctx.goat_client.btc_spv_latest_height().await?; - if goat_confirmed_height < pegin_height { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()) - * (pegin_height - goat_confirmed_height); - let message = GOATMessage::new( - ctx.actor.clone(), - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::GoatSpvPending, - "pegin-confirm block is not available through GOAT SPV", - ) - .await?; - tracing::info!( - "Retry postPeginData later for {instance_id}: pegin confirm tx block not posted to goat spv contract yet" - ); - return Ok(()); - } - let result = ctx - .goat_client - .gateway_post_pegin_data(ctx.btc_client, &instance_id, &pegin_tx, &endorse_sigs) - .await; - ctx.metrics_state.record_pegin_post(result.is_ok()); - result?; - } else { - // already posted - } - // 2. (Relayer)call Gateway.postGraphData on GoatChain - let graph_ids = get_graph_ids_for_instance(ctx.local_db, instance_id).await?; - let mut missing_graph_endorsements = false; - for graph_id in &graph_ids { - let graph_data = ctx.goat_client.gateway_get_graph_data(graph_id).await?; - if graph_data.operator_pubkey != [0u8; 32] { - // already posted - continue; - } - let endorsement_sigs = - get_committee_endorsements_for_graph(ctx.local_db, instance_id, *graph_id) - .await? - .into_iter() - .map(|(_, _, sig)| sig) - .collect::>(); - if endorsement_sigs.len() != committee_pubkeys.len() { - missing_graph_endorsements = true; - tracing::warn!( - "Defer postGraphData for {instance_id}:{graph_id}: not enough endorse sigs for graph: {}", - endorsement_sigs.len() - ); - continue; - } - let graph = get_graph(ctx.local_db, instance_id, *graph_id) - .await? - .ok_or_else(|| anyhow!("Graph not found for {instance_id}:{graph_id}"))?; - let graph = BitvmGcGraph::from_simplified(&graph)?; - let graph_data = build_graph_data(&graph)?; - ctx.goat_client - .gateway_post_graph_data(&instance_id, graph_id, &graph_data, &endorsement_sigs) - .await?; - } - if missing_graph_endorsements { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = GOATMessage::new( - ctx.actor.clone(), - GOATMessageContent::PostReady(PostReady { instance_id }), - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::CommitteeEndorsementsPending, - "waiting for committee graph endorsements", - ) - .await?; - tracing::info!( - "Retry postGraphData later for {instance_id}: waiting for committee graph endorsements" - ); - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_kickoff_ready_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by InitWithdraw event from GoatChain - let message = make_message(ctx, content); - let graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - instance_id, - graph_id, - &message, - ) - .await? - { - Some(g) => g, - None => return Ok(()), - }; - let mut graph = BitvmGcGraph::from_simplified(&graph)?; - let operator_pubkey = graph.parameters.operator_pubkey; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let node_pubkey: PublicKey = operator_master_key.master_keypair().public_key().into(); - if node_pubkey != operator_pubkey { - tracing::warn!("Ignore KickoffReady for {instance_id}:{graph_id}: not my graph"); - return Ok(()); - } - // 1. check the withdraw status on GoatChain - let withdraw_status = ctx.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; - if withdraw_status != WithdrawStatus::Initialized { - tracing::warn!( - "Ignore KickoffReady for {instance_id}:{graph_id}: invalid withdraw status: {withdraw_status:?}" - ); - return Ok(()); - } - // 2. check prekickoff nonce & broadcast previous pre-kickoff if needed - let start_nonce = - match get_latest_pegout_finalized_graph(ctx.local_db, &operator_pubkey).await? { - Some((n, _)) => n + 1, - None => 0, - }; - for current_nonce in start_nonce..graph.parameters.graph_nonce { - let (current_instance_id, current_graph_id) = match get_graph_id_by_nonce( - ctx.local_db, - current_nonce, - &operator_pubkey, - ) - .await? - { - Some(v) => v, - None => { - tracing::warn!( - "Ignore KickoffReady for {instance_id}:{graph_id}: missing previous graph {current_nonce}" - ); - return Ok(()); - } - }; - let current_graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - current_instance_id, - current_graph_id, - &message, - ) - .await? - { - Some(g) => g, - None => return Ok(()), - }; - let mut current_graph = BitvmGcGraph::from_simplified(¤t_graph)?; - let current_graph_refresh = refresh_graph( - ctx.local_db, - ctx.btc_client, - ctx.goat_client, - current_instance_id, - current_graph_id, - ¤t_graph, - ) - .await?; - let current_graph_status = current_graph_refresh.status; - if current_graph_refresh.status_transition_accepted { - compensate_graph_events( - ctx.local_db, - ctx.btc_client, - current_instance_id, - current_graph_id, - ¤t_graph, - current_graph_refresh.scan.as_ref(), - // This is a recovery scan, not a transition from the local - // projection. Start from the protocol baseline so a previous - // crash between status persistence and message enqueue can be - // repaired by idempotent message upserts. - GraphStatus::OperatorPresigned, - current_graph_status, - ) - .await?; - } - if current_graph_status.is_closed() { - continue; - } else if current_graph_status.is_pegout_started() { - tracing::warn!( - "Ignore KickoffReady for {instance_id}:{graph_id}: previous graph {current_graph_id} already started pegout" - ); - let nonce_interval = - graph.parameters.graph_nonce - current_graph.parameters.graph_nonce; - let min_pegout_time_secs = take1_timelock_with_config( - ctx.btc_client.network(), - ¤t_graph.parameters.timelock_config, - ) as u64 - * avg_block_time_secs(ctx.btc_client.network()); - let delay_secs = min_pegout_time_secs * nonce_interval; - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::PreviousGraphPending, - "previous graph has an active pegout flow", - ) - .await?; - return Ok(()); - } else if current_graph_status.is_obsoleted() { - operator_skip_graph(ctx.btc_client, &mut current_graph).await?; - tracing::info!( - "Operator {operator_pubkey} skipped obsoleted graph {current_instance_id}:{current_graph_id}" - ); - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 blocks - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::ChainStatePending, - "waiting for the previous obsoleted graph skip transaction to propagate", - ) - .await?; - return Ok(()); - } else { - let graph_data_on_goat = - ctx.goat_client.gateway_get_graph_data(¤t_graph_id).await?; - if graph_data_on_goat.operator_pubkey != [0u8; 32] { - tracing::warn!( - "Ignore KickoffReady for {instance_id}:{graph_id}: previous available graph exists for Operator {operator_pubkey}: {current_instance_id}:{current_graph_id}, please withdraw it first" - ); - let nonce_interval = - graph.parameters.graph_nonce - current_graph.parameters.graph_nonce; - let min_pegout_time_secs = take1_timelock_with_config( - ctx.btc_client.network(), - ¤t_graph.parameters.timelock_config, - ) as u64 - * avg_block_time_secs(ctx.btc_client.network()); - let delay_secs = min_pegout_time_secs * nonce_interval; - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::PreviousGraphPending, - "previous graph is available for pegout", - ) - .await?; - return Ok(()); - } else { - operator_skip_graph(ctx.btc_client, &mut current_graph).await?; - tracing::info!( - "Operator {operator_pubkey} skipped non-posted graph {current_instance_id}:{current_graph_id}" - ); - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 blocks - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::ChainStatePending, - "waiting for the previous graph skip transaction to propagate", - ) - .await?; - return Ok(()); - } - } - } - // 3. sign & broadcast prekickoff & kickoff txns - operator_kickoff(ctx.btc_client, &mut graph).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_kickoff_sent_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Kickoff tx - // 1. update status - let (graph, _graph_status, _graph_sub_status) = - match refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::OperatorKickOff) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if !is_relayer() { - return Ok(()); - } - // 2. (Relayer) try to call Gateway.proceedWithdraw - let withdraw_status = ctx.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; - if withdraw_status != WithdrawStatus::Initialized { - tracing::warn!( - "Ignore KickoffSent for {instance_id}:{graph_id}: invalid withdraw status: {withdraw_status:?}" - ); - return Ok(()); - } - let kickoff_txid = graph.kickoff.tx().compute_txid(); - let kickoff_tx = match ctx.btc_client.get_tx(&kickoff_txid).await? { - Some(tx) => tx, - None => { - tracing::warn!( - "Ignore KickoffSent for {instance_id}:{graph_id}: kickoff tx not found on chain: {kickoff_txid}" - ); - return Ok(()); - } - }; - let kickoff_height = match ctx.btc_client.get_tx_status(&kickoff_txid).await?.block_height { - Some(height) => height as u64, - None => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "kickoff transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry proceedWithdraw later for {instance_id}:{graph_id}: kickoff tx not confirmed on btc yet" - ); - return Ok(()); - } - }; - let goat_confirmed_btc_height = ctx.goat_client.btc_spv_latest_height().await?; - if goat_confirmed_btc_height < kickoff_height { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()) - * (kickoff_height - goat_confirmed_btc_height); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::GoatSpvPending, - "kickoff block is not available through GOAT SPV", - ) - .await?; - tracing::info!( - "Retry proceedWithdraw later for {instance_id}:{graph_id}: kickoff tx block not posted to goat spv contract yet" - ); - return Ok(()); - } - ctx.goat_client.gateway_process_withdraw(ctx.btc_client, &graph_id, &kickoff_tx).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_kickoff_sent_verifier( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Kickoff tx - let message = make_message(ctx, content); - let (graph, _graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::OperatorKickOff, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - // 1. check kickoff tx status on Bitcoin chain - let kickoff_txid = graph.kickoff.tx().compute_txid(); - let kickoff_height = match ctx.btc_client.get_tx_status(&kickoff_txid).await?.block_height { - Some(height) => height, - None => { - // Retry after Bitcoin confirmation. - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "kickoff transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry KickoffSent later for {instance_id}:{graph_id}: kickoff tx not confirmed yet" - ); - return Ok(()); - } - }; - handle_previous_graph_after_prekickoff(ctx, instance_id, graph_id, &graph, &message).await?; - let take1_txid = graph.take1.tx().compute_txid(); - let (challenge_tx, _) = export_challenge_tx(&graph).unwrap(); - let kickoff_challenge_outpoint = challenge_tx.input[0].previous_output; - if let Some(spent_txid) = outpoint_spent_txid( - ctx.btc_client, - &kickoff_challenge_outpoint.txid, - kickoff_challenge_outpoint.vout as u64, - ) - .await? - { - let spent_tx_name = if spent_txid == take1_txid { "Take1" } else { "Challenge" }; - tracing::warn!( - "Ignore KickoffSent for {instance_id}:{graph_id}: challenge connector already spent by {spent_tx_name} tx: {spent_txid}" - ); - return Ok(()); - } - // 2. check withdraw status, if it's invalid, sign & broadcast challenge txn - let withdraw_status = ctx.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; - let goat_confirmed_btc_height = ctx.goat_client.btc_spv_latest_height().await? as u32; - if [WithdrawStatus::None, WithdrawStatus::Canceled].contains(&withdraw_status) { - if kickoff_height >= goat_confirmed_btc_height { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()) - * (kickoff_height - goat_confirmed_btc_height) as u64; - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::GoatSpvPending, - "kickoff block is not available through GOAT SPV", - ) - .await?; - tracing::info!( - "Retry Challenge later for {instance_id}:{graph_id}: kickoff tx block is not posted to GOAT SPV yet" - ); - return Ok(()); - } - - let (challenge_tx, _) = export_challenge_tx(&graph).unwrap(); - let challenge_txid = challenge_tx.compute_txid(); - if ctx.btc_client.get_tx(&challenge_txid).await?.is_none() { - send_challenge_tx(ctx.btc_client, &graph).await?; - } - } else { - tracing::info!( - "Ignore KickoffSent for {instance_id}:{graph_id}: withdraw already initiated, status: {withdraw_status:?}" - ); - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_kickoff_sent_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Kickoff tx - let message = make_message(ctx, content); - let _graph = refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::OperatorKickOff, - ) - .await?; - Ok(()) -} - -async fn handle_previous_graph_after_prekickoff( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &BitvmGcGraph, - message: &GOATMessage, -) -> Result<()> { - if !tx_on_chain( - ctx.btc_client, - &graph.parameters.prekickoff_parameters.cur_prekickoff_txn.tx().compute_txid(), - ) - .await? - { - tracing::warn!( - "Ignore prekickoff follow-up for {instance_id}:{graph_id}: prekickoff tx not on chain" - ); - return Ok(()); - } - - let graph_nonce = graph.parameters.graph_nonce; - if graph_nonce == 0 { - return Ok(()); - } - - let (prev_instance_id, prev_graph_id) = - get_graph_id_by_nonce(ctx.local_db, graph_nonce - 1, &graph.parameters.operator_pubkey) - .await? - .ok_or_else(|| anyhow!("Prev graph not found for {instance_id}:{graph_id}"))?; - let prev_graph = match get_graph_or_defer( - ctx.swarm, - ctx.local_db, - ctx.goat_client, - prev_instance_id, - prev_graph_id, - message, - ) - .await? - { - Some(g) => g, - None => return Ok(()), - }; - let prev_graph = BitvmGcGraph::from_simplified(&prev_graph)?; - let (prev_graph_status, _prev_graph_sub_status, status_transition_accepted) = - refresh_and_compensate( - ctx, - prev_instance_id, - prev_graph_id, - &prev_graph, - // Do not use the persisted status as a compensation anchor: it - // may have been committed just before a crash. Idempotent message - // upserts make a baseline recovery scan safe to retry. - GraphStatus::OperatorPresigned, - ) - .await?; - if !status_transition_accepted { - tracing::warn!( - "Ignore prekickoff follow-up for {instance_id}:{graph_id}: previous graph status scan was rejected" - ); - return Ok(()); - } - if !tx_on_chain(ctx.btc_client, &prev_graph.kickoff.tx().compute_txid()).await? { - verifier_force_skip_kickoff(ctx.btc_client, &prev_graph).await?; - } else if !prev_graph_status.is_closed() { - verifier_quick_challenge(ctx.btc_client, &prev_graph).await?; - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_prekickoff_sent_verifier( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by PreKickoff tx - let message = make_message(ctx, content); - let graph = match get_graph_for_refresh_or_defer(ctx, instance_id, graph_id, &message).await? { - Some(graph) => graph, - None => return Ok(()), - }; - - let (_, _, status_transition_accepted) = - refresh_and_compensate(ctx, instance_id, graph_id, &graph, GraphStatus::PreKickoff).await?; - if !status_transition_accepted { - // PreKickoffSent may arrive after KickoffSent or an event-watcher - // update. The state transition is then correctly rejected as stale, - // but the confirmed pre-kickoff still proves that the previous graph - // must be reconciled. - tracing::info!( - event = "prekickoff_reconciliation", - outcome = "current_status_stale", - "reconciling previous graph despite stale PreKickoff transition" - ); - } - - // Check the previous graph independently from whether this graph could - // still transition to PreKickoff. The helper revalidates the on-chain - // pre-kickoff transaction before it can broadcast any disprove action. - handle_previous_graph_after_prekickoff(ctx, instance_id, graph_id, &graph, &message).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_prekickoff_sent_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - // triggered by PreKickoff tx - let _graph = - refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::PreKickoff).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_challenge_sent_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - challenge_txid: Txid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Challenge tx - let message = make_message(ctx, content); - let (mut graph, _graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - // 1. check the challenge tx status on Bitcoin chain - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - if tx_on_chain(ctx.btc_client, &watchtower_challenge_init_txid).await? { - tracing::warn!( - "Ignore ChallengeSent for {instance_id}:{graph_id}: watchtower challenge init already sent" - ); - return Ok(()); - } - if let Some(challenge_tx) = ctx.btc_client.get_tx(&challenge_txid).await? { - let challenge_outpoint = graph - .kickoff - .connector_a_input() - .map_err(|e| anyhow!("failed to get connector-a input: {e}"))? - .outpoint; - if challenge_tx.input[0].previous_output != challenge_outpoint { - tracing::warn!( - "Ignore ChallengeSent for {instance_id}:{graph_id}: invalid challenge tx input" - ); - return Ok(()); - } - } else { - tracing::warn!( - "Ignore ChallengeSent for {instance_id}:{graph_id}: challenge tx not found on chain" - ); - return Ok(()); - } - // 2. if the challenge is confirmed, sign & broadcast watchtower-challenge-init txn - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let watchtower_challenge_init_tx = - operator_sign_watchtower_challenge_init(operator_master_key.master_keypair(), &mut graph)?; - let anchor_vout = watchtower_challenge_init_tx.output.len() as u64 - 1; - let watchtower_challenge_init_tx_total_input_amount = - graph.watchtower_challenge_init.prev_outs().iter().map(|o| o.value).sum(); - let child_tx = build_cpfp_txns( - ctx.btc_client, - &watchtower_challenge_init_tx, - anchor_vout, - watchtower_challenge_init_tx_total_input_amount, - ) - .await?; - match child_tx { - Some(tx) => { - broadcast_package(ctx.btc_client, &[watchtower_challenge_init_tx, tx], true).await? - } - None => broadcast_tx(ctx.btc_client, &watchtower_challenge_init_tx).await?, - }; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_challenge_sent_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - // triggered by Challenge tx - let _graph = - refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::Challenge).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_watchtower_challenge_init_sent_watchtower( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by WatchtowerChallengeInit tx - let message = make_message(ctx, content); - let (graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::Challenge { - tracing::warn!( - "Ignore WatchtowerChallengeInitSent for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - let watchtower_keypair = WatchtowerMasterKey::new(get_bitvm_key()?).master_keypair(); - let node_index = match graph - .parameters - .watchtower_pubkeys - .iter() - .position(|pk| *pk == watchtower_keypair.public_key().x_only_public_key().0) - { - Some(index) => index, - None => { - tracing::warn!( - "Ignore WatchtowerChallengeInitSent for {instance_id}:{graph_id}: node not a watchtower" - ); - return Ok(()); - } - }; - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - if !tx_on_chain(ctx.btc_client, &watchtower_challenge_init_txid).await? { - tracing::warn!( - "Ignore WatchtowerChallengeInitSent for {instance_id}:{graph_id}: watchtower challenge init not on chain" - ); - return Ok(()); - } - let watchtower_challenge_vout = - output_topology::watchtower_challenge_init::watchtower_connector(node_index) as u64; - if outpoint_spent_txid( - ctx.btc_client, - &watchtower_challenge_init_txid, - watchtower_challenge_vout, - ) - .await? - .is_some() - { - tracing::warn!( - "Ignore WatchtowerChallengeInitSent for {instance_id}:{graph_id}: watchtower challenge already spent" - ); - return Ok(()); - } - // watchtower should always challenge - let watchtower_proof = match get_watchtower_commitment( - ctx.local_db, - ctx.btc_client, - ctx.http_client, - instance_id, - graph_id, - ) - .await? - { - (Some(p), _) => p, - (None, wait_secs) => { - tracing::warn!( - "Retry WatchtowerChallengeInitSent for {instance_id}:{graph_id} later: watchtower proof not ready, retry after {wait_secs} seconds" - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - wait_secs, - MessageDeferReason::ProofPending, - "watchtower proof commitment is not ready", - ) - .await?; - return Ok(()); - } - }; - let watchtower_challenge_txid = match send_watchtower_challenge_tx( - ctx.btc_client, - &graph, - node_index, - watchtower_proof, - ) - .await - { - Ok(txid) => txid, - Err(e) => { - // Retry broadcast failures until the connector is spent. - tracing::warn!( - "Retry WatchtowerChallengeInitSent for {instance_id}:{graph_id} later: failed to send watchtower challenge tx: {e}" - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - 30, - MessageDeferReason::BitcoinTransactionPending, - "watchtower challenge transaction could not be broadcast", - ) - .await?; - return Ok(()); - } - }; - tracing::info!( - "WatchtowerChallengeSent for {instance_id}:{graph_id}: watchtower_index={node_index}, txid={watchtower_challenge_txid}" - ); - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id, watchtower_index = watchtower_index))] -async fn handle_watchtower_challenge_sent_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - watchtower_index: usize, - content: &GOATMessageContent, -) -> Result<()> { - let message = make_message(ctx, content); - let (graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::Challenge { - tracing::warn!( - "Ignore WatchtowerChallengeSent for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - let watchtower_num = validate_watchtower_branches(&graph)?; - if watchtower_index >= watchtower_num { - bail!("watchtower index {watchtower_index} out of range for {watchtower_num} slots"); - } - - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let watchtower_vout = - output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) as u64; - let Some((watchtower_spent_txid, watchtower_vin, _txin)) = - outpoint_spent_txin(ctx.btc_client, &watchtower_challenge_init_txid, watchtower_vout) - .await? - else { - tracing::warn!( - "Ignore WatchtowerChallengeSent for {instance_id}:{graph_id}: watchtower connector {watchtower_index} is not spent yet" - ); - return Ok(()); - }; - if watchtower_vin != 0 { - tracing::warn!( - "Ignore WatchtowerChallengeSent for {instance_id}:{graph_id}: watchtower connector {watchtower_index} was spent by tx {watchtower_spent_txid} at unexpected input index {watchtower_vin}" - ); - return Ok(()); - } - let timeout_txid = graph.watchtower_challenge_timeouts[watchtower_index].tx().compute_txid(); - if watchtower_spent_txid == timeout_txid { - tracing::warn!( - "Ignore WatchtowerChallengeSent for {instance_id}:{graph_id}: watchtower connector {watchtower_index} was spent by timeout tx {timeout_txid}" - ); - return Ok(()); - } - - let ack_vout = - output_topology::watchtower_challenge_init::ack_connector(watchtower_index) as u64; - if let Some(spent_txid) = - outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, ack_vout).await? - { - tracing::warn!( - "Ignore WatchtowerChallengeSent for {instance_id}:{graph_id}: ack connector {watchtower_index} already spent by {spent_txid}" - ); - return Ok(()); - } - - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let preimage = operator_master_key.preimage_for_graph(graph_id, watchtower_index); - let ack_connector_input = graph - .watchtower_challenge_init - .ack_connector_input(watchtower_index) - .map_err(|e| anyhow!("failed to get ack connector input: {e}"))?; - let ack_input = operator_sign_challenge_ack(&graph, watchtower_index, &preimage)?; - let ack_txid = build_sign_and_broadcast_tx( - ctx.btc_client, - operator_master_key.master_keypair(), - vec![ack_input], - ack_connector_input.amount, - vec![goat::scripts::p2a_output()], - ) - .await?; - tracing::info!( - "Operator challenge ACK sent for {instance_id}:{graph_id}: watchtower_index={watchtower_index}, txid={ack_txid}" - ); - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_watchtower_challenge_timeout_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - let message = make_message(ctx, content); - let (mut graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::Challenge { - tracing::warn!( - "Ignore WatchtowerChallengeTimeout for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - - let watchtower_num = validate_watchtower_branches(&graph)?; - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - for watchtower_index in 0..watchtower_num { - let watchtower_vout = - output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) - as u64; - let ack_vout = - output_topology::watchtower_challenge_init::ack_connector(watchtower_index) as u64; - if outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, watchtower_vout) - .await? - .is_some() - || outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, ack_vout) - .await? - .is_some() - { - continue; - } - - let timeout_tx = operator_sign_watchtower_challenge_timeout( - operator_master_key.master_keypair(), - &mut graph, - watchtower_index, - )?; - if tx_on_chain(ctx.btc_client, &timeout_tx.compute_txid()).await? { - continue; - } - let timeout_txid = timeout_tx.compute_txid(); - let timeout_tx_total_input_amount = graph.watchtower_challenge_timeouts[watchtower_index] - .prev_outs() - .iter() - .map(|o| o.value) - .sum::(); - broadcast_tx_with_cpfp(ctx.btc_client, timeout_tx, timeout_tx_total_input_amount).await?; - tracing::info!( - "Watchtower challenge timeout sent for {instance_id}:{graph_id}: watchtower_index={watchtower_index}, txid={timeout_txid}" - ); - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_nack_ready_verifier( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - let message = make_message(ctx, content); - let (graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::Challenge { - tracing::warn!( - "Ignore NackReady for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - let watchtower_num = validate_watchtower_branches(&graph)?; - - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let connector_f_vout = - output_topology::watchtower_challenge_init::connector_f(watchtower_num) as u64; - if outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, connector_f_vout) - .await? - .is_some() - { - tracing::warn!("Ignore NackReady for {instance_id}:{graph_id}: connector-F already spent"); - return Ok(()); - } - - for watchtower_index in 0..watchtower_num { - let watchtower_vout = - output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) - as u64; - let Some(watchtower_spent_txid) = - outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, watchtower_vout) - .await? - else { - continue; - }; - - let timeout_txid = - graph.watchtower_challenge_timeouts[watchtower_index].tx().compute_txid(); - if watchtower_spent_txid == timeout_txid { - continue; - } - - let ack_vout = - output_topology::watchtower_challenge_init::ack_connector(watchtower_index) as u64; - if outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, ack_vout) - .await? - .is_some() - { - continue; - } - - let nack = graph - .operator_challenge_nacks - .get(watchtower_index) - .ok_or_else(|| anyhow!("invalid watchtower index {watchtower_index}"))?; - let nack_tx = nack.tx().clone(); - let nack_txid = nack_tx.compute_txid(); - if tx_on_chain(ctx.btc_client, &nack_txid).await? { - return Ok(()); - } - let nack_tx_total_input_amount = nack.prev_outs().iter().map(|o| o.value).sum::(); - broadcast_tx_with_cpfp(ctx.btc_client, nack_tx, nack_tx_total_input_amount).await?; - tracing::info!( - "Operator challenge NACK sent for {instance_id}:{graph_id}: watchtower_index={watchtower_index}, txid={nack_txid}" - ); - return Ok(()); - } - - tracing::warn!( - "Ignore NackReady for {instance_id}:{graph_id}: no watchtower branch is ready for NACK" - ); - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_operator_commit_pubin_ready_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - let message = make_message(ctx, content); - let Some((graph, _graph_status, _graph_sub_status)) = - refresh_graph_status(ctx, instance_id, graph_id, Some(&message), GraphStatus::Challenge) - .await? - else { - return Ok(()); - }; - - // compute the 96-byte guest pubin - let watchtower_challenge_init_txid = - SerializableTxid::from(graph.watchtower_challenge_init.tx().compute_txid()); - let num_watchtowers = graph.parameters.watchtower_pubkeys.len(); - let watchtower_timeout_txids: Vec = - graph.watchtower_challenge_timeouts.iter().map(|tx| tx.tx().compute_txid()).collect(); - let wait_secs = avg_block_time_secs(ctx.btc_client.network()) as usize; - let WatchtowerChallengeInfo { - included_watchtowers: included_watchtowers_bits, - resolved_branch_txids, - .. - } = match get_watchtower_challenge_info( - ctx.btc_client, - &watchtower_challenge_init_txid, - &watchtower_timeout_txids, - num_watchtowers, - ) - .await - { - Ok(info) => info, - Err(e) => { - tracing::info!( - "Retry OperatorCommitPubinReady later for {instance_id}:{graph_id}: challenge info is not ready: {e}" - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - wait_secs, - MessageDeferReason::ProtocolInputsPending, - "watchtower challenge branches are not resolved yet", - ) - .await?; - return Ok(()); - } - }; - let (btc_best_block_hash, included_watchtowers_bitmap) = - match compute_operator_pubin_blockhash_and_bitmap( - ctx.btc_client, - &resolved_branch_txids, - &included_watchtowers_bits, - ) - .await - { - Ok(inputs) => inputs, - Err(e) => { - tracing::info!( - "Retry OperatorCommitPubinReady later for {instance_id}:{graph_id}: operator pubin inputs are not ready: {e}" - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - wait_secs, - MessageDeferReason::ProtocolInputsPending, - &e.to_string(), - ) - .await?; - return Ok(()); - } - }; - let guest_pubin = build_operator_guest_pubin( - &btc_best_block_hash, - &graph.parameters.pubin_disprove_constant, - &included_watchtowers_bitmap, - ); - - // sign and broadcast - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let commit_pubin_wots_sk = operator_master_key.commit_pubin_wots_keypair_for_graph(graph_id).0; - let signed_input = operator_sign_commit_pubin(&graph, &commit_pubin_wots_sk, &guest_pubin)?; - let connector_e_amount = graph - .watchtower_challenge_init - .connector_e_input() - .map_err(|e| anyhow!("failed to get connector-e input: {e}"))? - .amount; - let operator_keypair = operator_master_key.master_keypair(); - build_sign_and_broadcast_tx( - ctx.btc_client, - operator_keypair, - vec![signed_input], - connector_e_amount, - vec![], - ) - .await?; - - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_operator_commit_pubin_timeout_verifier( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - let message = make_message(ctx, content); - let (graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::Challenge { - tracing::warn!( - "Ignore OperatorCommitPubinTimeout for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let watchtower_num = graph.parameters.watchtower_pubkeys.len(); - let connector_e_vout = - output_topology::watchtower_challenge_init::connector_e(watchtower_num) as u64; - let connector_f_vout = - output_topology::watchtower_challenge_init::connector_f(watchtower_num) as u64; - if outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, connector_e_vout) - .await? - .is_some() - || outpoint_spent_txid(ctx.btc_client, &watchtower_challenge_init_txid, connector_f_vout) - .await? - .is_some() - { - tracing::warn!( - "Ignore OperatorCommitPubinTimeout for {instance_id}:{graph_id}: timeout input already spent" - ); - return Ok(()); - } - - let timeout_tx = graph.operator_commit_timeout.tx().clone(); - let timeout_txid = timeout_tx.compute_txid(); - if tx_on_chain(ctx.btc_client, &timeout_txid).await? { - return Ok(()); - } - let timeout_tx_total_input_amount = - graph.operator_commit_timeout.prev_outs().iter().map(|o| o.value).sum::(); - broadcast_tx_with_cpfp(ctx.btc_client, timeout_tx, timeout_tx_total_input_amount).await?; - tracing::info!( - "Operator commit pubin timeout sent for {instance_id}:{graph_id}: txid={timeout_txid}" - ); - Ok(()) -} - -// after the watchtower challenge flow is complete, build proof and broadcast Assert transaction. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_assert_ready_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - let message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::AssertReady(AssertReady { instance_id, graph_id }), - ); - let (mut graph, _graph_status, _graph_sub_status) = - match refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::Challenge).await? - { - Some(v) => v, - None => return Ok(()), - }; - let (operator_proof, wait_secs) = get_operator_proof( - ctx.local_db, - ctx.http_client, - &graph, - ctx.btc_client, - instance_id, - graph_id, - ) - .await?; - - if wait_secs > 0 { - tracing::info!( - "Retry AssertReady later for {instance_id}:{graph_id}: operator proof is not ready" - ); - defer_or_enqueue_message( - ctx.local_db, - &message, - wait_secs, - MessageDeferReason::ProofPending, - "operator proof is not ready", - ) - .await?; - return Ok(()); - } - - let Some(operator_proof) = operator_proof else { - return Ok(()); - }; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let assert_secret_key = operator_master_key.assert_wots_keypair_for_graph(graph_id).0; - - let dynamic_input = operator_assert_dynamic_input(&operator_proof.public_inputs)?; - let assert_witness = - build_assert_witness(&operator_proof.proof, &assert_secret_key, dynamic_input)?; - let assert_message = assert_wots_message(&assert_witness)?; - let mut asserted_operator_proof = Vec::new(); - operator_proof.proof.serialize_compressed(&mut asserted_operator_proof)?; - let mut setup_state = load_babe_setup_state(ctx.local_db, instance_id, graph_id)? - .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; - let operator_state = setup_state - .operator - .as_mut() - .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; - if let Some(existing) = &operator_state.asserted_operator_proof - && existing != &asserted_operator_proof - { - bail!("operator assertion proof conflicts with persisted proof"); - } - operator_state.asserted_operator_proof = Some(asserted_operator_proof); - save_babe_setup_state(ctx.local_db, instance_id, graph_id, &setup_state)?; - - let assert_tx = operator_sign_assert( - &mut graph, - &assert_secret_key, - &assert_message, - &assert_witness.pi2, - &assert_witness.pi3, - )?; - let assert_tx_total_input_amount = - graph.operator_assert.prev_outs().iter().map(|o| o.value).sum::(); - broadcast_tx_with_cpfp(ctx.btc_client, assert_tx, assert_tx_total_input_amount).await?; - - Ok(()) -} - -/// Recovers a `BabeChallengeAssertWitness` from the on-chain challenge assert tx. -fn recover_challenge_assert_witness( - challenge_assert_tx: &bitcoin::Transaction, - verifier_index: usize, -) -> Result { - let txin = challenge_assert_tx - .input - .first() - .ok_or_else(|| anyhow!("challenge assert tx has no input"))?; - let items: Vec> = txin.witness.to_vec(); - let expected_len = 2 * WOTS_SIG_COUNT + goat::assert_scripts::INPUT_WIRE_NUM + 2; - if items.len() != expected_len { - bail!( - "unexpected challenge assert witness length: {} (expected {expected_len})", - items.len() - ); - } - let mut bitcoin_witness = bitcoin::Witness::new(); - for item in &items[..2 * WOTS_SIG_COUNT] { - bitcoin_witness.push(item); - } - let wots_sig = Wots96::raw_witness_to_signature(&bitcoin_witness).to_vec(); - let input_labels: Vec<[u8; 16]> = items - [2 * WOTS_SIG_COUNT..2 * WOTS_SIG_COUNT + goat::assert_scripts::INPUT_WIRE_NUM] - .iter() - .map(|item| item.as_slice().try_into()) - .collect::>() - .map_err(|_| anyhow!("challenge assert label item is not 16 bytes"))?; - Ok(BabeChallengeAssertWitness { - verifier_index, - witness: ChallengeAssertWitnessRaw { input_labels, wots_sig }, - }) -} - -/// Collects the on-chain ACK TxIns from all watchtowers that submitted a challenge_ack. -/// Unspent ACK connectors are skipped; spent connectors must resolve to the spending input. -async fn collect_ack_txins( - btc_client: &BTCClient, - wci_txid: &Txid, - watchtower_timeout_txids: &[Txid], -) -> Result> { - let mut ack_txins = Vec::new(); - for (i, timeout_txid) in watchtower_timeout_txids.iter().enumerate() { - let ack_vout = output_topology::watchtower_challenge_init::ack_connector(i) as u64; - let Some(ack_spent_txid) = outpoint_spent_txid(btc_client, wci_txid, ack_vout).await? - else { - continue; - }; - if &ack_spent_txid == timeout_txid { - continue; - } - let Some((_spent_txid, _vin, txin)) = - outpoint_spent_txin(btc_client, wci_txid, ack_vout).await? - else { - bail!("ACK spending tx {ack_spent_txid} for watchtower {i} is unavailable"); - }; - ack_txins.push(txin); - } - Ok(ack_txins) -} - -// verify Operator DynamicPublicInput and Proof; broadcast PubinDisprove or ChallengeAssert as needed. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_assert_sent_verifier( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - assert_txid: Txid, - content: &GOATMessageContent, -) -> Result<()> { - let (graph, _graph_status, _graph_sub_status) = - match refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::Challenge).await? - { - Some(v) => v, - None => return Ok(()), - }; - - let Some(assert_tx) = ctx.btc_client.get_tx(&assert_txid).await? else { - tracing::warn!( - "Ignore AssertSent for {instance_id}:{graph_id}: assert tx {assert_txid} not found on chain" - ); - return Ok(()); - }; - - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let verifier_pubkey = verifier_master_key.master_keypair().public_key().into(); - let Some(verifier_index) = - find_verifier_index_by_pubkey(&graph.parameters.gc_data, &verifier_pubkey)? - else { - tracing::debug!( - "Ignore AssertSent for {instance_id}:{graph_id}: local verifier has no graph slot" - ); - return Ok(()); - }; - validate_verifier_slot(&graph, verifier_index)?; - - // ChallengeAssert may only be skipped after this graph's commit-pubin is - // on-chain and proves consistent with the assert witness. OperatorAssert - // can otherwise be broadcast before connector-E is spent. - let connector_e_input = graph - .watchtower_challenge_init - .connector_e_input() - .map_err(|error| anyhow!("failed to resolve connector-E input: {error}"))?; - let connector_e_outpoint = connector_e_input.outpoint; - let Some(connector_e_spent_txid) = outpoint_spent_txid( - ctx.btc_client, - &connector_e_outpoint.txid, - connector_e_outpoint.vout as u64, - ) - .await? - else { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::ProtocolInputsPending, - "operator commit-pubin is not on chain yet", - ) - .await?; - tracing::info!( - "Retry AssertSent later for {instance_id}:{graph_id}: operator commit-pubin is not on chain yet" - ); - return Ok(()); - }; - - let mut pubin_consistent = false; - if connector_e_spent_txid != graph.operator_commit_timeout.tx().compute_txid() { - if !ctx.btc_client.get_tx_status(&connector_e_spent_txid).await?.confirmed { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::ProtocolInputsPending, - "operator commit-pubin transaction is not confirmed yet", - ) - .await?; - return Ok(()); - } - let Some(commit_pubin_tx) = ctx.btc_client.get_tx(&connector_e_spent_txid).await? else { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::ProtocolInputsPending, - "operator commit-pubin transaction is not available yet", - ) - .await?; - return Ok(()); - }; - let assert_txin = assert_tx - .input - .first() - .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; - let commit_pubin_txin = commit_pubin_tx - .input - .first() - .ok_or_else(|| anyhow!("operator commit-pubin transaction has no input"))?; - let wci_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let watchtower_timeout_txids = graph - .watchtower_challenge_timeouts - .iter() - .map(|tx| tx.tx().compute_txid()) - .collect::>(); - let ack_txins = - match collect_ack_txins(ctx.btc_client, &wci_txid, &watchtower_timeout_txids).await { - Ok(txins) => txins, - Err(error) => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::ProtocolInputsPending, - &format!("operator ACK inputs are not ready: {error}"), - ) - .await?; - return Ok(()); - } - }; - match validate_pubin_disprove(&graph, commit_pubin_txin, assert_txin, &ack_txins) { - Ok(Some((witness_data, _))) => { - // PubinDisprove is not pre-signed, so fund it directly rather - // than relying on CPFP. - let pubin_disprove_tx_total_input_amount = graph - .operator_assert - .connector_d_input() - .map_err(|e| anyhow!("failed to get connector-d input: {e}"))? - .amount; - let pubin_disprove_txin = build_pubin_disprove_txin(&graph, witness_data)?; - let pubin_disprove_tx = bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![pubin_disprove_txin], - output: vec![ - goat::scripts::p2a_output(), - // Keep the transaction above Bitcoin Core's minimum - // non-witness relay size. - bitcoin::TxOut { - value: Amount::ZERO, - script_pubkey: goat::scripts::generate_opreturn_script( - PUBIN_DISPROVE_OP_RETURN_DATA.to_vec(), - ), - }, - ], - }; - let verifier_keypair = VerifierMasterKey::new(get_bitvm_key()?).master_keypair(); - build_sign_and_broadcast_tx( - ctx.btc_client, - verifier_keypair, - pubin_disprove_tx.input, - pubin_disprove_tx_total_input_amount, - pubin_disprove_tx.output, - ) - .await?; - return Ok(()); - } - Ok(None) => { - pubin_consistent = true; - } - Err(error) => tracing::warn!( - "PubinDisprove validation failed for {instance_id}:{graph_id}: {error}; proceeding to ChallengeAssert" - ), - } - } - - // recover TxAssertWitness from the assert tx witness. - let assert_txin = assert_tx - .input - .first() - .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; - let assert_witness = extract_operator_assert_witness_for_challenge(&graph, assert_txin) - .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; - let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for operator assert")?; - let static_input = derive_operator_static_input()?; - if pubin_consistent && assert_witness.verify_groth16_proof(&vk, &static_input) { - tracing::info!( - "Skip ChallengeAssert for {instance_id}:{graph_id}: operator assert proof is valid" - ); - return Ok(()); - } - - let strict = false; - broadcast_verifier_challenge_assert_tx( - ctx.local_db, - ctx.btc_client, - &graph, - instance_id, - graph_id, - &assert_tx, - strict, - ) - .await?; - - Ok(()) -} - -const PUBIN_DISPROVE_OP_RETURN_DATA: &[u8] = b"pubin-disprove"; - -pub async fn broadcast_verifier_challenge_assert_tx( - local_db: &LocalDB, - btc_client: &BTCClient, - graph: &BitvmGcGraph, - instance_id: Uuid, - graph_id: Uuid, - assert_tx: &bitcoin::Transaction, - strict: bool, -) -> Result> { - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let verifier_pubkey = verifier_master_key.master_keypair().public_key().into(); - let Some(verifier_index) = - find_verifier_index_by_pubkey(&graph.parameters.gc_data, &verifier_pubkey)? - else { - if strict { - bail!("local verifier has no graph slot for {instance_id}:{graph_id}"); - } - tracing::debug!( - "Ignore AssertSent for {instance_id}:{graph_id}: local verifier has no graph slot" - ); - return Ok(None); - }; - validate_verifier_slot(graph, verifier_index)?; - - let assert_txin = assert_tx - .input - .first() - .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; - let assert_witness = extract_operator_assert_witness_for_challenge(graph, assert_txin) - .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; - let vk = crate::vk::get_vk().await.context("load Groth16 verifying key for operator assert")?; - let static_input = derive_operator_static_input()?; - - let Some(saved_verifier_state) = - load_babe_setup_state(local_db, instance_id, graph_id)?.and_then(|state| state.verifier) - else { - if strict { - bail!("missing BABE verifier setup state for {instance_id}:{graph_id}"); - } - tracing::warn!( - "Ignore AssertSent for {instance_id}:{graph_id}: missing BABE verifier setup state" - ); - return Ok(None); - }; - if saved_verifier_state.soldering_proof_ready.is_none() { - if strict { - bail!("missing soldering proof reference for {instance_id}:{graph_id}"); - } - tracing::warn!( - "Ignore AssertSent for {instance_id}:{graph_id}: missing soldering proof reference" - ); - return Ok(None); - } - let challenge_witness = build_real_challenge_assert_witness( - &saved_verifier_state.private_state, - &saved_verifier_state.setup_package, - &saved_verifier_state.finalized_indices, - &vk, - static_input, - &assert_witness, - verifier_index, - )?; - let labels: [Vec; goat::assert_scripts::INPUT_WIRE_NUM] = challenge_witness - .witness - .input_labels - .iter() - .map(|label| label.to_vec()) - .collect::>() - .try_into() - .map_err(|labels: Vec>| { - anyhow!( - "BABE challenge witness exposes {} labels; expected {}", - labels.len(), - goat::assert_scripts::INPUT_WIRE_NUM - ) - })?; - let operator_assert_txin = assert_tx - .input - .first() - .cloned() - .ok_or_else(|| anyhow!("operator assert transaction has no input"))?; - let challenge_assert_tx = - build_verifier_assert_tx(graph, operator_assert_txin, verifier_index, labels)?; - let challenge_assert_txid = challenge_assert_tx.compute_txid(); - if btc_client.get_tx(&challenge_assert_txid).await?.is_some() { - tracing::info!( - "Verifier ChallengeAssert already exists for {instance_id}:{graph_id}: verifier_index={verifier_index}, txid={challenge_assert_txid}" - ); - return Ok(Some((challenge_assert_txid, verifier_index))); - } - let challenge_assert_tx_total_input_amount = - graph.verifier_asserts[verifier_index].prev_outs().iter().map(|o| o.value).sum::(); - broadcast_tx_with_cpfp(btc_client, challenge_assert_tx, challenge_assert_tx_total_input_amount) - .await?; - - Ok(Some((challenge_assert_txid, verifier_index))) -} - -// compute msg after ChallengeAssert is broadcast and broadcast WronglyChallenge transaction. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_challenge_assert_sent_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - challenge_assert_txid: Txid, - verifier_index: usize, - content: &GOATMessageContent, -) -> Result<()> { - let (graph, _graph_status, _graph_sub_status) = - match refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::Challenge).await? - { - Some(v) => v, - None => return Ok(()), - }; - validate_verifier_slot(&graph, verifier_index)?; - validate_expected_challenge_assert_txid(&graph, verifier_index, challenge_assert_txid)?; - - let Some(challenge_assert_tx) = ctx.btc_client.get_tx(&challenge_assert_txid).await? else { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinTransactionPending, - "ChallengeAssert transaction is not available from the Bitcoin backend", - ) - .await?; - tracing::info!( - "Retry ChallengeAssertSent later for {instance_id}:{graph_id}: challenge assert tx {challenge_assert_txid} not found on chain" - ); - return Ok(()); - }; - - let challenge_witness = recover_challenge_assert_witness(&challenge_assert_tx, verifier_index)?; - validate_challenge_witness_index(verifier_index, &challenge_witness)?; - - let labels: [Vec; goat::assert_scripts::INPUT_WIRE_NUM] = challenge_witness - .witness - .input_labels - .iter() - .map(|label| label.to_vec()) - .collect::>() - .try_into() - .map_err(|labels: Vec>| { - anyhow!( - "BABE challenge witness exposes {} labels; expected {}", - labels.len(), - goat::assert_scripts::INPUT_WIRE_NUM - ) - })?; - let operator_assert_txid = challenge_assert_tx - .input - .first() - .ok_or_else(|| anyhow!("published verifier assert transaction has no input"))? - .previous_output - .txid; - let operator_assert_tx = - ctx.btc_client.get_tx(&operator_assert_txid).await?.ok_or_else(|| { - anyhow!("published operator assert transaction {operator_assert_txid} is unavailable") - })?; - let operator_assert_txin = operator_assert_tx - .input - .first() - .cloned() - .ok_or_else(|| anyhow!("published operator assert transaction has no input"))?; - let operator_assert_witness = extract_operator_assert_witness(&graph, &operator_assert_txin) - .map_err(|e| anyhow!("failed to extract operator assert witness: {e}"))?; - let expected_challenge_assert = - build_verifier_assert_tx(&graph, operator_assert_txin.clone(), verifier_index, labels)?; - if expected_challenge_assert.input.first().map(|input| &input.witness) - != challenge_assert_tx.input.first().map(|input| &input.witness) - { - bail!("published verifier assert witness does not match BABE challenge labels"); - } - - let setup_state = load_babe_setup_state(ctx.local_db, instance_id, graph_id)? - .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; - let operator_state = setup_state - .operator - .ok_or_else(|| anyhow!("missing operator BABE setup state for graph {graph_id}"))?; - let candidate = selected_candidate_for_graph_index(&operator_state, verifier_index)?; - let soldering_proof_ready = candidate.soldering_proof_ready.as_ref().ok_or_else(|| { - anyhow!("missing soldering proof reference for verifier slot {verifier_index}") - })?; - let store_base_path = get_soldering_proof_payload_store_path()?; - let payload_path = soldering_proof_payload_store_path( - &store_base_path, - instance_id, - graph_id, - soldering_proof_ready.candidate_index, - &soldering_proof_ready.payload_hash, - )?; - let payload = read_soldering_proof_store_payload(&payload_path) - .await - .with_context(|| format!("read soldering proof payload from {payload_path}"))?; - let payload = decode_soldering_proof_payload(soldering_proof_ready, &payload)?; - let (_, finalized, soldering) = expand_compact_soldering_proof_payload(payload) - .context("expand compact soldering proof payload for challenge")?; - let prover_state = build_babe_prover_state(&candidate.setup_package, finalized, soldering)?; - let assert_witness = TxAssertWitness { - wots_sig: challenge_witness.witness.wots_sig.clone(), - pi2: operator_assert_witness.pi2, - pi3: operator_assert_witness.pi3, - }; - let proof = recover_operator_proof_from_assert_witness(&assert_witness) - .context("recover asserted operator proof from on-chain assert witness")?; - let vk = crate::vk::get_vk() - .await - .context("load Groth16 verifying key for BABE wrongly challenged")?; - let (_, dyn_pubin) = assert_witness.try_recover_pi1_xd().ok_or_else(|| { - anyhow!("cannot recover dynamic input from challenge witness WOTS signature") - })?; - let wrongly_challenged_witness = recover_real_wrongly_challenged_witness( - &prover_state, - &challenge_witness, - &proof, - vk, - dyn_pubin, - )?; - let (wrongly_challenged_input, input_amount) = operator_sign_wrongly_challenged( - &graph, - verifier_index, - &wrongly_challenged_witness.final_msg, - )?; - - let operator_keypair = OperatorMasterKey::new(get_bitvm_key()?).master_keypair(); - build_sign_and_broadcast_tx( - ctx.btc_client, - operator_keypair, - vec![wrongly_challenged_input], - input_amount, - vec![ - goat::scripts::p2a_output(), - // This makes the non-witness transaction size safely exceed the - // relay minimum even before the locally funded fee input is added. - bitcoin::TxOut { - value: Amount::ZERO, - script_pubkey: goat::scripts::generate_opreturn_script( - WRONGLY_CHALLENGED_OP_RETURN_DATA.to_vec(), - ), - }, - ], - ) - .await?; - - Ok(()) -} - -const WRONGLY_CHALLENGED_OP_RETURN_DATA: &[u8] = b"wrongly-challenged"; - -// broadcast NoWithdraw after the ChallengeAssert timelock expires. -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_wrongly_challenge_timeout_verifier( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - challenge_assert_txid: Txid, - verifier_index: usize, - content: &GOATMessageContent, -) -> Result<()> { - let (graph, _graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - None, - GraphStatus::Disprove, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - validate_verifier_slot(&graph, verifier_index)?; - validate_expected_challenge_assert_txid(&graph, verifier_index, challenge_assert_txid)?; - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let local_verifier_pubkey: PublicKey = verifier_master_key.master_keypair().public_key().into(); - if graph.parameters.gc_data[verifier_index].verifier_pubkey != local_verifier_pubkey { - tracing::debug!( - "Ignore WronglyChallengeTimeout for {instance_id}:{graph_id}: local verifier does not own slot {verifier_index}" - ); - return Ok(()); - } - - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - let message = make_message(ctx, content); - if ctx.btc_client.get_tx(&challenge_assert_txid).await?.is_none() { - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinTransactionPending, - "ChallengeAssert transaction is not available from the Bitcoin backend", - ) - .await?; - tracing::info!( - "Retry WronglyChallengeTimeout later for {instance_id}:{graph_id}: challenge assert tx {challenge_assert_txid} not found on chain" - ); - return Ok(()); - } - - let challenge_assert_height = match ctx - .btc_client - .get_tx_status(&challenge_assert_txid) - .await? - .block_height - { - Some(height) => height as u64, - None => { - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "ChallengeAssert transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry WronglyChallengeTimeout later for {instance_id}:{graph_id}: challenge assert tx {challenge_assert_txid} is not confirmed" - ); - return Ok(()); - } - }; - - let disprove_height = challenge_assert_height - + disprove_timelock_with_config( - graph.parameters.instance_parameters.network, - &graph.parameters.timelock_config, - ) as u64; - let bitcoin_height = ctx.btc_client.get_height().await? as u64; - if bitcoin_height < disprove_height { - let retry_secs = - avg_block_time_secs(ctx.btc_client.network()) * (disprove_height - bitcoin_height); - defer_or_enqueue_message( - ctx.local_db, - &message, - retry_secs as usize, - MessageDeferReason::TimelockPending, - "disprove timelock has not expired", - ) - .await?; - tracing::info!( - "Retry WronglyChallengeTimeout later for {instance_id}:{graph_id}: disprove timelock has not expired" - ); - return Ok(()); - } - if let Some(spent_txid) = outpoint_spent_txid(ctx.btc_client, &challenge_assert_txid, 0).await? - { - tracing::info!( - "Skip Disprove for {instance_id}:{graph_id} slot {verifier_index}: verifier assertion output already spent by {spent_txid}" - ); - return Ok(()); - } - - let disprove_tx = build_disprove_tx(&graph, verifier_index, None)?; - let disprove_tx_total_input_amount = - graph.disproves[verifier_index].prev_outs().iter().map(|o| o.value).sum::(); - broadcast_tx_with_cpfp(ctx.btc_client, disprove_tx, disprove_tx_total_input_amount).await?; - - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id -))] -async fn handle_disprove_sent_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - disprove_type: DisproveTxType, - index: usize, - challenge_finish_txid: Txid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Disprove tx - // 1. update graph status - let message = make_message(ctx, content); - let (graph, _graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - None, - GraphStatus::Disprove, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if !is_relayer() { - return Ok(()); - } - // 2. (Relayer) call finalizeWithdrawDisprove on GoatChain - let withdraw_status = ctx.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; - if withdraw_status == WithdrawStatus::Disproved { - tracing::warn!( - "Relayer Ignore finishWithdrawDisproved for {instance_id}:{graph_id}: already posted" - ); - return Ok(()); - } - let kickoff_txid = graph.kickoff.tx().compute_txid(); - let take1_txid = graph.take1.tx().compute_txid(); - let connector_a_vout = output_topology::kickoff::connector_a() as u64; - let challenge_start_tx = if let Some(spent_txid) = - outpoint_spent_txid(ctx.btc_client, &kickoff_txid, connector_a_vout).await? - { - if spent_txid == take1_txid { - tracing::warn!( - "Ignore DisproveSent for {instance_id}:{graph_id}: graph already finalized by Take1 tx: {spent_txid}" - ); - return Ok(()); - } - ctx.btc_client.get_tx(&spent_txid).await? - } else { - None - }; - let challenge_finish_tx = match ctx.btc_client.get_tx(&challenge_finish_txid).await? { - Some(tx) => tx, - None => { - tracing::warn!( - "Ignore DisproveSent for {instance_id}:{graph_id}: challenge finish tx {challenge_finish_txid} not found on chain" - ); - return Ok(()); - } - }; - let challenge_finish_height = match ctx - .btc_client - .get_tx_status(&challenge_finish_txid) - .await? - .block_height - { - Some(height) => height as u64, - None => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "challenge finish transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry finishWithdrawDisproved later for {instance_id}:{graph_id}: challenge finish tx not confirmed on btc yet" - ); - return Ok(()); - } - }; - let goat_confirmed_height = ctx.goat_client.btc_spv_latest_height().await?; - if goat_confirmed_height < challenge_finish_height { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()) - * (challenge_finish_height - goat_confirmed_height); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::GoatSpvPending, - "challenge finish block is not available through GOAT SPV", - ) - .await?; - tracing::info!( - "Retry finishWithdrawDisproved later for {instance_id}:{graph_id}: challenge finish tx block not posted to goat spv contract yet" - ); - return Ok(()); - } - let result = ctx - .goat_client - .gateway_finish_withdraw_disproved( - ctx.btc_client, - &graph_id, - disprove_type, - index as u64, - challenge_start_tx.as_ref(), - &challenge_finish_tx, - ) - .await; - ctx.metrics_state.record_withdraw_finalize(result.is_ok()); - result?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_disprove_sent_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - // triggered by Disprove tx - let _graph = - refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::Disprove).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_take1_ready_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by timeout task - let message = make_message(ctx, content); - let (mut graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::OperatorKickOff, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::OperatorKickOff { - tracing::warn!( - "Ignore Take1Ready for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - let kickoff_txid = graph.kickoff.tx().compute_txid(); - let connector_a_vout = output_topology::kickoff::connector_a() as u64; - let guardian_connector_vout = output_topology::kickoff::guardian_connector() as u64; - if outpoint_spent_txid(ctx.btc_client, &kickoff_txid, connector_a_vout).await?.is_some() - || outpoint_spent_txid(ctx.btc_client, &kickoff_txid, guardian_connector_vout) - .await? - .is_some() - { - tracing::warn!("Ignore Take1Ready for {instance_id}:{graph_id}: connectors already spent"); - return Ok(()); - } - let kickoff_height = match ctx.btc_client.get_tx_status(&kickoff_txid).await?.block_height { - Some(height) => height, - None => { - tracing::warn!( - "Ignore Take1Ready for {instance_id}:{graph_id}: kickoff tx not confirmed yet" - ); - return Ok(()); - } - }; - if !is_take1_timelock_expired(ctx.btc_client, kickoff_height, &graph.parameters.timelock_config) - .await? - { - tracing::warn!( - "Ignore Take1Ready for {instance_id}:{graph_id}: kickoff tx timelock not expired yet" - ); - return Ok(()); - } - // 1. sign & broadcast take1 txn - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let operator_graph_keypair = operator_master_key.master_keypair(); - let take1_tx = operator_sign_take1(operator_graph_keypair, &mut graph)?; - let anchor_vout = take1_tx.output.len() as u64 - 1; - let take1_tx_total_input_amount = graph.take1.prev_outs().iter().map(|o| o.value).sum(); - let child_tx = - build_cpfp_txns(ctx.btc_client, &take1_tx, anchor_vout, take1_tx_total_input_amount) - .await?; - match child_tx { - Some(tx) => broadcast_package(ctx.btc_client, &[take1_tx, tx], true).await?, - None => broadcast_tx(ctx.btc_client, &take1_tx).await?, - }; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_take1_sent_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Take1 tx - // 1. update graph status - let message = make_message(ctx, content); - let (graph, _graph_status, _graph_sub_status) = - match refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::OperatorTake1) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if !is_relayer() { - return Ok(()); - } - // 2. (Relayer) call finalizeWithdrawHappyPath on GoatChain - let take1_txid = graph.take1.tx().compute_txid(); - let take1_tx = match ctx.btc_client.get_tx(&take1_txid).await? { - Some(tx) => tx, - None => { - tracing::warn!( - "Ignore Take1Sent for {instance_id}:{graph_id}: take1 tx not found on chain" - ); - return Ok(()); - } - }; - let withdraw_status = ctx.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; - if withdraw_status == WithdrawStatus::Initialized { - // Kickoff not posted yet, wait for it - let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * 6; // wait for 6 blocks - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::WithdrawKickoffPending, - "withdraw is initialized but kickoff has not been posted", - ) - .await?; - tracing::info!( - "Retry finishWithdrawHappyPath later for {instance_id}:{graph_id} as kickoff not posted yet" - ); - return Ok(()); - } - if withdraw_status != WithdrawStatus::Processing { - tracing::warn!( - "Relayer Ignore finishWithdrawHappyPath for {instance_id}:{graph_id}: invalid withdraw status: {withdraw_status}" - ); - return Ok(()); - } - let take1_height = match ctx.btc_client.get_tx_status(&take1_txid).await?.block_height { - Some(height) => height as u64, - None => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 block - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "take1 transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry finishWithdrawHappyPath later for {instance_id}:{graph_id} as take1 tx not confirmed on btc yet" - ); - return Ok(()); - } - }; - let goat_confirmed_height = ctx.goat_client.btc_spv_latest_height().await?; - if goat_confirmed_height < take1_height { - let delay_secs = - avg_block_time_secs(ctx.btc_client.network()) * (take1_height - goat_confirmed_height); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::GoatSpvPending, - "take1 block is not available through GOAT SPV", - ) - .await?; - tracing::info!( - "Retry finishWithdrawHappyPath later for {instance_id}:{graph_id} as take1 tx block not posted to goat spv contract yet" - ); - return Ok(()); - } - ctx.goat_client - .gateway_finish_withdraw_happy_path(ctx.btc_client, &graph_id, &take1_tx) - .await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_take1_sent_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - // triggered by Take1 tx - // 1. update graph status - let _graph = - refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::OperatorTake1).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_take2_ready_operator( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by timeout task - let message = make_message(ctx, content); - let (mut graph, graph_status, _graph_sub_status) = match refresh_graph_status( - ctx, - instance_id, - graph_id, - Some(&message), - GraphStatus::Challenge, - ) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if graph_status != GraphStatus::Challenge { - tracing::warn!( - "Ignore Take2Ready for {instance_id}:{graph_id}: graph status is {graph_status:?}" - ); - return Ok(()); - } - let operator_assert_txid = graph.operator_assert.tx().compute_txid(); - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - // Take2 spends Connector-0, Connector-D, Connector-F, and Guardian Connector. Recheck every input before - // signing so a stale Take2Ready cannot race an already-spent connector. - let take2_inputs = - graph.take2.tx().input.iter().map(|txin| txin.previous_output).collect::>(); - for previous_output in take2_inputs { - if let Some(spent_txid) = - outpoint_spent_txid(ctx.btc_client, &previous_output.txid, previous_output.vout as u64) - .await? - { - tracing::warn!( - "Ignore Take2Ready for {instance_id}:{graph_id}: take2 input {}:{} already spent by {}", - previous_output.txid, - previous_output.vout, - spent_txid - ); - return Ok(()); - } - } - let operator_assert_height = match ctx - .btc_client - .get_tx_status(&operator_assert_txid) - .await? - .block_height - { - Some(height) => height, - None => { - tracing::warn!( - "Ignore Take2Ready for {instance_id}:{graph_id}: assert init tx not confirmed yet" - ); - return Ok(()); - } - }; - let watchtower_challenge_init_height = match ctx - .btc_client - .get_tx_status(&watchtower_challenge_init_txid) - .await? - .block_height - { - Some(height) => height, - None => { - tracing::warn!( - "Ignore Take2Ready for {instance_id}:{graph_id}: watchtower challenge init tx not confirmed yet" - ); - return Ok(()); - } - }; - if !is_take2_timelock_expired( - ctx.btc_client, - operator_assert_height, - watchtower_challenge_init_height, - &graph.parameters.timelock_config, - ) - .await? - { - tracing::warn!( - "Ignore Take2Ready for {instance_id}:{graph_id}: take2 timelock not expired yet" - ); - return Ok(()); - } - // 1. sign & broadcast take2 txn - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let operator_graph_keypair = operator_master_key.master_keypair(); - let take2_tx = operator_sign_take2(operator_graph_keypair, &mut graph)?; - let anchor_vout = take2_tx.output.len() as u64 - 1; - let take2_tx_total_input_amount = graph.take2.prev_outs().iter().map(|o| o.value).sum(); - let child_tx = - build_cpfp_txns(ctx.btc_client, &take2_tx, anchor_vout, take2_tx_total_input_amount) - .await?; - match child_tx { - Some(tx) => broadcast_package(ctx.btc_client, &[take2_tx, tx], true).await?, - None => broadcast_tx(ctx.btc_client, &take2_tx).await?, - }; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_take2_sent_committee( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - content: &GOATMessageContent, -) -> Result<()> { - // triggered by Take2 tx - // 1. update graph status - let message = make_message(ctx, content); - let (graph, _graph_status, _graph_sub_status) = - match refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::OperatorTake2) - .await? - { - Some(v) => v, - None => return Ok(()), - }; - if !is_relayer() { - return Ok(()); - } - // 2. (Relayer) call finalizeWithdrawUnhappyPath on GoatChain - let take2_txid = graph.take2.tx().compute_txid(); - let take2_tx = match ctx.btc_client.get_tx(&take2_txid).await? { - Some(tx) => tx, - None => { - tracing::warn!( - "Ignore Take2Sent for {instance_id}:{graph_id}: take2 tx not found on chain" - ); - return Ok(()); - } - }; - let withdraw_status = ctx.goat_client.gateway_get_withdraw_data(&graph_id).await?.status; - if withdraw_status == WithdrawStatus::Initialized { - // Kickoff not posted yet, wait for it - let delay_secs = avg_block_time_secs(ctx.btc_client.network()) * 6; // wait for 6 blocks - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::WithdrawKickoffPending, - "withdraw is initialized but kickoff has not been posted", - ) - .await?; - tracing::info!( - "Retry finishWithdrawUnhappyPath later for {instance_id}:{graph_id} as kickoff not posted yet" - ); - return Ok(()); - } - if withdraw_status != WithdrawStatus::Processing { - tracing::warn!( - "Relayer Ignore finishWithdrawUnhappyPath for {instance_id}:{graph_id}: invalid withdraw status: {withdraw_status}" - ); - return Ok(()); - } - let take2_height = match ctx.btc_client.get_tx_status(&take2_txid).await?.block_height { - Some(height) => height as u64, - None => { - let delay_secs = avg_block_time_secs(ctx.btc_client.network()); // wait for 1 block - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::BitcoinConfirmationPending, - "take2 transaction is not confirmed on Bitcoin", - ) - .await?; - tracing::info!( - "Retry finishWithdrawUnhappyPath later for {instance_id}:{graph_id} as take2 tx not confirmed on btc yet" - ); - return Ok(()); - } - }; - let goat_confirmed_height = ctx.goat_client.btc_spv_latest_height().await?; - if goat_confirmed_height < take2_height { - let delay_secs = - avg_block_time_secs(ctx.btc_client.network()) * (take2_height - goat_confirmed_height); - defer_or_enqueue_message( - ctx.local_db, - &message, - delay_secs as usize, - MessageDeferReason::GoatSpvPending, - "take2 block is not available through GOAT SPV", - ) - .await?; - tracing::info!( - "Retry finishWithdrawUnhappyPath later for {instance_id}:{graph_id} as take2 tx block not posted to goat spv contract yet" - ); - return Ok(()); - } - ctx.goat_client - .gateway_finish_withdraw_unhappy_path(ctx.btc_client, &graph_id, &take2_tx) - .await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_take2_sent_default( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - // triggered by Take2 tx - // 1. update graph status - let _graph = - refresh_graph_status(ctx, instance_id, graph_id, None, GraphStatus::OperatorTake2).await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_sync_graph_request( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - // sent by other nodes when they find a graph is missing locally - // 1. (Relayer) send SyncGraph response if have the graph - if !is_relayer() { - tracing::debug!("Ignore SyncGraphRequest for {instance_id}:{graph_id}: not a relayer node"); - return Ok(()); - } - // Check graph ownership before charging the response budget or recording cooldown. - let now = std::time::Instant::now(); - let graph_key = graph_id.to_string(); - if crate::p2p_admission::sync_graph_response_gate().is_cooling(&graph_key, now) { - return Ok(()); - } - // Require the graph to belong to the requested instance. - if !ctx.local_db.acquire().await?.has_graph_of_instance(&instance_id, &graph_id).await? { - tracing::debug!("Graph not found for SyncGraphRequest {instance_id}:{graph_id}"); - return Ok(()); - } - let requester_registered = - crate::p2p_admission::peer_registry().sender_class(&ctx.from_peer_id, now).is_registered(); - if !crate::p2p_admission::sync_graph_response_budget(requester_registered).allow(now) { - tracing::debug!( - "Defer SyncGraphRequest for {instance_id}:{graph_id}: sync-graph response budget spent" - ); - return Ok(()); - } - let Some(graph) = get_graph(ctx.local_db, instance_id, graph_id).await? else { - // TODO: if no relayer has the graph, how to recover? - tracing::warn!("Graph not loadable for SyncGraphRequest {instance_id}:{graph_id}"); - return Ok(()); - }; - // The cooldown is taken only for a response that is actually going out. - if !crate::p2p_admission::sync_graph_response_gate().allow(&graph_key, now) { - return Ok(()); - } - let message_content = GOATMessageContent::SyncGraph(SyncGraph { instance_id, graph_id, graph }); - send_protocol_message( - ctx.swarm, - ctx.local_db, - GOATMessage::new(Actor::All, message_content), - StoredValue::Fresh, - ) - .await?; - Ok(()) -} - -#[tracing::instrument(level = "info", skip_all, fields(instance_id = %instance_id, graph_id = %graph_id))] -async fn handle_sync_graph( - ctx: &mut HandlerContext<'_>, - instance_id: Uuid, - graph_id: Uuid, - graph: &SimplifiedBitvmGcGraph, -) -> Result<()> { - // Accept only responses to a recorded local request. - if !crate::p2p_admission::requested_graphs() - .contains(&graph_id.to_string(), std::time::Instant::now()) - { - tracing::debug!( - "Ignore SyncGraph for {instance_id}:{graph_id}: no matching local sync request" - ); - return Ok(()); - } - // Require a cached committee identity for the relayer. - if !crate::p2p_admission::peer_registry() - .is_committee(&ctx.from_peer_id, std::time::Instant::now()) - { - tracing::debug!( - "Ignore SyncGraph for {instance_id}:{graph_id}: sender {} is not a committee peer", - ctx.from_peer_id - ); - return Ok(()); - } - // Apply rejection cooldown per graph and sender; RPC failures leave the gate open. - let gate_key = format!("{graph_id}:{}", ctx.from_peer_id); - if crate::p2p_admission::sync_graph_validation_gate() - .is_cooling(&gate_key, std::time::Instant::now()) - { - tracing::debug!( - "Ignore SyncGraph for {instance_id}:{graph_id}: this sender's graph was rejected recently" - ); - return Ok(()); - } - let reject = |why: String| { - crate::p2p_admission::sync_graph_validation_gate() - .allow(&gate_key, std::time::Instant::now()); - tracing::warn!("Ignore SyncGraph for {instance_id}:{graph_id}: {why}"); - }; - - if !message_identity_matches( - "SyncGraph", - instance_id, - graph_id, - None, - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id, - graph.parameters.graph_nonce, - ) { - reject("envelope and graph parameters name different graphs".to_string()); - return Ok(()); - } - - validate_graph_id_on_goat(ctx.goat_client, instance_id, graph_id).await.with_context(|| { - format!("Failed to validate graph_id on GoatChain for SyncGraph {instance_id}:{graph_id}") - })?; - let validation = validate_graph_instance_parameters( - ctx.btc_client, - ctx.goat_client, - &graph.parameters.instance_parameters, - ) - .await; - ctx.metrics_state.record_graph_validation(validation.is_ok()); - if let Err(e) = validation { - // This check talks to both chains. "Could not ask" is not a verdict on - // the graph: let the inbox retry instead of rejecting a good answer. - if crate::action::is_retryable_external_rpc_error(&e) { - return Err(e).context("validate SyncGraph instance parameters"); - } - reject(format!("invalid instance parameters: {e}")); - return Ok(()); - } - let graph = BitvmGcGraph::from_simplified(graph)?; - let graph_data = build_graph_data(&graph)?; - let graph_data_on_goat = ctx.goat_client.gateway_get_graph_data(&graph_id).await?; - if graph_data != graph_data_on_goat { - reject("reconstructed graph data does not match GoatChain".to_string()); - return Ok(()); - } - - if let Err(e) = verify_graph_operator_pre_signatures(&graph) { - reject(format!("invalid operator pre-signatures: {e}")); - return Ok(()); - } - if let Err(e) = verify_graph_committee_pre_signatures(&graph) { - reject(format!("invalid committee pre-signatures: {e}")); - return Ok(()); - } - let simplified_graph = graph.to_simplified()?; - let _ = store_finalized_graph_if_needed(ctx.local_db, &simplified_graph).await?; - refresh_and_compensate(ctx, instance_id, graph_id, &graph, GraphStatus::OperatorPresigned) - .await?; - // Close only after compensation succeeds. Otherwise a transient refresh - // failure would make this inbox row's retry look unsolicited and drop it. - crate::p2p_admission::requested_graphs().remove(&graph_id.to_string()); - Ok(()) -} - -/// A `NodeInfo` payload is entirely self-declared, so only accept it as the -/// sender's own record: the registry is keyed by `peer_id`, and without this -/// check any peer could overwrite any other node's row. -fn accept_node_info(ctx: &HandlerContext<'_>, node_info: &NodeInfo, message_kind: &str) -> bool { - let Err(reason) = check_node_info(ctx, node_info) else { - return true; - }; - // Log registered-peer failures individually and aggregate unregistered-peer failures. - let registered = crate::p2p_admission::peer_registry() - .sender_class(&ctx.from_peer_id, std::time::Instant::now()) - .is_registered(); - if registered { - tracing::warn!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); - } else { - crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::NodeInfoRejected); - tracing::debug!("Ignore {message_kind} from {}: {reason}", ctx.from_peer_id); - } - false -} - -fn check_node_info( - ctx: &HandlerContext<'_>, - node_info: &NodeInfo, -) -> std::result::Result<(), String> { - if !node_info_matches_sender(node_info, &ctx.from_peer_id) { - return Err(format!("payload claims peer_id {}", node_info.peer_id)); - } - validate_node_info_payload(node_info)?; - if node_info.binding_sig.is_empty() { - return Ok(()); - } - // A present-but-invalid binding is a forgery attempt (claiming another key's - // authorisation), not a benign omission, so reject the whole payload. - if crate::action::verify_node_info_binding(node_info).is_none() { - return Err("node info binding signature is invalid".to_string()); - } - // `binding_issued_at` orders re-bindings, newest winning. One dated far ahead - // would win against every honest re-binding until that date. - if node_info.binding_issued_at - > current_time_secs() + crate::action::NODE_INFO_BINDING_MAX_FUTURE_SECS - { - return Err("node info binding is dated in the future".to_string()); - } - Ok(()) -} - -/// Record verified operator bindings by x-only key; confirm stake asynchronously. -async fn learn_node_info_binding( - local_db: &LocalDB, - goat_client: &Arc, - node_info: &NodeInfo, -) { - if node_info.actor != Actor::Operator.to_string() { - return; - } - let Some(operator_key) = crate::action::verify_node_info_binding(node_info) else { - return; - }; - let operator_key = crate::p2p_admission::operator_key(&operator_key); - if let Ok(peer_id) = PeerId::from_str(&node_info.peer_id) { - let released_confirmed_key = crate::p2p_admission::peer_registry() - .observe_operator_binding(&peer_id, &operator_key, node_info.binding_issued_at); - if released_confirmed_key { - // The peer moved to another key: what was confirmed for the old one - // must not be there to be restored after a restart. - crate::p2p_admission::revoke_persisted_operator(local_db, &peer_id).await; - } - crate::p2p_admission::refresh_operator_binding_in_background( - local_db, - goat_client, - peer_id, - operator_key, - ); - } -} - -/// Store accepted NodeInfo within the unregistered-row limit and learn its binding. -async fn record_node_info(ctx: &HandlerContext<'_>, node_info: &NodeInfo) -> Result<()> { - let registered = crate::p2p_admission::peer_registry() - .sender_class(&ctx.from_peer_id, std::time::Instant::now()) - .is_registered(); - let admissible = registered - || ctx - .local_db - .acquire() - .await? - .node_row_admissible( - &node_info.peer_id, - crate::action::NODE_TABLE_MAX_UNREGISTERED_ROWS, - current_time_secs() - crate::action::NODE_TABLE_EVICTABLE_AFTER_SECS, - &crate::env::get_peer_id(), - ) - .await?; - if admissible { - save_node_info(ctx.local_db, node_info).await?; - } else { - crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::NodeTableFull); - } - // The binding is learned either way: it lives in the bounded registry, and - // an operator must be able to become registered while the table is full. - learn_node_info_binding(ctx.local_db, ctx.goat_client, node_info).await; - Ok(()) -} - -async fn handle_request_node_info( - ctx: &mut HandlerContext<'_>, - node_info: &NodeInfo, -) -> Result<()> { - if accept_node_info(ctx, node_info, "RequestNodeInfo") { - record_node_info(ctx, node_info).await?; - } - // Answer regardless: the response only carries this node's own public info, - // and staying silent would break discovery for a misconfigured peer. - // Coalesce recent requests into one broadcast response. - if crate::p2p_admission::node_info_response_gate().try_respond(std::time::Instant::now()) { - publish_node_info_response(ctx.swarm).await?; - } - Ok(()) -} - -async fn publish_node_info_response(swarm: &mut dyn MessagePublisher) -> Result<()> { - let message_content = GOATMessageContent::ResponseNodeInfo(crate::env::get_local_node_info()); - send_to_peer(swarm, GOATMessage::new(Actor::All, message_content)).await?; - Ok(()) -} - -/// Answer the `RequestNodeInfo`s that arrived while the previous response was -/// still cooling down. Called from the regular tick. -pub async fn flush_deferred_node_info_response(swarm: &mut dyn MessagePublisher) { - if !crate::p2p_admission::node_info_response_gate().take_pending(std::time::Instant::now()) { - return; - } - if let Err(error) = publish_node_info_response(swarm).await { - tracing::debug!( - event = "p2p_node_info", - outcome = "deferred_response_failed", - error = %error, - "failed to publish the deferred node info response; peers will ask again" - ); - } -} - -async fn handle_response_node_info( - ctx: &mut HandlerContext<'_>, - node_info: &NodeInfo, -) -> Result<()> { - if accept_node_info(ctx, node_info, "ResponseNodeInfo") { - record_node_info(ctx, node_info).await?; - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use bitcoin::hashes::Hash; - use bitvm_lib::babe_adapter::{ - BABE_M_CC, BabeProverState, build_setup_package, open_and_solder, - }; - use std::str::FromStr; - - use super::*; - - #[tokio::test] - async fn missing_pegin_event_is_ignored_without_enqueuing_retry() { - let local_db = store::create_local_db("sqlite::memory:").await; - let instance_id = Uuid::new_v4(); - - let metadata = - resolve_pegin_request_metadata(&local_db, instance_id, "0xuntrusted", i64::MAX) - .await - .unwrap(); - - assert!(metadata.is_none()); - let mut storage_processor = local_db.acquire().await.unwrap(); - let queued = storage_processor - .find_message_by_business_id(&instance_id, "PeginRequest") - .await - .unwrap(); - assert!(queued.is_none()); - } - - fn verifier_pubkey() -> PublicKey { - PublicKey::from_str("0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798") - .unwrap() - } - - fn gc_submission() -> (CACSetupPackage, BitvmGcCircuitData, BabeProverState) { - let package = build_setup_package(BABE_M_CC + 1).unwrap(); - let selected = (0..BABE_M_CC).collect::>(); - let (_, finalized, soldering) = open_and_solder(&package, &selected).unwrap(); - let epk = &package.commits[finalized[0].index].epk; - let h_msgs: Vec<[u8; 20]> = - finalized.iter().map(|f| package.commits[f.index].h_msg).collect(); - let gc_data = extract_gc_circuit_data(verifier_pubkey(), epk, &h_msgs).unwrap(); - let prover_state = BabeProverState { - package: package.clone(), - finalized, - soldering, - h_msgs: gc_data.final_msg_hashlocks.clone(), - }; - (package, gc_data, prover_state) - } - - fn soldering_proof_ready(hash_byte: u8) -> SolderingProofReady { - SolderingProofReady { - instance_id: Uuid::nil(), - graph_id: Uuid::nil(), - candidate_index: 0, - payload_hash: [hash_byte; 32], - total_len: 1, - } - } - - fn operator_state(package: CACSetupPackage) -> OperatorBabeSetupState { - OperatorBabeSetupState { - candidate_verifier_pubkeys: Some(vec![verifier_pubkey()]), - candidates: vec![OperatorVerifierCandidate { - verifier_peer_id: vec![1], - verifier_pubkey: verifier_pubkey(), - setup_package: package, - candidate_index: Some(0), - selected_circuit_indexes: (0..BABE_M_CC).collect(), - gc_data: None, - soldering_proof_ready: None, - }], - candidate_collection_started_at: None, - proof_collection_started_at: None, - selected_verifier_pubkeys: None, - asserted_operator_proof: None, - } - } - - /// Verify retries fill missing CutCircuits entries without changing existing entries. - #[tokio::test] - async fn retry_after_a_cut_off_freeze_completes_the_cut_circuits() { - use bitcoin::secp256k1::{Secp256k1, SecretKey}; - - let sender_peer_id = generate_local_key().public().to_peer_id().to_string(); - - let local_db = store::create_local_db("sqlite::memory:").await; - let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); - let package = build_setup_package(BABE_M_CC + 1).unwrap(); - let secp = Secp256k1::new(); - let mut state = OperatorBabeSetupState { - candidate_verifier_pubkeys: None, - candidates: (1..=3u8) - .map(|index| OperatorVerifierCandidate { - verifier_peer_id: PeerId::random().to_bytes(), - verifier_pubkey: PublicKey::new( - SecretKey::from_slice(&[index; 32]).unwrap().public_key(&secp), - ), - setup_package: package.clone(), - candidate_index: None, - selected_circuit_indexes: vec![], - gc_data: None, - soldering_proof_ready: None, - }) - .collect(), - candidate_collection_started_at: None, - proof_collection_started_at: None, - selected_verifier_pubkeys: None, - asserted_operator_proof: None, - }; - let now = current_time_secs(); - assert!(cut_circuits_owed(&state, now).is_empty(), "nothing is owed before the freeze"); - freeze_operator_candidates(&mut state).unwrap(); - let outbox_id = |candidate: &OperatorVerifierCandidate| { - format!("cut-circuits:{instance_id}:{graph_id}:{}", candidate.verifier_pubkey) - }; - let outbox_state = async |message_id: String| -> Option<(String, i64)> { - local_db.acquire().await.unwrap().p2p_outbox_entry_state(&message_id).await.unwrap() - }; - - // First run: the set is frozen and saved, one CutCircuits is written, and - // then the handler is cut off. - let owed = cut_circuits_owed(&state, now); - assert_eq!(owed.len(), 3); - ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed[..1], &sender_peer_id) - .await - .unwrap(); - // That verifier acknowledges it before the retry comes round. - let first_peer = PeerId::from_bytes(&owed[0].verifier_peer_id).unwrap().to_string(); - let first_delivery_id = local_db - .acquire() - .await - .unwrap() - .p2p_outbox_delivery_id(&outbox_id(&owed[0])) - .await - .unwrap() - .unwrap(); - assert!( - local_db - .acquire() - .await - .unwrap() - .acknowledge_p2p_outbox_message(&first_delivery_id, &first_peer) - .await - .unwrap() - ); - assert!(outbox_state(outbox_id(&owed[1])).await.is_none(), "never written"); - - // The retry: the set is frozen already, and everything is still owed. - let owed = cut_circuits_owed(&state, now); - assert_eq!(owed.len(), 3); - ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed, &sender_peer_id) - .await - .unwrap(); - assert_eq!( - outbox_state(outbox_id(&owed[0])).await, - Some(("Processed".to_string(), 0)), - "the acknowledged entry is left as it is, not re-opened" - ); - for candidate in &owed[1..] { - let (entry_state, len) = outbox_state(outbox_id(candidate)).await.unwrap(); - assert_eq!(entry_state, "Pending"); - assert!(len > 0, "the missing CutCircuits is now queued for delivery"); - } - // And again: nothing changes. - ensure_cut_circuits_outbox(&local_db, instance_id, graph_id, &owed, &sender_peer_id) - .await - .unwrap(); - assert_eq!(outbox_state(outbox_id(&owed[0])).await, Some(("Processed".to_string(), 0))); - - // A candidate that has delivered its proof is owed nothing more ... - state.candidates[0].soldering_proof_ready = Some(soldering_proof_ready(1)); - assert_eq!(cut_circuits_owed(&state, now).len(), 2); - // ... nobody is once the proof window has closed with enough proofs in - // hand, so a late retry does not set verifiers garbling for nothing ... - let (_, gc_data, _) = gc_submission(); - state.candidates[0].gc_data = Some(gc_data); - state.proof_collection_started_at = Some(now); - assert_eq!(cut_circuits_owed(&state, now).len(), 2, "the window is still open"); - let after_window = now + get_verifier_candidate_collection_window_secs(); - assert!(cut_circuits_owed(&state, after_window).is_empty()); - // ... and nobody is once the selection is sealed. - state.proof_collection_started_at = None; - state.selected_verifier_pubkeys = Some(vec![state.candidates[0].verifier_pubkey]); - assert!(cut_circuits_owed(&state, now).is_empty()); - } - - #[test] - fn freeze_operator_candidate_selects_protocol_finalized_count() { - let package = build_setup_package(BABE_M_CC + 1).unwrap(); - let mut state = OperatorBabeSetupState { - candidate_verifier_pubkeys: None, - candidates: vec![OperatorVerifierCandidate { - verifier_peer_id: vec![1], - verifier_pubkey: verifier_pubkey(), - setup_package: package, - candidate_index: None, - selected_circuit_indexes: vec![], - gc_data: None, - soldering_proof_ready: None, - }], - candidate_collection_started_at: None, - proof_collection_started_at: None, - selected_verifier_pubkeys: None, - asserted_operator_proof: None, - }; - - freeze_operator_candidates(&mut state).unwrap(); - - assert_eq!(state.candidates[0].selected_circuit_indexes.len(), BABE_M_CC); - } - - #[test] - fn candidate_records_one_slot_and_payload_reference_idempotently() { - let (package, gc_data, prover_state) = gc_submission(); - let mut state = operator_state(package.clone()); - let soldering_proof_ready = soldering_proof_ready(1); - - record_candidate_gc_data( - &mut state, - verifier_pubkey(), - 0, - &package, - gc_data.clone(), - &prover_state, - soldering_proof_ready.clone(), - ) - .unwrap(); - - assert_eq!( - state.candidates[0].gc_data.as_ref().unwrap().final_msg_hashlocks.len(), - BABE_M_CC - ); - assert_eq!( - state.candidates[0].soldering_proof_ready.as_ref(), - Some(&soldering_proof_ready) - ); - - record_candidate_gc_data( - &mut state, - verifier_pubkey(), - 0, - &package, - gc_data.clone(), - &prover_state, - soldering_proof_ready.clone(), - ) - .unwrap(); - - let mut conflict = prover_state; - conflict.h_msgs[0][0] ^= 1; - assert!( - record_candidate_gc_data( - &mut state, - verifier_pubkey(), - 0, - &package, - gc_data, - &conflict, - soldering_proof_ready, - ) - .is_err() - ); - } - - #[test] - fn rebuilds_prover_state_from_persisted_payload_data() { - let package = build_setup_package(BABE_M_CC + 1).unwrap(); - let selected = (0..BABE_M_CC).collect::>(); - let (_, finalized, soldering) = open_and_solder(&package, &selected).unwrap(); - - let restored = - build_babe_prover_state(&package, finalized.clone(), soldering.clone()).unwrap(); - - assert_eq!(restored.package, package); - assert_eq!(restored.finalized, finalized); - assert_eq!(restored.soldering, soldering); - assert_eq!(restored.h_msgs.len(), BABE_M_CC); - } - - // ── collect_ack_txins ───────────────────────────────────────────────────── - - fn make_txid(byte: u8) -> Txid { - Txid::from_byte_array([byte; 32]) - } - - fn make_confirmed_ack_tx(ack_txid: Txid, wci_txid: Txid, ack_vout: u32) -> esplora_client::Tx { - use esplora_client::{TxStatus, Vin}; - let preimage = b"test-ack-preimage".to_vec(); - esplora_client::Tx { - txid: ack_txid, - version: 2, - locktime: 0, - vin: vec![Vin { - txid: wci_txid, - vout: ack_vout, - prevout: None, - scriptsig: bitcoin::ScriptBuf::default(), - witness: vec![preimage], - sequence: 1, - is_coinbase: false, - }], - vout: vec![], - size: 1, - weight: 1, - status: TxStatus { - confirmed: true, - block_height: Some(800_000), - block_hash: None, - block_time: Some(1_000_000), - }, - fee: 0, - } - } - - #[tokio::test] - async fn collect_ack_txins_returns_only_spent_connectors() { - let (btc_client, mock) = client::btc_chain::BTCClient::new_mock_client(); - let wci_txid = make_txid(0x00); - let ack_txid_0 = make_txid(0x01); - let timeout_txid_1 = make_txid(0x03); - let timeout_txids = vec![make_txid(0x10), timeout_txid_1, make_txid(0x12)]; - - // Watchtower 0 ACKed - let vout_0 = output_topology::watchtower_challenge_init::ack_connector(0) as u32; - mock.set_tx(ack_txid_0, make_confirmed_ack_tx(ack_txid_0, wci_txid, vout_0)); - - // Watchtower 1 timed out; the timeout spend must not count as an ACK. - let vout_1 = output_topology::watchtower_challenge_init::ack_connector(1) as u32; - mock.set_tx(timeout_txid_1, make_confirmed_ack_tx(timeout_txid_1, wci_txid, vout_1)); - - // Watchtower 2 ACKed - let ack_txid_2 = make_txid(0x02); - let vout_2 = output_topology::watchtower_challenge_init::ack_connector(2) as u32; - mock.set_tx(ack_txid_2, make_confirmed_ack_tx(ack_txid_2, wci_txid, vout_2)); - - let txins = collect_ack_txins(&btc_client, &wci_txid, &timeout_txids).await.unwrap(); - - assert_eq!(txins.len(), 2); - assert_eq!(txins[0].previous_output.vout, vout_0); - assert_eq!(txins[1].previous_output.vout, vout_2); - } - - // ── recover_challenge_assert_witness ────────────────────────────────────── - - fn build_challenge_assert_tx( - sig: &::Signature, - labels: &[[u8; 16]], - ) -> bitcoin::Transaction { - let mut witness = bitcoin::Witness::new(); - let raw_wots = Wots96::signature_to_raw_witness(sig); - for item in raw_wots.iter() { - witness.push(item); - } - for label in labels { - witness.push(label.as_slice()); - } - witness.push([0xde, 0xad]); // placeholder script - witness.push([0xbe, 0xef]); // placeholder control block - bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![bitcoin::TxIn { witness, ..Default::default() }], - output: vec![], - } - } - - #[test] - fn recover_challenge_assert_witness_extracts_wots_sig_and_labels() { - use bitvm_lib::operator::generate_assert_wots_key; - - let (sk, _) = generate_assert_wots_key("test-challenge-witness"); - let msg = [0x42u8; 96]; - let sig = Wots96::sign(&sk, &msg); - - let labels: Vec<[u8; 16]> = - (0..goat::assert_scripts::INPUT_WIRE_NUM).map(|i| [(i % 256) as u8; 16]).collect(); - - let tx = build_challenge_assert_tx(&sig, &labels); - let verifier_index = 3; - let recovered = recover_challenge_assert_witness(&tx, verifier_index).unwrap(); - - assert_eq!(recovered.verifier_index, verifier_index); - assert_eq!(recovered.witness.wots_sig.len(), WOTS_SIG_COUNT); - assert_eq!(recovered.witness.input_labels.len(), goat::assert_scripts::INPUT_WIRE_NUM); - - let mut bw = bitcoin::Witness::new(); - for item in Wots96::signature_to_raw_witness(&sig).iter() { - bw.push(item); - } - assert_eq!(recovered.witness.wots_sig, Wots96::raw_witness_to_signature(&bw).to_vec()); - - // labels are correctly extracted - for (i, label) in recovered.witness.input_labels.iter().enumerate() { - assert_eq!(label[0], (i % 256) as u8, "label {i} mismatch"); - } - } - - #[test] - fn recover_challenge_assert_witness_rejects_wrong_length() { - use bitvm_lib::operator::generate_assert_wots_key; - - let (sk, _) = generate_assert_wots_key("test-wrong-len"); - let sig = Wots96::sign(&sk, &[0u8; 96]); - let labels: Vec<[u8; 16]> = vec![[0u8; 16]; goat::assert_scripts::INPUT_WIRE_NUM]; - - // one label short - let mut short_labels = labels.clone(); - short_labels.pop(); - let bad_tx = build_challenge_assert_tx(&sig, &short_labels); - assert!(recover_challenge_assert_witness(&bad_tx, 0).is_err()); - - // no inputs at all - let empty_tx = bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![], - output: vec![], - }; - assert!(recover_challenge_assert_witness(&empty_tx, 0).is_err()); - } - - #[test] - fn wrongly_challenge_outputs_exceed_minimum_non_witness_size() { - // Bitcoin Core rejects standard transactions smaller than 65 non-witness bytes. - let tx = bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![bitcoin::TxIn::default()], - output: vec![ - goat::scripts::p2a_output(), - bitcoin::TxOut { - value: Amount::ZERO, - script_pubkey: goat::scripts::generate_opreturn_script( - WRONGLY_CHALLENGED_OP_RETURN_DATA.to_vec(), - ), - }, - ], - }; - - assert!( - tx.base_size() >= 65, - "wrongly-challenge transaction is {} non-witness bytes", - tx.base_size() - ); - } - - #[test] - fn pubin_disprove_outputs_exceed_minimum_non_witness_size() { - // Bitcoin Core rejects standard transactions smaller than 65 non-witness bytes. - let tx = bitcoin::Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![bitcoin::TxIn::default()], - output: vec![ - goat::scripts::p2a_output(), - bitcoin::TxOut { - value: Amount::ZERO, - script_pubkey: goat::scripts::generate_opreturn_script( - PUBIN_DISPROVE_OP_RETURN_DATA.to_vec(), - ), - }, - ], - }; - - assert!( - tx.base_size() >= 65, - "pubin-disprove transaction is {} non-witness bytes", - tx.base_size() - ); - } - #[tokio::test] - async fn late_round_messages_are_dropped_only_once_the_round_is_provably_over() { - let local_db = store::create_local_db("sqlite::memory:").await; - let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); - // No row yet: the full path must run, it is what asks for the graph. - assert!(!graph_signing_round_is_over(&local_db, instance_id, graph_id).await.unwrap()); - - let mut storage = local_db.acquire().await.unwrap(); - storage - .upsert_graph_definition(&store::Graph { - graph_id, - instance_id, - status: GraphStatus::OperatorPresigned.to_string(), - definition_hash: "definition".to_owned(), - ..Default::default() - }) - .await - .unwrap(); - drop(storage); - assert!(!graph_signing_round_is_over(&local_db, instance_id, graph_id).await.unwrap()); - - // Obsoleted straight from OperatorPresigned: the round never finished. - let obsoleted = Uuid::new_v4(); - let mut storage = local_db.acquire().await.unwrap(); - storage - .upsert_graph_definition(&store::Graph { - graph_id: obsoleted, - instance_id, - status: GraphStatus::OperatorPresigned.to_string(), - definition_hash: "definition".to_owned(), - ..Default::default() - }) - .await - .unwrap(); - storage - .transition_graph_status( - instance_id, - obsoleted, - GraphStatus::Obsoleted, - store::GraphStatusSource::ChainReconcile, - None, - ) - .await - .unwrap(); - storage - .transition_graph_status( - instance_id, - graph_id, - GraphStatus::CommitteePresigned, - store::GraphStatusSource::Definition, - None, - ) - .await - .unwrap(); - drop(storage); - assert!(!graph_signing_round_is_over(&local_db, instance_id, obsoleted).await.unwrap()); - assert!(graph_signing_round_is_over(&local_db, instance_id, graph_id).await.unwrap()); - // The row answers for its own instance only. - assert!(!graph_signing_round_is_over(&local_db, Uuid::new_v4(), graph_id).await.unwrap()); - } -} diff --git a/node/src/lib.rs b/node/src/lib.rs index 78665b20f..d584a90f8 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -1,11 +1,16 @@ -pub mod action; +pub mod aggregates; +pub mod clock; +pub mod control; +pub mod engine_bridge; pub mod env; -pub mod handle; +pub mod messages; pub mod metrics_service; pub mod middleware; pub mod p2p_admission; pub mod p2p_msg_handler; +#[cfg(feature = "mock-gc")] +pub mod mock_gc; pub mod rpc_service; mod scheduled_tasks; mod soldering_payload_store; diff --git a/node/src/main.rs b/node/src/main.rs index 0227df492..0b0f1cc12 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -21,13 +21,14 @@ use bitvm_noded::utils::{ self, generate_local_key, save_local_info, set_node_external_socket_addr_env, }; use bitvm_noded::{ - rpc_service, run_maintenance_tasks, run_sequencer_set_hash_monitor_task, run_watch_event_task, + engine_bridge, rpc_service, run_maintenance_tasks, run_sequencer_set_hash_monitor_task, + run_watch_event_task, }; use anyhow::Result; -use bitvm_noded::action::{load_persisted_peer_bindings, reclaim_stale_queue_claims}; use bitvm_noded::metrics_service::{MetricsState, set_node_metrics_state}; use bitvm_noded::middleware::swarm::{BitvmNetworkManager, BitvmSwarmConfig}; +use bitvm_noded::p2p_admission::load_persisted_peer_bindings; use bitvm_noded::p2p_msg_handler::BitvmNodeProcessor; use client::http_client::async_client::HttpAsyncClient; use futures::future; @@ -196,24 +197,44 @@ async fn main() -> Result<(), Box> { version = env!("CARGO_PKG_VERSION"), ); let _node_span_guard = node_span.enter(); - let local_db = store::create_local_db(&opt.db_path).await; - // Claims left by a previous process are provably abandoned: charge and - // release them before any dispatcher can wait on their leases. - let (reclaimed_local, reclaimed_inbox) = reclaim_stale_queue_claims(&local_db).await?; - if reclaimed_local + reclaimed_inbox > 0 { - tracing::warn!( - event = "message_queue_startup", - outcome = "claims_reclaimed", - reclaimed_local, - reclaimed_inbox, - "reclaimed queue claims left behind by a previous process" - ); - } + let local_db = + store::create_local_db_with_busy_timeout(&opt.db_path, engine_bridge::db_busy_timeout()) + .await; + // The execution engine's metrics must be registered before the registry is shared. + let engine_metrics = engine_bridge::register_engine_metrics(&mut metric_registry); let metric_registry = Arc::new(Mutex::new(metric_registry)); let metrics_state = MetricsState::new(metric_registry); set_node_metrics_state(metrics_state.clone()); let goat_client = Arc::new(GOATClient::new(env::goat_config_from_env().await, env::get_goat_network())); + let btc_client = Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())); + let http_client = Arc::new(HttpAsyncClient::new(None)); + let soldering_builder = if actor_needs_soldering_builder(&actor) { + // Under `mock-gc` the builder is a free unit struct, so a configurable delay stands in for + // the Ziren soldering-guest setup the real builder performs here. + #[cfg(feature = "mock-gc")] + bitvm_noded::mock_gc::simulate(bitvm_noded::mock_gc::GcOp::BuilderInit).await; + Some(Arc::new(BabeBundleBuilder::new())) + } else { + None + }; + // The execution engine owns the migrated aggregates (Instance and Graph). + let (engine, engine_acks) = engine_bridge::build_engine( + bitvm_network_manager.publisher(), + engine_metrics, + engine_bridge::EngineDeps { + actor: actor.clone(), + local_peer: peer_id_string.clone(), + local_db: local_db.clone(), + btc_client: btc_client.clone(), + goat_client: goat_client.clone(), + http_client: http_client.clone(), + soldering_builder: soldering_builder.clone(), + metrics_state: metrics_state.clone(), + }, + ); + let engine_handle = engine.handle(); + engine_bridge::install_engine_handle(engine_handle.clone()); // Restore persisted operator bindings into the admission registry. match load_persisted_peer_bindings(&local_db, &goat_client).await { Ok(loaded) if loaded > 0 => tracing::info!( @@ -232,15 +253,12 @@ async fn main() -> Result<(), Box> { } let handler = BitvmNodeProcessor { local_db: local_db.clone(), - btc_client: Arc::new(BTCClient::new(get_network(), get_btc_url_from_env().as_deref())), goat_client, - http_client: Arc::new(HttpAsyncClient::new(None)), - soldering_builder: actor_needs_soldering_builder(&actor) - .then(|| Arc::new(BabeBundleBuilder::new())), metrics_state: metrics_state.clone(), shutdown_token: cancellation_token.clone(), worker: Default::default(), }; + handler.start(bitvm_network_manager.publisher()); tracing::info!( event = "service_started", @@ -252,6 +270,13 @@ async fn main() -> Result<(), Box> { rpc_addr = %opt.rpc_addr, metrics_addr = opt.metrics_addr.as_deref().unwrap_or("disabled"), min_required_verifier, + btc_chain_url = get_btc_url_from_env().unwrap_or_default(), + goat_chain_url = %env::get_goat_url_from_env(), + gateway_graph_url = env::get_goat_gateway_the_graph_urls_from_env(), + btc_request_timeout_secs = client::timeout_config::get_btc_request_timeout_secs(), + goat_rpc_timeout_secs = client::timeout_config::get_goat_rpc_timeout_secs(), + http_request_timeout_secs = client::timeout_config::get_http_request_timeout_secs(), + maintenance_interval_secs = env::REGULAR_TASK_INTERVAL_SECOND, "node initialization completed" ); @@ -277,6 +302,7 @@ async fn main() -> Result<(), Box> { actor.clone(), peer_id_string.clone(), metrics_state.clone(), + Some(engine_handle.clone()), ) .await?; @@ -415,6 +441,7 @@ async fn main() -> Result<(), Box> { let cancel_token_clone = cancellation_token.clone(); let maintenance_task_span = node_span.clone(); let maintenance_metrics = metrics_state.clone(); + let maintenance_engine = engine_handle.clone(); task_handles.push(tokio::spawn( async move { let goat_client = @@ -429,6 +456,7 @@ async fn main() -> Result<(), Box> { 10, cancel_token_clone, maintenance_metrics, + maintenance_engine, ) .await { @@ -451,6 +479,57 @@ async fn main() -> Result<(), Box> { )); task_names.push("maintenance"); + { + let acks = engine_acks; + let cancel_token_clone = cancellation_token.clone(); + let engine_task_span = node_span.clone(); + task_handles.push(tokio::spawn( + async move { + match engine.run(cancel_token_clone).await { + Ok(()) => Ok("engine stopped".to_string()), + Err(error) => { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "engine", + outcome = "failed", + error_class = "engine", + error = %error, + "execution engine exited with an error" + ); + Err("engine_error".to_string()) + } + } + } + .instrument(engine_task_span), + )); + task_names.push("engine"); + + let cancel_token_clone = cancellation_token.clone(); + let ack_task_span = node_span.clone(); + let ack_publisher = bitvm_network_manager.publisher(); + task_handles.push(tokio::spawn( + async move { + engine_bridge::run_ack_forwarder(acks, ack_publisher, cancel_token_clone) + .await + .map_err(|error| { + tracing::error!( + event = "core_task_wrapper_error", + service = "bitvm-noded", + task = "engine_ack", + outcome = "failed", + error_class = "engine", + error = %error, + "engine ACK forwarder exited with an error" + ); + "engine_ack_error".to_string() + }) + } + .instrument(ack_task_span), + )); + task_names.push("engine_ack"); + } + let swarm_actor = actor.clone(); let cancel_token_clone = cancellation_token.clone(); let p2p_task_span = node_span.clone(); diff --git a/node/src/messages.rs b/node/src/messages.rs new file mode 100644 index 000000000..2aeeeabb4 --- /dev/null +++ b/node/src/messages.rs @@ -0,0 +1,462 @@ +//! Wire messages of the node: the envelope, the message kinds and their +//! routing table, and the signatures that bind InitGraph and NodeInfo to +//! their authors. + +use alloy::primitives::Address as EvmAddress; +use anyhow::{Context, Result}; +use bitcoin::{PublicKey, XOnlyPublicKey}; +use bitvm_lib::actors::Actor; +use bitvm_lib::babe_adapter::CACSetupPackage; +use bitvm_lib::committee::*; +use bitvm_lib::types::SimplifiedBitvmGcGraph; +use musig2::{PartialSignature, PubNonce}; +use node_macros::MessageBusinessRef; +use secp256k1::{ + Keypair, Message as SecpMessage, SECP256K1, schnorr::Signature as SchnorrSignature, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::str::FromStr; +use strum::{Display, EnumDiscriminants, EnumIter, IntoStaticStr}; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Clone)] +pub struct GOATMessage { + pub actor: Actor, + pub content: GOATMessageContent, +} + +const GOAT_MESSAGE_BIN_PREFIX: &[u8] = b"GOATBIN1"; +/// Maximum accepted binding timestamp lead. +pub const NODE_INFO_BINDING_MAX_FUTURE_SECS: i64 = 10 * 60; +/// Delivery semantics for externally received P2P messages. +/// +/// Durable messages are recorded as engine inbox events of their aggregate. +/// Immediate messages carry recoverable peer state or ACKs and can be resent. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum P2PMessageDelivery { + Inbox, + Immediate, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BusinessRef { + Instance { instance_id: Uuid }, + Graph { instance_id: Uuid, graph_id: Uuid }, + Unscoped, +} + +pub trait HasBusinessRef { + fn business_ref(&self) -> BusinessRef; +} + +#[derive(Serialize, Deserialize, Clone, EnumDiscriminants, MessageBusinessRef)] +#[strum_discriminants(name(MessageKind))] +#[strum_discriminants(derive(Display, EnumIter, IntoStaticStr))] +pub enum GOATMessageContent { + #[business_ref(graph)] + CreateGraph(CreateGraph), + #[business_ref(graph)] + InitGraph(InitGraph), + #[business_ref(graph)] + GenCircuits(GenCircuits), + #[business_ref(graph)] + CutCircuits(CutCircuits), + #[business_ref(graph)] + SolderingProofReady(SolderingProofReady), + #[business_ref(unscoped)] + GraphSetupAck(GraphSetupAck), + #[business_ref(graph)] + VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement), + #[business_ref(graph)] + NonceGeneration(NonceGeneration), + #[business_ref(graph)] + AggNonceConsensus(AggNonceConsensus), + #[business_ref(graph)] + CommitteePresign(CommitteePresign), + #[business_ref(graph)] + EndorseGraph(EndorseGraph), + #[business_ref(graph)] + GraphFinalize(GraphFinalize), + #[business_ref(instance)] + PeginConfirmNonce(PeginConfirmNonce), + #[business_ref(instance)] + PeginConfirmNonceConsensus(PeginConfirmNonceConsensus), + #[business_ref(instance)] + PeginConfirmPartialSig(PeginConfirmPartialSig), + #[business_ref(unscoped)] + RequestNodeInfo(NodeInfo), + #[business_ref(unscoped)] + ResponseNodeInfo(NodeInfo), +} + +impl MessageKind { + /// Receiver roles that take the kind: the control dispatcher for the + /// immediate kinds, the Graph aggregate (`aggregates::graph::route_action`) + /// for the graph phases, the Instance aggregate + /// (`aggregates::instance::route_action`) for the PeginConfirm round. + pub fn handled_by(self, actor: &Actor) -> bool { + match self { + Self::GraphSetupAck | Self::RequestNodeInfo | Self::ResponseNodeInfo => true, + Self::InitGraph + | Self::GenCircuits + | Self::CutCircuits + | Self::SolderingProofReady + | Self::CreateGraph + | Self::VerifierGraphParamsEndorsement + | Self::NonceGeneration + | Self::AggNonceConsensus + | Self::CommitteePresign + | Self::EndorseGraph + | Self::GraphFinalize => { + let kind: &'static str = self.into(); + crate::aggregates::graph::engine_handles(kind, actor) + } + Self::PeginConfirmNonce + | Self::PeginConfirmNonceConsensus + | Self::PeginConfirmPartialSig => { + let kind: &'static str = self.into(); + crate::aggregates::instance::engine_handles(kind, actor) + } + } + } +} + +impl GOATMessageContent { + pub fn kind(&self) -> MessageKind { + self.into() + } + + /// New messages default to the durable inbox and must explicitly opt into + /// immediate processing when they are safe to drop. + pub const fn p2p_delivery(&self) -> P2PMessageDelivery { + match self { + Self::RequestNodeInfo(_) | Self::ResponseNodeInfo(_) | Self::GraphSetupAck(_) => { + P2PMessageDelivery::Immediate + } + _ => P2PMessageDelivery::Inbox, + } + } + + /// Stable message name for logs/metrics. Keep this independent of `Debug`, whose + /// output can include protocol payloads (and, for proofs, be very large). + pub fn event_type(&self) -> &'static str { + self.kind().into() + } +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct InitGraph { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub operator_pubkey: PublicKey, + pub operator_peer_id: Vec, + pub signature: SchnorrSignature, +} + +const INIT_GRAPH_SIGNATURE_DOMAIN: &[u8] = b"bitvm2-node/init-graph/v1"; + +fn init_graph_signature_message( + instance_id: Uuid, + graph_id: Uuid, + operator_pubkey: &PublicKey, + operator_peer_id: &[u8], +) -> SecpMessage { + let mut hasher = Sha256::new(); + hasher.update(INIT_GRAPH_SIGNATURE_DOMAIN); + hasher.update(instance_id.as_bytes()); + hasher.update(graph_id.as_bytes()); + hasher.update(operator_pubkey.to_bytes()); + hasher.update((operator_peer_id.len() as u32).to_be_bytes()); + hasher.update(operator_peer_id); + SecpMessage::from_digest(hasher.finalize().into()) +} + +pub fn sign_init_graph( + instance_id: Uuid, + graph_id: Uuid, + operator_keypair: &Keypair, + operator_peer_id: Vec, +) -> InitGraph { + let operator_pubkey = operator_keypair.public_key().into(); + let signature = SECP256K1.sign_schnorr( + &init_graph_signature_message(instance_id, graph_id, &operator_pubkey, &operator_peer_id), + operator_keypair, + ); + InitGraph { instance_id, graph_id, operator_pubkey, operator_peer_id, signature } +} + +pub fn verify_init_graph_signature(message: &InitGraph) -> bool { + SECP256K1 + .verify_schnorr( + &message.signature, + &init_graph_signature_message( + message.instance_id, + message.graph_id, + &message.operator_pubkey, + &message.operator_peer_id, + ), + &XOnlyPublicKey::from(message.operator_pubkey), + ) + .is_ok() +} +#[derive(Serialize, Deserialize, Clone)] +pub struct GenCircuits { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub verifier_pubkey: PublicKey, + pub setup_package: CACSetupPackage, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct CutCircuits { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub verifier_pubkey: PublicKey, + pub candidate_index: usize, + pub selected_circuit_indexes: Vec, +} +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct SolderingProofReady { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub candidate_index: usize, + pub payload_hash: [u8; 32], + pub total_len: usize, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct GraphSetupAck { + pub delivery_id: String, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct VerifierGraphParamsEndorsement { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub verifier_pubkey: PublicKey, + pub verifier_index: usize, + pub canonical_graph_params_hash: [u8; 32], + pub signature: SchnorrSignature, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct CreateGraph { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub graph_nonce: u64, + pub graph: SimplifiedBitvmGcGraph, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct NonceGeneration { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub committee_pubkey: PublicKey, + pub pub_nonces: CommitteePubNonces, + pub nonce_sigs: CommitteeNonceSignatures, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct AggNonceConsensus { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub committee_pubkey: PublicKey, + pub consensus_hash: [u8; 32], + pub signature: SchnorrSignature, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct CommitteePresign { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub committee_pubkey: PublicKey, + pub committee_partial_sigs: CommitteePartialSignatures, + pub agg_nonces: CommitteeAggNonces, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct EndorseGraph { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub committee_pubkey: PublicKey, + pub committee_evm_address: EvmAddress, + pub committee_sig_for_graph: Vec, // ECDSA signature signed with committee evm keypair + pub committee_sig_for_params: Vec, // ECDSA signature over canonical_graph_params_hash +} +#[derive(Serialize, Deserialize, Clone)] +pub struct GraphFinalize { + pub instance_id: Uuid, + pub graph_id: Uuid, + pub graph_nonce: u64, + pub graph: SimplifiedBitvmGcGraph, + pub endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, + pub params_endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct PeginConfirmNonce { + pub instance_id: Uuid, + pub committee_pubkey: PublicKey, + pub pub_nonce: PubNonce, + pub nonce_sig: SchnorrSignature, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct PeginConfirmNonceConsensus { + pub instance_id: Uuid, + pub committee_pubkey: PublicKey, + pub consensus_hash: [u8; 32], + pub signature: SchnorrSignature, +} +#[derive(Serialize, Deserialize, Clone)] +pub struct PeginConfirmPartialSig { + pub instance_id: Uuid, + pub committee_pubkey: PublicKey, + pub partial_sig: PartialSignature, + pub endorse_sig: Vec, // ECDSA signature signed with committee evm keypair +} +/// Others + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct NodeInfo { + pub peer_id: String, + pub actor: String, + pub goat_addr: String, + pub btc_pub_key: String, + pub socket_addr: String, + pub node_name: String, + pub service_fee_rate: f64, + pub available_peg_btc: String, + /// Hex master-key Schnorr signature over peer ID, public key and issuance time. + #[serde(default)] + pub binding_sig: String, + /// Unix seconds when the binding was signed. Orders re-bindings so one key + /// maps to a single peer id at a time. + #[serde(default)] + pub binding_issued_at: i64, +} + +const NODE_INFO_BINDING_DOMAIN: &[u8] = b"bitvm2-node/node-info-binding/v1"; + +fn node_info_binding_message(peer_id: &str, btc_pub_key: &str, issued_at: i64) -> SecpMessage { + let mut hasher = Sha256::new(); + hasher.update(NODE_INFO_BINDING_DOMAIN); + hasher.update((peer_id.len() as u32).to_be_bytes()); + hasher.update(peer_id.as_bytes()); + hasher.update((btc_pub_key.len() as u32).to_be_bytes()); + hasher.update(btc_pub_key.as_bytes()); + hasher.update(issued_at.to_be_bytes()); + SecpMessage::from_digest(hasher.finalize().into()) +} + +/// Sign the binding that proves `master_keypair`'s key authorised `peer_id`. +pub fn sign_node_info_binding( + peer_id: &str, + btc_pub_key: &str, + issued_at: i64, + master_keypair: &Keypair, +) -> String { + let signature = SECP256K1 + .sign_schnorr(&node_info_binding_message(peer_id, btc_pub_key, issued_at), master_keypair); + hex::encode(signature.serialize()) +} + +/// Verify the NodeInfo binding and return its parsed public key. +pub fn verify_node_info_binding(node_info: &NodeInfo) -> Option { + if node_info.binding_sig.is_empty() { + return None; + } + let pubkey = PublicKey::from_str(&node_info.btc_pub_key).ok()?; + let xonly = XOnlyPublicKey::from(pubkey); + let signature_bytes = hex::decode(&node_info.binding_sig).ok()?; + let signature = SchnorrSignature::from_slice(&signature_bytes).ok()?; + SECP256K1 + .verify_schnorr( + &signature, + &node_info_binding_message( + &node_info.peer_id, + &node_info.btc_pub_key, + node_info.binding_issued_at, + ), + &xonly, + ) + .ok() + .map(|()| xonly) +} + +impl GOATMessage { + pub fn new(actor: Actor, content: GOATMessageContent) -> Self { + Self { actor, content } + } + + pub fn content(&self) -> &GOATMessageContent { + &self.content + } + + pub async fn serialize_message(&self) -> Result> { + let cloned = self.clone(); + tokio::task::spawn_blocking(move || { + if matches!(&cloned.content, GOATMessageContent::GenCircuits(_)) { + let mut encoded = bincode::serialize(&cloned) + .context("failed to serialize bincode GOATMessage")?; + let mut message = Vec::with_capacity(GOAT_MESSAGE_BIN_PREFIX.len() + encoded.len()); + message.extend_from_slice(GOAT_MESSAGE_BIN_PREFIX); + message.append(&mut encoded); + Ok(message) + } else { + serde_json::to_vec(&cloned).context("failed to serialize JSON GOATMessage") + } + }) + .await? + } + + /// Whether `message` uses the bincode envelope, which only `GenCircuits` does. + pub fn is_binary_envelope(message: &[u8]) -> bool { + message.starts_with(GOAT_MESSAGE_BIN_PREFIX) + } + + pub async fn deserialize_message(message: &[u8]) -> Result { + let cloned = message.to_vec(); + tokio::task::spawn_blocking(move || { + if let Some(encoded) = cloned.strip_prefix(GOAT_MESSAGE_BIN_PREFIX) { + bincode::deserialize(encoded).context("failed to deserialize bincode GOATMessage") + } else { + serde_json::from_slice(&cloned).context("failed to deserialize JSON GOATMessage") + } + }) + .await? + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn soldering_proof_ready_is_descriptor_only() { + let ready = SolderingProofReady { + instance_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), + graph_id: Uuid::parse_str("22222222-2222-2222-2222-222222222222").unwrap(), + candidate_index: 3, + payload_hash: [0xabu8; 32], + total_len: 1024, + }; + + let value = serde_json::to_value(ready).unwrap(); + let object = value.as_object().unwrap(); + + assert!(object.contains_key("instance_id")); + assert!(object.contains_key("graph_id")); + assert!(object.contains_key("candidate_index")); + assert!(object.contains_key("payload_hash")); + assert!(object.contains_key("total_len")); + assert!(!object.contains_key("payload_path")); + assert!(!object.contains_key("payload")); + assert!(!object.contains_key("setup_package")); + assert!(!object.contains_key("verifier_pubkey")); + } + + #[test] + fn every_wire_kind_is_routed_or_immediate() { + use strum::IntoEnumIterator; + + let roles = [Actor::Committee, Actor::Operator, Actor::Verifier, Actor::Watchtower]; + for kind in MessageKind::iter() { + let handled = roles.iter().any(|role| kind.handled_by(role)); + assert!(handled, "{kind} is kept by no role: it would only ever be relayed"); + } + } +} diff --git a/node/src/metrics_service.rs b/node/src/metrics_service.rs index 319c3eaa9..7fbfa31da 100644 --- a/node/src/metrics_service.rs +++ b/node/src/metrics_service.rs @@ -1,9 +1,11 @@ +use std::collections::HashMap; use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; -use crate::rpc_service::{AppState, current_time_secs}; +use crate::clock::current_time_secs; +use crate::rpc_service::AppState; use alloy::primitives::U256; use axum::extract::{MatchedPath, Request, State}; use axum::http::{HeaderMap, StatusCode}; @@ -15,7 +17,7 @@ use prometheus_client::metrics::counter::Counter; use prometheus_client::metrics::family::Family; use prometheus_client::metrics::gauge::Gauge; use prometheus_client::metrics::histogram::{Histogram, exponential_buckets}; -use store::{GraphStatus, InstanceBridgeInStatus, MessageState, SwapEscrowStatus}; +use store::{GraphStatus, InstanceBridgeInStatus, SwapEscrowStatus}; use tokio::time::Instant; const METRICS_CONTENT_TYPE: &str = "application/openmetrics-text;charset=utf-8;version=1.0.0"; @@ -34,6 +36,23 @@ pub fn node_metrics_state() -> Option<&'static MetricsState> { NODE_METRICS_STATE.get() } +/// Backend availability hooks for code that only has the process-wide handle. +pub fn record_backend_ok(backend: &'static str) { + if let Some(state) = node_metrics_state() { + state.record_backend_call(backend, true, None); + } +} + +pub fn record_backend_failure( + backend: &'static str, + operation: &'static str, + error: &dyn std::fmt::Display, +) { + if let Some(state) = node_metrics_state() { + state.backend_probe_failed(backend, operation, error); + } +} + /// Creates a duration histogram using the shared HTTP and task latency buckets. fn duration_histogram() -> Histogram { Histogram::new(exponential_buckets(0.005, 2.0, 15)) @@ -79,6 +98,11 @@ struct EventWatchStateLabels { state: String, } +#[derive(Debug, Clone, Hash, PartialEq, Eq, prometheus_client::encoding::EncodeLabelSet)] +struct ReasonLabels { + reason: String, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum EventWatchState { Healthy, @@ -106,6 +130,59 @@ struct ReadinessState { event_watcher_ready: AtomicBool, } +/// Readiness bits as served by `/v1/health`. +#[derive(Debug, Clone, serde::Serialize)] +pub struct ReadinessSnapshot { + pub ready: bool, + pub startup: bool, + pub database: bool, + pub backend: bool, + pub event_watcher: bool, +} + +/// The backends whose availability is tracked as one signal each. +pub const BACKENDS: [&str; 4] = ["btc", "goat", "graph", "proof"]; +/// The backends the node needs to be ready: probed continuously, unlike the +/// proof builder, which is only called while a proof is being fetched. +const READINESS_BACKENDS: [&str; 3] = ["btc", "goat", "graph"]; +/// Consecutive failed calls before a backend is declared unavailable. +const BACKEND_UNAVAILABLE_AFTER_FAILURES: u32 = 2; +/// Seconds without any failed call before an unavailable backend counts as +/// recovered, so a flaky backend yields one outage instead of a transition per call. +const BACKEND_RECOVERY_QUIET_SECS: i64 = 30; + +#[derive(Debug, Clone, Hash, PartialEq, Eq, prometheus_client::encoding::EncodeLabelSet)] +struct BackendLabels { + backend: String, +} + +#[derive(Debug, Default)] +struct BackendHealth { + consecutive_failures: u32, + failed_calls: u64, + unavailable_since: Option, + last_failure_at: Option, + last_error: Option, +} + +/// One backend's availability as served by `/v1/health`. +#[derive(Debug, Clone, serde::Serialize)] +pub struct BackendHealthSnapshot { + pub backend: &'static str, + pub up: bool, + pub unavailable_since: Option, + pub consecutive_failures: u32, + pub last_error: Option, +} + +/// Event watcher state as served by `/v1/health`. +#[derive(Debug, Clone, serde::Serialize)] +pub struct EventWatchSnapshot { + pub state: &'static str, + pub lag_blocks: i64, + pub last_success_timestamp: i64, +} + #[derive(Debug, Clone, Hash, PartialEq, Eq, prometheus_client::encoding::EncodeLabelSet)] struct InstanceLabels { flow: String, @@ -129,10 +206,6 @@ pub struct MetricsState { message_dispatch_total: Family, instances: Family, graphs: Family, - messages: Family, - oldest_pending_message_age_seconds: Gauge, - p2p_inbox_messages: Family, - oldest_pending_p2p_inbox_age_seconds: Gauge, ready: Gauge, db_busy_retries_total: Counter, db_errors_total: Counter, @@ -143,7 +216,6 @@ pub struct MetricsState { fee_wallet_balance_sats: Gauge, fee_wallet_spendable_utxos: Gauge, graph_validation_total: Family, - message_retry_total: Counter, p2p_publish_total: Family, p2p_receive_total: Family, p2p_oversized_messages_total: Counter, @@ -158,6 +230,7 @@ pub struct MetricsState { pegin_graph_setup_total: Family, pegin_confirm_total: Family, pegin_post_total: Family, + instance_alerts_total: Family, pegout_disprove_total: Counter, withdraw_finalize_total: Family, goat_gas_balance_wei: Gauge, @@ -165,6 +238,13 @@ pub struct MetricsState { soldering_payload_io_total: Family, btc_tx_broadcast_total: Family, event_watch_lag_blocks: Gauge, + backend_up: Family, + backend_unavailable_since_seconds: Family, + backend_health: Arc>>, + /// No readiness backend is currently declared unavailable. + backends_available: Arc, + event_watch_state_current: Arc, + started_at: Instant, readiness: Arc, peg_btc_decimals: Arc, } @@ -191,11 +271,15 @@ impl MetricsState { let message_dispatch_total = Family::default(); let instances = Family::default(); let graphs = Family::default(); - let messages = Family::default(); - let oldest_pending_message_age_seconds = Gauge::default(); - let p2p_inbox_messages = Family::default(); - let oldest_pending_p2p_inbox_age_seconds = Gauge::default(); let ready = Gauge::default(); + let backend_up: Family = Family::default(); + let backend_unavailable_since_seconds: Family = Family::default(); + for backend in BACKENDS { + backend_up.get_or_create(&BackendLabels { backend: backend.to_owned() }).set(1); + backend_unavailable_since_seconds + .get_or_create(&BackendLabels { backend: backend.to_owned() }) + .set(0); + } let db_busy_retries_total = Counter::default(); let db_errors_total = Counter::default(); let pegin_oldest_active_age_seconds = Gauge::default(); @@ -205,7 +289,6 @@ impl MetricsState { let fee_wallet_balance_sats = Gauge::default(); let fee_wallet_spendable_utxos = Gauge::default(); let graph_validation_total = Family::default(); - let message_retry_total = Counter::default(); let p2p_publish_total = Family::default(); let p2p_receive_total = Family::default(); let p2p_oversized_messages_total = Counter::default(); @@ -225,6 +308,7 @@ impl MetricsState { let pegin_graph_setup_total = Family::default(); let pegin_confirm_total = Family::default(); let pegin_post_total = Family::default(); + let instance_alerts_total = Family::default(); let pegout_disprove_total = Counter::default(); let withdraw_finalize_total = Family::default(); let goat_gas_balance_wei = Gauge::default(); @@ -269,7 +353,7 @@ impl MetricsState { ); registry.register( "bitvm_node_message_dispatch", - "Total number of protocol message dispatches", + "Total number of control message dispatches (NodeInfo, GraphSetupAck)", message_dispatch_total.clone(), ); registry.register( @@ -278,26 +362,6 @@ impl MetricsState { instances.clone(), ); registry.register("bitvm_node_graphs", "Number of graphs by status", graphs.clone()); - registry.register( - "bitvm_node_messages", - "Number of queued messages by state", - messages.clone(), - ); - registry.register( - "bitvm_node_oldest_pending_message_age_seconds", - "Age in seconds of the oldest pending message", - oldest_pending_message_age_seconds.clone(), - ); - registry.register( - "bitvm_node_p2p_inbox_messages", - "Number of durable P2P inbox messages by state", - p2p_inbox_messages.clone(), - ); - registry.register( - "bitvm_node_oldest_pending_p2p_inbox_age_seconds", - "Age in seconds of the oldest pending durable P2P inbox message", - oldest_pending_p2p_inbox_age_seconds.clone(), - ); registry.register( "bitvm_node_ready", "Whether the node is ready to process work", @@ -348,11 +412,6 @@ impl MetricsState { "Total graph validation outcomes", graph_validation_total.clone(), ); - registry.register( - "bitvm_node_message_retry", - "Total deferred protocol messages", - message_retry_total.clone(), - ); registry.register( "bitvm_node_p2p_publish", "Total P2P message publish outcomes", @@ -388,6 +447,16 @@ impl MetricsState { "Unix timestamp of the last successful Goat backend health probe", goat_backend_last_success_timestamp_seconds.clone(), ); + registry.register( + "bitvm_node_backend_up", + "Whether the backend answered its latest calls (0 after consecutive failures)", + backend_up.clone(), + ); + registry.register( + "bitvm_node_backend_unavailable_since_seconds", + "Unix timestamp since when the backend has been unavailable (0 when up)", + backend_unavailable_since_seconds.clone(), + ); registry.register( "bitvm_node_spv_lag_blocks", "BTC tip minus Goat SPV height", @@ -423,6 +492,11 @@ impl MetricsState { "Total postPeginData outcomes", pegin_post_total.clone(), ); + registry.register( + "bitvm_node_instance_alerts", + "Total bridge-in instance conditions that need manual handling, by reason", + instance_alerts_total.clone(), + ); registry.register( "bitvm_node_pegout_disprove", "Total confirmed Disprove terminal events", @@ -471,10 +545,6 @@ impl MetricsState { message_dispatch_total, instances, graphs, - messages, - oldest_pending_message_age_seconds, - p2p_inbox_messages, - oldest_pending_p2p_inbox_age_seconds, ready, db_busy_retries_total, db_errors_total, @@ -485,7 +555,6 @@ impl MetricsState { fee_wallet_balance_sats, fee_wallet_spendable_utxos, graph_validation_total, - message_retry_total, p2p_publish_total, p2p_receive_total, p2p_oversized_messages_total, @@ -500,6 +569,7 @@ impl MetricsState { pegin_graph_setup_total, pegin_confirm_total, pegin_post_total, + instance_alerts_total, pegout_disprove_total, withdraw_finalize_total, goat_gas_balance_wei, @@ -507,11 +577,170 @@ impl MetricsState { soldering_payload_io_total, btc_tx_broadcast_total, event_watch_lag_blocks, + backend_up, + backend_unavailable_since_seconds, + backend_health: Arc::new(Mutex::new(HashMap::new())), + backends_available: Arc::new(AtomicBool::new(true)), + event_watch_state_current: Arc::new(AtomicU8::new(0)), + started_at: Instant::now(), readiness, peg_btc_decimals, } } + /// Records one call to a backend and logs only the transitions: one WARN + /// when it becomes unavailable, one INFO when it answers again. + pub fn record_backend_call(&self, backend: &'static str, success: bool, error: Option<&str>) { + self.record_backend_call_at(backend, success, error, current_time_secs()); + } + + fn record_backend_call_at( + &self, + backend: &'static str, + success: bool, + error: Option<&str>, + now: i64, + ) { + let labels = BackendLabels { backend: backend.to_owned() }; + let mut health = + self.backend_health.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let entry = health.entry(backend).or_default(); + if success { + entry.consecutive_failures = 0; + let quiet_for = entry.last_failure_at.map_or(i64::MAX, |at| now.saturating_sub(at)); + if entry.unavailable_since.is_some() && quiet_for >= BACKEND_RECOVERY_QUIET_SECS { + let since = entry.unavailable_since.take().unwrap_or(now); + tracing::info!( + event = "backend_availability", + backend, + outcome = "recovered", + downtime_secs = now.saturating_sub(since), + failed_calls = entry.failed_calls, + "backend answers again" + ); + self.backend_up.get_or_create(&labels).set(1); + self.backend_unavailable_since_seconds.get_or_create(&labels).set(0); + entry.failed_calls = 0; + entry.last_error = None; + } + } else { + entry.consecutive_failures += 1; + entry.failed_calls += 1; + entry.last_failure_at = Some(now); + entry.last_error = error.map(str::to_owned); + if entry.unavailable_since.is_none() + && entry.consecutive_failures >= BACKEND_UNAVAILABLE_AFTER_FAILURES + { + entry.unavailable_since = Some(now); + tracing::warn!( + event = "backend_availability", + backend, + outcome = "unavailable", + consecutive_failures = entry.consecutive_failures, + error = error.unwrap_or("unknown"), + "backend unavailable; further failures are logged at debug until it recovers" + ); + self.backend_up.get_or_create(&labels).set(0); + self.backend_unavailable_since_seconds.get_or_create(&labels).set(now); + } + } + let available = READINESS_BACKENDS + .iter() + .all(|name| health.get(name).is_none_or(|h| h.unavailable_since.is_none())); + drop(health); + self.backends_available.store(available, Ordering::Relaxed); + self.refresh_ready(); + } + + /// Whether the backend is currently declared unavailable. + pub fn backend_unavailable(&self, backend: &str) -> bool { + self.backend_health + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(backend) + .is_some_and(|health| health.unavailable_since.is_some()) + } + + /// Records a failed backend call: WARN while the backend still counts as + /// up, debug once it is known to be unavailable. + pub fn backend_probe_failed( + &self, + backend: &'static str, + operation: &'static str, + error: &dyn std::fmt::Display, + ) { + let known_down = self.backend_unavailable(backend); + match backend { + "btc" => self + .btc_backend_requests_total + .get_or_create(&OutcomeLabels { outcome: outcome(false) }) + .inc(), + "goat" => self + .goat_backend_requests_total + .get_or_create(&OutcomeLabels { outcome: outcome(false) }) + .inc(), + _ => 0, + }; + let message = error.to_string(); + self.record_backend_call(backend, false, Some(&message)); + if known_down { + tracing::debug!(event = "metrics_backend_probe", backend, operation, error = %message, "backend probe failed (backend already unavailable)"); + } else { + tracing::warn!(event = "metrics_backend_probe", backend, operation, error = %message, "backend probe failed"); + } + } + + pub fn backend_health_snapshot(&self) -> Vec { + let health = self.backend_health.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + BACKENDS + .iter() + .map(|backend| { + let entry = health.get(backend); + BackendHealthSnapshot { + backend, + up: entry.is_none_or(|h| h.unavailable_since.is_none()), + unavailable_since: entry.and_then(|h| h.unavailable_since), + consecutive_failures: entry.map_or(0, |h| h.consecutive_failures), + last_error: entry.and_then(|h| h.last_error.clone()), + } + }) + .collect() + } + + pub fn readiness_snapshot(&self) -> ReadinessSnapshot { + let startup = self.readiness.startup_ready.load(Ordering::Relaxed); + let database = self.readiness.database_ready.load(Ordering::Relaxed); + // The periodic probe (every 6th tick) and the per-call availability + // states both have to agree that the backends answer. + let backend = self.readiness.backend_ready.load(Ordering::Relaxed) + && self.backends_available.load(Ordering::Relaxed); + let event_watcher = self.readiness.event_watcher_ready.load(Ordering::Relaxed); + ReadinessSnapshot { + ready: startup && database && backend && event_watcher, + startup, + database, + backend, + event_watcher, + } + } + + pub fn event_watch_snapshot(&self) -> EventWatchSnapshot { + let index = usize::from(self.event_watch_state_current.load(Ordering::Relaxed)); + EventWatchSnapshot { + state: EventWatchState::ALL.get(index).map_or("unknown", |state| state.as_str()), + lag_blocks: self.event_watch_lag_blocks.get(), + last_success_timestamp: self.event_watch_last_success_timestamp_seconds.get(), + } + } + + pub fn spv_lag_blocks(&self) -> i64 { + self.spv_lag_blocks.get() + } + + pub fn uptime_secs(&self) -> u64 { + self.started_at.elapsed().as_secs() + } + /// Records a task outcome and duration, updating its success timestamp when applicable. pub fn record_task_run(&self, task: &str, outcome: &str, duration: Duration) { self.task_runs_total @@ -565,6 +794,9 @@ impl MetricsState { .set(i64::from(candidate == state)); } + let index = + EventWatchState::ALL.iter().position(|candidate| *candidate == state).unwrap_or(0); + self.event_watch_state_current.store(index as u8, Ordering::Relaxed); let healthy = state == EventWatchState::Healthy; self.readiness.event_watcher_ready.store(healthy, Ordering::Relaxed); self.event_watch_ready.set(i64::from(healthy)); @@ -572,11 +804,7 @@ impl MetricsState { } fn refresh_ready(&self) { - let ready = self.readiness.startup_ready.load(Ordering::Relaxed) - && self.readiness.database_ready.load(Ordering::Relaxed) - && self.readiness.backend_ready.load(Ordering::Relaxed) - && self.readiness.event_watcher_ready.load(Ordering::Relaxed); - self.ready.set(i64::from(ready)); + self.ready.set(i64::from(self.readiness_snapshot().ready)); } pub fn record_db_error(&self, error: &anyhow::Error) { @@ -595,10 +823,6 @@ impl MetricsState { .inc(); } - pub fn record_message_retry(&self) { - self.message_retry_total.inc(); - } - pub fn record_p2p_publish(&self, success: bool) { self.p2p_publish_total.get_or_create(&OutcomeLabels { outcome: outcome(success) }).inc(); } @@ -618,6 +842,7 @@ impl MetricsState { if success { self.btc_backend_last_success_timestamp_seconds.set(current_time_secs()); } + self.record_backend_call("btc", success, None); } pub fn record_goat_backend_probe(&self, success: bool) { @@ -627,6 +852,7 @@ impl MetricsState { if success { self.goat_backend_last_success_timestamp_seconds.set(current_time_secs()); } + self.record_backend_call("goat", success, None); } pub fn set_peg_btc_decimals(&self, decimals: u8) { @@ -647,6 +873,11 @@ impl MetricsState { self.pegin_post_total.get_or_create(&OutcomeLabels { outcome: outcome(success) }).inc(); } + /// Record an instance alert, such as a late confirmation, rejected post or failed key deletion. + pub fn record_instance_alert(&self, reason: &str) { + self.instance_alerts_total.get_or_create(&ReasonLabels { reason: reason.to_owned() }).inc(); + } + pub fn record_pegout_disprove(&self) { self.pegout_disprove_total.inc(); } @@ -699,11 +930,6 @@ impl MetricsState { fn apply_database_metrics(&self, counts: &[store::MetricsStateCount]) { self.instances.clear(); self.graphs.clear(); - self.messages.clear(); - self.oldest_pending_message_age_seconds.set(0); - self.p2p_inbox_messages.clear(); - self.oldest_pending_p2p_inbox_age_seconds.set(0); - let now = current_time_secs(); for count in counts { match count.category.as_str() { @@ -723,32 +949,6 @@ impl MetricsState { let status = known_status::(&count.state); self.graphs.get_or_create(&StatusLabels { status }).inc_by(count.count); } - "message" => { - let status = known_status::(&count.state); - self.messages - .get_or_create(&StatusLabels { status: status.clone() }) - .inc_by(count.count); - if status == "Pending" { - self.oldest_pending_message_age_seconds.set( - count - .oldest_created_at - .map_or(0, |created_at| now.saturating_sub(created_at).max(0)), - ); - } - } - "p2p_inbox" => { - let status = known_status::(&count.state); - self.p2p_inbox_messages - .get_or_create(&StatusLabels { status: status.clone() }) - .inc_by(count.count); - if status == "Pending" { - self.oldest_pending_p2p_inbox_age_seconds.set( - count - .oldest_created_at - .map_or(0, |created_at| now.saturating_sub(created_at).max(0)), - ); - } - } _ => {} } } @@ -907,8 +1107,8 @@ mod tests { last_success_at: None, }, store::MetricsStateCount { - category: "p2p_inbox".to_string(), - state: "Pending".to_string(), + category: "instance".to_string(), + state: "Presigned".to_string(), count: 4, oldest_created_at: Some(current_time_secs() - 30), last_success_at: None, @@ -923,8 +1123,7 @@ mod tests { assert!(!output.contains("bitvm_node_graphs{status=\"OperatorPresigned\"}")); assert!(!output.contains("unexpected-id-like-value")); assert!(!output.contains("another-unexpected-value")); - assert!(output.contains("bitvm_node_p2p_inbox_messages{status=\"Pending\"} 4")); - assert!(output.contains("bitvm_node_oldest_pending_p2p_inbox_age_seconds 30")); + assert!(output.contains("bitvm_node_instances{flow=\"bridge_in\",status=\"Presigned\"} 4")); } #[test] @@ -932,7 +1131,7 @@ mod tests { let state = MetricsState::new(Arc::new(Mutex::new(Registry::default()))); state.record_task_run("maintenance", "success", Duration::from_millis(25)); state.record_task_run("history_sync", "failed", Duration::from_millis(5)); - state.record_message_dispatch("Tick", "success"); + state.record_message_dispatch("RequestNodeInfo", "success"); let output = encoded(&state); assert!( @@ -940,7 +1139,7 @@ mod tests { .contains("bitvm_node_task_runs_total{task=\"maintenance\",outcome=\"success\"} 1") ); assert!(output.contains( - "bitvm_node_message_dispatch_total{message_type=\"Tick\",outcome=\"success\"} 1" + "bitvm_node_message_dispatch_total{message_type=\"RequestNodeInfo\",outcome=\"success\"} 1" )); assert!( !output @@ -985,4 +1184,51 @@ mod tests { assert!(output.contains("bitvm_node_event_watch_state{state=\"syncing\"} 0")); assert!(output.contains("bitvm_node_event_watch_state{state=\"failed\"} 1")); } + + #[test] + fn backend_availability_transitions_after_consecutive_failures() { + let state = MetricsState::new(Arc::new(Mutex::new(Registry::default()))); + state.record_backend_call_at("goat", false, Some("timeout"), 1_000); + assert!(!state.backend_unavailable("goat"), "one failure is not an outage"); + state.record_backend_call_at("goat", false, Some("timeout"), 1_005); + assert!(state.backend_unavailable("goat")); + let snapshot = state.backend_health_snapshot(); + let goat = snapshot.iter().find(|b| b.backend == "goat").unwrap(); + assert!(!goat.up); + assert_eq!(goat.unavailable_since, Some(1_005)); + assert_eq!(goat.last_error.as_deref(), Some("timeout")); + // a success right after a failure is a flap, not a recovery + state.record_backend_call_at("goat", true, None, 1_010); + assert!(state.backend_unavailable("goat")); + state.record_backend_call_at("goat", false, Some("502"), 1_015); + // recovered once no call failed for the quiet window + state.record_backend_call_at("goat", true, None, 1_015 + BACKEND_RECOVERY_QUIET_SECS); + assert!(!state.backend_unavailable("goat")); + let goat = + state.backend_health_snapshot().into_iter().find(|b| b.backend == "goat").unwrap(); + assert!(goat.up && goat.unavailable_since.is_none() && goat.consecutive_failures == 0); + assert!(goat.last_error.is_none()); + // a single failure after recovery keeps it up + state.record_backend_call_at("goat", false, None, 2_000); + assert!(!state.backend_unavailable("goat")); + } + + #[test] + fn readiness_follows_backend_availability() { + let state = MetricsState::new(Arc::new(Mutex::new(Registry::default()))); + state.mark_startup_ready(); + state.mark_database_ready(true); + state.mark_backend_ready(true); + state.set_event_watch_state(EventWatchState::Healthy); + assert!(state.readiness_snapshot().ready); + state.record_backend_call_at("graph", false, Some("down"), 100); + state.record_backend_call_at("graph", false, Some("down"), 105); + assert!(!state.readiness_snapshot().ready, "an unavailable subgraph is not ready"); + assert!(encoded(&state).contains("bitvm_node_ready 0")); + // the proof builder is informational only + state.record_backend_call_at("graph", true, None, 200); + state.record_backend_call_at("proof", false, Some("down"), 200); + state.record_backend_call_at("proof", false, Some("down"), 205); + assert!(state.readiness_snapshot().ready); + } } diff --git a/node/src/middleware/behaviour.rs b/node/src/middleware/behaviour.rs index e130bfc93..de1d206e5 100644 --- a/node/src/middleware/behaviour.rs +++ b/node/src/middleware/behaviour.rs @@ -37,6 +37,8 @@ pub struct AllBehaviours { pub kademlia: kad::Behaviour, //pub mdns: mdns::tokio::Behaviour, pub gossipsub: gossipsub::Behaviour, + /// Direct graph definition pulls. + pub get_graph: super::get_graph::GetGraphBehaviour, } impl AllBehaviours { pub fn new(key: &Keypair) -> Self { @@ -81,7 +83,8 @@ impl AllBehaviours { crate::env::get_p2p_inbound_registered_reserve() as usize, crate::env::get_p2p_reserved_peers(), ); - Self { connection_gate, connection_limits, kademlia, gossipsub } + let get_graph = super::get_graph::new_get_graph_behaviour(); + Self { connection_gate, connection_limits, kademlia, gossipsub, get_graph } } } diff --git a/node/src/middleware/get_graph.rs b/node/src/middleware/get_graph.rs new file mode 100644 index 000000000..0ddaffe41 --- /dev/null +++ b/node/src/middleware/get_graph.rs @@ -0,0 +1,157 @@ +//! Direct request-response protocol for fetching a graph definition. +//! +//! Responses use the request's stream and authenticated peer connection. +//! Definition validation is handled by the requester. + +use async_trait::async_trait; +use futures::prelude::*; +use libp2p::request_response::{self, ProtocolSupport}; +use libp2p::swarm::StreamProtocol; +use serde::{Deserialize, Serialize}; +use std::io; +use std::time::Duration; +use uuid::Uuid; + +pub type GetGraphBehaviour = request_response::Behaviour; + +/// JSON on the stream, with limits large enough for a full graph definition. +#[derive(Clone, Debug, Default)] +pub struct GetGraphCodec; + +#[async_trait] +impl request_response::Codec for GetGraphCodec { + type Protocol = StreamProtocol; + type Request = GetGraphRequest; + type Response = GetGraphResponse; + + async fn read_request( + &mut self, + _: &Self::Protocol, + io: &mut T, + ) -> io::Result + where + T: AsyncRead + Unpin + Send, + { + let mut bytes = Vec::new(); + io.take(MAX_REQUEST_BYTES).read_to_end(&mut bytes).await?; + Ok(serde_json::from_slice(&bytes)?) + } + + async fn read_response( + &mut self, + _: &Self::Protocol, + io: &mut T, + ) -> io::Result + where + T: AsyncRead + Unpin + Send, + { + let mut bytes = Vec::new(); + io.take(MAX_RESPONSE_BYTES).read_to_end(&mut bytes).await?; + Ok(serde_json::from_slice(&bytes)?) + } + + async fn write_request( + &mut self, + _: &Self::Protocol, + io: &mut T, + request: GetGraphRequest, + ) -> io::Result<()> + where + T: AsyncWrite + Unpin + Send, + { + io.write_all(&serde_json::to_vec(&request)?).await?; + io.close().await + } + + async fn write_response( + &mut self, + _: &Self::Protocol, + io: &mut T, + response: GetGraphResponse, + ) -> io::Result<()> + where + T: AsyncWrite + Unpin + Send, + { + io.write_all(&serde_json::to_vec(&response)?).await?; + io.close().await + } +} + +/// A definition is addressed by graph id or by operator slot. `finalized` +/// asks for the copy with committee pre-signatures; a peer holding only the +/// operator's copy answers not found instead of sending a copy the requester +/// would reject after the transfer. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GetGraphRequest { + pub instance_id: Option, + pub graph_id: Option, + pub operator_pubkey: Option, + pub graph_nonce: Option, + #[serde(default)] + pub finalized: bool, +} + +impl GetGraphRequest { + pub fn by_id(instance_id: Uuid, graph_id: Uuid, finalized: bool) -> Self { + Self { + instance_id: Some(instance_id), + graph_id: Some(graph_id), + operator_pubkey: None, + graph_nonce: None, + finalized, + } + } + + pub fn by_slot(operator_pubkey: String, graph_nonce: u64) -> Self { + Self { + instance_id: None, + graph_id: None, + operator_pubkey: Some(operator_pubkey), + graph_nonce: Some(graph_nonce), + finalized: false, + } + } +} + +/// The stored raw definition (simplified graph JSON) when the peer has it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GetGraphResponse { + pub instance_id: Option, + pub graph_id: Option, + pub graph: Option, +} + +impl GetGraphResponse { + pub fn not_found() -> Self { + Self::default() + } + + pub fn found(instance_id: Uuid, graph_id: Uuid, graph: String) -> Self { + Self { instance_id: Some(instance_id), graph_id: Some(graph_id), graph: Some(graph) } + } +} + +pub const GET_GRAPH_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); +const MAX_REQUEST_BYTES: u64 = 64 * 1024; +const MAX_RESPONSE_BYTES: u64 = 64 * 1024 * 1024; + +pub fn get_graph_protocol() -> StreamProtocol { + let protocol = crate::env::get_proto_base(); + StreamProtocol::try_from_owned(format!("/{protocol}/get-graph/2")) + .expect("valid get-graph protocol name") +} + +/// Open `GetGraph` streams per connection. Each may hold one serialized +/// definition until the peer reads it, so this is a memory bound against a +/// peer that requests and never reads. +pub const GET_GRAPH_MAX_CONCURRENT_STREAMS: usize = 4; + +pub fn new_get_graph_behaviour() -> GetGraphBehaviour { + request_response::Behaviour::with_codec( + GetGraphCodec, + [(get_graph_protocol(), ProtocolSupport::Full)], + request_response::Config::default() + .with_request_timeout(GET_GRAPH_REQUEST_TIMEOUT) + .with_max_concurrent_streams(GET_GRAPH_MAX_CONCURRENT_STREAMS), + ) +} diff --git a/node/src/middleware/mod.rs b/node/src/middleware/mod.rs index 7c7ec9480..90f4b9465 100644 --- a/node/src/middleware/mod.rs +++ b/node/src/middleware/mod.rs @@ -1,5 +1,6 @@ pub mod behaviour; pub mod connection_gate; +pub mod get_graph; pub mod publisher; pub mod swarm; diff --git a/node/src/middleware/publisher.rs b/node/src/middleware/publisher.rs index b4b454bf4..faa60b6bd 100644 --- a/node/src/middleware/publisher.rs +++ b/node/src/middleware/publisher.rs @@ -1,12 +1,18 @@ -//! Bounded, acknowledged access to the swarm from the business worker. +//! Bounded, acknowledged access to the swarm from tasks that do not own it: +//! gossip publishes and direct `GetGraph` requests. +use crate::messages::GOATMessage; +use anyhow::Result; use futures::future::BoxFuture; +use libp2p::request_response::{OutboundRequestId, ResponseChannel}; use libp2p::{ - Swarm, + PeerId, Swarm, gossipsub::{IdentTopic, MessageId}, }; +use std::collections::HashMap; use tokio::sync::{mpsc, oneshot}; use super::AllBehaviours; +use super::get_graph::{GET_GRAPH_REQUEST_TIMEOUT, GetGraphRequest, GetGraphResponse}; pub trait MessagePublisher: Send { fn publish_message( @@ -38,18 +44,50 @@ impl MessagePublisher for super::swarm::BitvmSwarmWrapper { } } -pub struct PublishCommand { - topic: IdentTopic, - data: Vec, - result: oneshot::Sender>, +/// Replies the swarm loop still owes to `GetGraph` requesters. +pub type PendingGraphRequests = + HashMap>>; + +pub enum PublishCommand { + Gossip { + topic: IdentTopic, + data: Vec, + result: oneshot::Sender>, + }, + GetGraph { + peer: PeerId, + request: GetGraphRequest, + result: oneshot::Sender>, + }, + /// The answer to an inbound `GetGraph`, produced off the network loop; + /// only the swarm can use the response channel. + GetGraphResponse { + channel: ResponseChannel, + response: GetGraphResponse, + }, } impl PublishCommand { - pub fn execute(self, swarm: &mut Swarm) { - // A timed-out/cancelled caller must not leave a delayed broadcast behind. - if !self.result.is_closed() { - let result = swarm.behaviour_mut().gossipsub.publish(self.topic, self.data); - let _ = self.result.send(result.map_err(Into::into)); + pub fn execute(self, swarm: &mut Swarm, pending: &mut PendingGraphRequests) { + match self { + PublishCommand::Gossip { topic, data, result } => { + // A timed-out/cancelled caller must not leave a delayed broadcast behind. + if !result.is_closed() { + let outcome = swarm.behaviour_mut().gossipsub.publish(topic, data); + let _ = result.send(outcome.map_err(Into::into)); + } + } + PublishCommand::GetGraph { peer, request, result } => { + if result.is_closed() { + return; + } + let request_id = swarm.behaviour_mut().get_graph.send_request(&peer, request); + pending.insert(request_id, result); + } + PublishCommand::GetGraphResponse { channel, response } => { + // The requester may have given up meanwhile; nothing to do then. + let _ = swarm.behaviour_mut().get_graph.send_response(channel, response); + } } } } @@ -65,6 +103,42 @@ pub fn channel() -> (NetworkPublisher, mpsc::Receiver) { (NetworkPublisher(tx), rx) } +impl NetworkPublisher { + /// Hand the swarm the answer to an inbound `GetGraph` request. + pub async fn respond_graph( + &self, + channel: ResponseChannel, + response: GetGraphResponse, + ) -> anyhow::Result<()> { + let send = self.0.send(PublishCommand::GetGraphResponse { channel, response }); + tokio::time::timeout(PUBLISH_TIMEOUT, send) + .await + .map_err(|_| { + anyhow::anyhow!("network task did not take the GetGraph response in time") + })? + .map_err(|_| anyhow::anyhow!("network publisher stopped")) + } + + /// Ask `peer` for a graph definition over the direct protocol. + pub async fn request_graph( + &self, + peer: PeerId, + request: GetGraphRequest, + ) -> anyhow::Result { + let (tx, rx) = oneshot::channel(); + let send = async { + self.0 + .send(PublishCommand::GetGraph { peer, request, result: tx }) + .await + .map_err(|_| anyhow::anyhow!("network publisher stopped"))?; + rx.await.map_err(|_| anyhow::anyhow!("network publisher stopped before answering"))? + }; + tokio::time::timeout(GET_GRAPH_REQUEST_TIMEOUT + PUBLISH_TIMEOUT, send) + .await + .map_err(|_| anyhow::anyhow!("GetGraph request to {peer} timed out"))? + } +} + impl MessagePublisher for NetworkPublisher { fn publish_message( &mut self, @@ -75,7 +149,7 @@ impl MessagePublisher for NetworkPublisher { let (tx, rx) = oneshot::channel(); let publish = async { self.0 - .send(PublishCommand { topic, data, result: tx }) + .send(PublishCommand::Gossip { topic, data, result: tx }) .await .map_err(|_| anyhow::anyhow!("network publisher stopped"))?; rx.await @@ -88,6 +162,82 @@ impl MessagePublisher for NetworkPublisher { } } +/// Receivers drop what exceeds their envelope limit without telling anyone, +/// so an outgrown message is reported loudly on the sending side. +pub(crate) fn note_oversized_payload(payload: &[u8], target: &str, message_type: &str) { + let limits = crate::p2p_admission::inbound_limits(); + let receiver_limit = if GOATMessage::is_binary_envelope(payload) { + limits.max_binary_bytes + } else { + limits.max_json_bytes + }; + if payload.len() > receiver_limit { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_p2p_oversized_message(); + } + tracing::error!( + event = "p2p_message_publish", + outcome = "oversized", + target, + message_type, + content_size = payload.len(), + receiver_limit, + "outbound protocol message exceeds the inbound size limit peers enforce" + ); + } +} + +/// Publish one message on its role topic; every sender goes through here. +pub async fn publish_to_role( + swarm: &mut dyn MessagePublisher, + message: GOATMessage, +) -> Result { + let target_actor = message.actor.to_string(); + let message_type = message.content.event_type(); + let topic = crate::middleware::get_topic_name(&target_actor); + let gossipsub_topic = IdentTopic::new(topic); + let serialized = match message.serialize_message().await { + Ok(serialized) => serialized, + Err(error) => { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_p2p_publish(false); + } + return Err(error); + } + }; + note_oversized_payload(&serialized, &target_actor, message_type); + match swarm.publish_message(gossipsub_topic, serialized).await { + Ok(message_id) => { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_p2p_publish(true); + } + tracing::info!( + event = "p2p_message_publish", + outcome = "published", + target_actor, + message_type, + message_id = ?message_id, + "published protocol message" + ); + Ok(message_id) + } + Err(err) => { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_p2p_publish(false); + } + tracing::warn!( + event = "p2p_message_publish", + outcome = "failed", + target_actor, + message_type, + error = %err, + "failed to publish protocol message" + ); + Err(anyhow::anyhow!("publish {message_type} to {target_actor}: {err}")) + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -100,7 +250,8 @@ mod tests { result = &mut send => panic!("must wait for network: {result:?}"), command = commands.recv() => command.unwrap(), }; - command.result.send(Err(anyhow::anyhow!("not published"))).unwrap(); + let PublishCommand::Gossip { result, .. } = command else { panic!("gossip command") }; + result.send(Err(anyhow::anyhow!("not published"))).unwrap(); assert!(send.await.unwrap_err().to_string().contains("not published")); let mut send = publisher.publish_message(IdentTopic::new("test"), vec![2]); @@ -109,6 +260,7 @@ mod tests { command = commands.recv() => command.unwrap(), }; drop(send); - assert!(command.result.is_closed()); + let PublishCommand::Gossip { result, .. } = command else { panic!("gossip command") }; + assert!(result.is_closed()); } } diff --git a/node/src/middleware/swarm.rs b/node/src/middleware/swarm.rs index 9d325dec0..cdc400b5f 100644 --- a/node/src/middleware/swarm.rs +++ b/node/src/middleware/swarm.rs @@ -1,4 +1,6 @@ use crate::middleware::behaviour::AllBehavioursEvent; +use crate::middleware::get_graph::{GetGraphRequest, GetGraphResponse}; +use crate::middleware::publisher::PendingGraphRequests; use crate::middleware::{AllBehaviours, split_topic_name}; use crate::{env, middleware}; use anyhow::bail; @@ -7,14 +9,17 @@ use bitvm_lib::actors::Actor; use futures::StreamExt; use libp2p::gossipsub::MessageId; use libp2p::multiaddr::Protocol; +use libp2p::request_response; +use libp2p::request_response::ResponseChannel; use libp2p::swarm::SwarmEvent; +use libp2p::swarm::dial_opts::{DialOpts, PeerCondition}; use libp2p::{Multiaddr, PeerId, Swarm, gossipsub, kad, noise, tcp, yamux}; use prometheus_client::metrics::gauge::Gauge; use prometheus_client::registry::Registry; use std::collections::HashMap; use std::net::Ipv4Addr; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::select; use tokio::time::interval; use tokio_util::sync::CancellationToken; @@ -103,6 +108,19 @@ pub trait P2pMessageHandler { async fn graceful_shutdown(&self) -> anyhow::Result<()> { Ok(()) } + + /// Take a direct `GetGraph` request off the network loop: admit it and + /// answer later through `publisher`, or hand the channel back for an + /// immediate refusal. The default refuses. + fn serve_graph_request( + &self, + _peer: PeerId, + _request: GetGraphRequest, + channel: ResponseChannel, + _publisher: super::publisher::NetworkPublisher, + ) -> Option> { + Some(channel) + } } #[derive(Clone, Debug)] @@ -115,6 +133,14 @@ pub struct BitvmSwarmConfig { pub regular_task_interval: u64, } +/// A bootnode whose last connection closed is re-dialled after this delay, then +/// every `BOOTNODE_REDIAL_RETRY` until it is connected again: libp2p does not +/// reconnect on its own and the periodic Kademlia bootstrap only runs every +/// five minutes. +const BOOTNODE_REDIAL_DELAY: Duration = Duration::from_secs(5); +const BOOTNODE_REDIAL_RETRY: Duration = Duration::from_secs(20); +const BOOTNODE_REDIAL_TICK: Duration = Duration::from_secs(5); + pub struct BitvmNetworkManager { config: BitvmSwarmConfig, peer_id: PeerId, @@ -122,6 +148,10 @@ pub struct BitvmNetworkManager { connected_peers: Gauge, required_topics_healthy: Gauge, publish_commands: tokio::sync::mpsc::Receiver, + pending_graph_requests: PendingGraphRequests, + bootnodes: Vec<(PeerId, Multiaddr)>, + /// Disconnected bootnodes and when to dial them next. + bootnode_redials: HashMap, } impl BitvmNetworkManager { pub fn new( @@ -155,9 +185,11 @@ impl BitvmNetworkManager { .build(); debug!("bootnodes: {:?}", config.bootnodes); + let mut bootnodes = Vec::with_capacity(config.bootnodes.len()); for peer in &config.bootnodes { let (peer_id, multi_addr) = parse_boot_node_str(peer)?; - swarm.behaviour_mut().kademlia.add_address(&peer_id, multi_addr); + swarm.behaviour_mut().kademlia.add_address(&peer_id, multi_addr.clone()); + bootnodes.push((peer_id, multi_addr)); } let (publisher, publish_commands) = super::publisher::channel(); Ok(BitvmNetworkManager { @@ -167,13 +199,52 @@ impl BitvmNetworkManager { connected_peers, required_topics_healthy, publish_commands, + pending_graph_requests: PendingGraphRequests::new(), + bootnodes, + bootnode_redials: HashMap::new(), }) } + fn is_bootnode(&self, peer_id: &PeerId) -> bool { + self.bootnodes.iter().any(|(id, _)| id == peer_id) + } + + /// Dials every bootnode whose redial is due and still disconnected. + fn redial_bootnodes(&mut self) { + let now = Instant::now(); + for (peer_id, address) in self.bootnodes.clone() { + if self.swarm.is_connected(&peer_id) { + self.bootnode_redials.remove(&peer_id); + continue; + } + if self.bootnode_redials.get(&peer_id).is_none_or(|due| *due > now) { + continue; + } + let opts = DialOpts::peer_id(peer_id) + .addresses(vec![address]) + .condition(PeerCondition::Disconnected) + .build(); + match self.swarm.dial(opts) { + Ok(()) => info!( + event = "p2p_bootnode_redial", + peer_id = %peer_id, + "re-dialling a disconnected bootnode" + ), + Err(error) => debug!(%peer_id, %error, "bootnode redial not started"), + } + self.bootnode_redials.insert(peer_id, now + BOOTNODE_REDIAL_RETRY); + } + } + pub fn get_peer_id_string(&self) -> String { self.peer_id.to_string() } + /// Bounded publisher handle for tasks that send through the swarm. + pub fn publisher(&self) -> super::publisher::NetworkPublisher { + self.swarm.publisher() + } + fn refresh_required_topics_health(&self, actor: &Actor) { let role_topic = middleware::get_topic_name(&actor.to_string()); let all_topic = middleware::get_topic_name(&Actor::All.to_string()); @@ -205,15 +276,27 @@ impl BitvmNetworkManager { .collect::>(); self.refresh_required_topics_health(&actor); + // P2P listen interface is configurable (default 0.0.0.0 = unchanged behaviour). Setting + // P2P_LISTEN_IP=127.0.0.1 keeps the p2p port bound to loopback only (required when the node + // runs on a shared host); in that case we must NOT skip the loopback NewListenAddr below, + // otherwise the swarm task hangs forever waiting for a non-loopback listen address. + let listen_ip = std::env::var("P2P_LISTEN_IP") + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| "0.0.0.0".to_string()); + let allow_loopback = matches!(listen_ip.as_str(), "127.0.0.1" | "localhost"); if self.config.p2p_port > 0 { - self.swarm.listen_on(format!("/ip4/0.0.0.0/tcp/{}", self.config.p2p_port).parse()?)?; + self.swarm + .listen_on(format!("/ip4/{}/tcp/{}", listen_ip, self.config.p2p_port).parse()?)?; } else { - self.swarm.listen_on("/ip4/0.0.0.0/tcp/0".parse()?)?; + self.swarm.listen_on(format!("/ip4/{}/tcp/0", listen_ip).parse()?)?; } let address = loop { if let SwarmEvent::NewListenAddr { address, .. } = self.swarm.select_next_some().await { - if address.iter().any(|e| e == Protocol::Ip4(Ipv4Addr::LOCALHOST)) { + if !allow_loopback + && address.iter().any(|e| e == Protocol::Ip4(Ipv4Addr::LOCALHOST)) + { debug!("Ignoring localhost address to make sure the example works in Firefox"); continue; } @@ -226,10 +309,17 @@ impl BitvmNetworkManager { let mut interval = interval(Duration::from_secs(self.config.regular_task_interval)); // Use Delay ticks and coalesced worker notifications. interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut redial_interval = tokio::time::interval(BOOTNODE_REDIAL_TICK); + redial_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { select! { Some(command) = self.publish_commands.recv() => { - command.execute(&mut self.swarm); + command.execute(&mut self.swarm, &mut self.pending_graph_requests); + } + _ = redial_interval.tick() => { + if !self.bootnode_redials.is_empty() { + self.redial_bootnodes(); + } } _ = cancellation_token.cancelled() => { info!("Swarm received shutdown signal"); @@ -322,6 +412,32 @@ impl BitvmNetworkManager { SwarmEvent::Behaviour(AllBehavioursEvent::Gossipsub(gossipsub::Event::Unsubscribed { peer_id, topic})) => { debug!("unsubscribed: {:?}, {:?}", peer_id, topic); } + SwarmEvent::Behaviour(AllBehavioursEvent::GetGraph(request_response::Event::Message { peer, message, .. })) => { + match message { + request_response::Message::Request { request, channel, .. } => { + let publisher = self.swarm.publisher(); + if let Some(channel) = msg_handler.serve_graph_request(peer, request, channel, publisher) + && self.swarm.behaviour_mut().get_graph.send_response(channel, GetGraphResponse::not_found()).is_err() + { + debug!(%peer, "GetGraph requester went away before the refusal"); + } + } + request_response::Message::Response { request_id, response } => { + if let Some(reply) = self.pending_graph_requests.remove(&request_id) { + let _ = reply.send(Ok(response)); + } + } + } + } + SwarmEvent::Behaviour(AllBehavioursEvent::GetGraph(request_response::Event::OutboundFailure { peer, request_id, error, .. })) => { + if let Some(reply) = self.pending_graph_requests.remove(&request_id) { + let _ = reply.send(Err(anyhow::anyhow!("GetGraph to {peer} failed: {error}"))); + } + } + SwarmEvent::Behaviour(AllBehavioursEvent::GetGraph(request_response::Event::InboundFailure { peer, error, .. })) => { + debug!(%peer, %error, "GetGraph inbound request failed"); + } + SwarmEvent::Behaviour(AllBehavioursEvent::GetGraph(request_response::Event::ResponseSent { .. })) => {} SwarmEvent::Behaviour(AllBehavioursEvent::Kademlia(kad::Event::RoutingUpdated{ peer, addresses,..})) => { debug!("routing updated: {:?}, addresses:{:?}", peer, addresses); } @@ -346,10 +462,17 @@ impl BitvmNetworkManager { } SwarmEvent::ConnectionEstablished {peer_id, connection_id, endpoint, .. } => { self.connected_peers.set(self.swarm.connected_peers().count() as i64); + if self.bootnode_redials.remove(&peer_id).is_some() { + info!(event = "p2p_bootnode_connected", peer_id = %peer_id, "bootnode connection restored"); + } debug!("connected to {peer_id}: {connection_id}, endpoint: {:?}", endpoint); } - SwarmEvent::ConnectionClosed {peer_id, connection_id, .. } => { + SwarmEvent::ConnectionClosed {peer_id, connection_id, num_established, cause, .. } => { self.connected_peers.set(self.swarm.connected_peers().count() as i64); + if num_established == 0 && self.is_bootnode(&peer_id) { + self.bootnode_redials.insert(peer_id, Instant::now() + BOOTNODE_REDIAL_DELAY); + info!(event = "p2p_bootnode_disconnected", peer_id = %peer_id, cause = ?cause, "lost the last connection to a bootnode; redial scheduled"); + } debug!("disconnected from {peer_id}: {connection_id}"); } e => { diff --git a/node/src/mock_gc.rs b/node/src/mock_gc.rs new file mode 100644 index 000000000..8ec2e42ff --- /dev/null +++ b/node/src/mock_gc.rs @@ -0,0 +1,196 @@ +//! Artificial, env-configurable delays for the mocked GC operations (only compiled under the +//! `mock-gc` feature). +//! +//! Each mocked GC operation is fast in reality, so to make the localnet behave like the real +//! protocol (minutes of setup, seconds of dispute work) every mocked operation `await`s a delay +//! read from the environment **at call time**. The delay is always an async sleep placed in async +//! context, *before* the `spawn_blocking` that runs the (now trivial) mock, so it never holds a +//! blocking-pool thread and is cancel-safe at shutdown. +//! +//! Per-operation env var (seconds, integer or fractional), else `MOCK_GC_DELAY_SECS`, else 0: +//! +//! | operation | env var | real work being simulated | +//! |----------------|----------------------------------|-----------------------------------------------| +//! | `BuilderInit` | `MOCK_GC_BUILDER_INIT_DELAY_SECS`| Ziren `client.setup` on the soldering guest | +//! | `Setup` | `MOCK_GC_SETUP_DELAY_SECS` | `BABEVerifier::new` garbling 181 instances | +//! | `Solder` | `MOCK_GC_SOLDER_DELAY_SECS` | opening 7 instances + Ziren STARK proof | +//! | `Verify` | `MOCK_GC_VERIFY_DELAY_SECS` | re-garble 174 opened instances + STARK verify | +//! | `Challenge` | `MOCK_GC_CHALLENGE_DELAY_SECS` | base-instance label derivation | +//! | `Eval` | `MOCK_GC_EVAL_DELAY_SECS` | GC evaluation + message decryption | +//! +//! Note: `Challenge` and `Eval` run inside the peg-out jobs (`verifier_assert`, +//! `wrongly_challenged`), which compete with the honest Operator on 1-3 block regtest +//! timelocks (WronglyChallenged / Disprove). A long delay there makes the Operator lose that +//! race and, past the engine job lease, lets a second holder re-run the job and double-broadcast +//! a dispute tx with a second, conflicting fee input. So `Challenge` and `Eval` are +//! **hard-capped** at [`LATENCY_CRITICAL_MAX_DELAY`]; a larger env value is clamped with a +//! warning. The three heavy setup ops carry no such cap and may be set to minutes. In practice +//! keep `Challenge`/`Eval` to a few seconds (below the block-mining cadence). + +use std::time::Duration; + +/// A mocked GC operation whose artificial delay can be configured independently. +#[derive(Debug, Clone, Copy)] +pub enum GcOp { + BuilderInit, + Setup, + Solder, + Verify, + Challenge, + Eval, +} + +impl GcOp { + const fn env_var(self) -> &'static str { + match self { + GcOp::BuilderInit => "MOCK_GC_BUILDER_INIT_DELAY_SECS", + GcOp::Setup => "MOCK_GC_SETUP_DELAY_SECS", + GcOp::Solder => "MOCK_GC_SOLDER_DELAY_SECS", + GcOp::Verify => "MOCK_GC_VERIFY_DELAY_SECS", + GcOp::Challenge => "MOCK_GC_CHALLENGE_DELAY_SECS", + GcOp::Eval => "MOCK_GC_EVAL_DELAY_SECS", + } + } + + const fn label(self) -> &'static str { + match self { + GcOp::BuilderInit => "builder_init", + GcOp::Setup => "setup", + GcOp::Solder => "solder", + GcOp::Verify => "verify", + GcOp::Challenge => "challenge", + GcOp::Eval => "eval", + } + } + + /// The maximum artificial delay this op may impose, or `None` for an unbounded op. + /// + /// `Challenge` and `Eval` run inline on the sequential P2P-inbox dispatch worker with no + /// lease renewal, so their delay is capped well under the inbox lease (see the module docs). + /// The heavy ops (`BuilderInit`/`Setup`/`Solder`/`Verify`) are spawned with lease renewal and + /// serialized by the heavy-task permit, so they may run for minutes and are left unbounded. + const fn max_delay(self) -> Option { + match self { + GcOp::Challenge | GcOp::Eval => Some(LATENCY_CRITICAL_MAX_DELAY), + GcOp::BuilderInit | GcOp::Setup | GcOp::Solder | GcOp::Verify => None, + } + } +} + +const DEFAULT_DELAY_VAR: &str = "MOCK_GC_DELAY_SECS"; + +/// Hard cap for the `Challenge`/`Eval` delays of the dispute jobs: well under the engine's +/// 5-minute job lease, so a misconfigured value can never let a second holder re-run the job +/// and double-broadcast a dispute transaction. +const LATENCY_CRITICAL_MAX_DELAY: Duration = Duration::from_secs(60); + +fn parse_secs(name: &str) -> Option { + let raw = std::env::var(name).ok()?; + let secs = raw.trim().parse::().ok()?; + if secs.is_finite() && secs > 0.0 { Some(Duration::from_secs_f64(secs)) } else { None } +} + +/// The env-configured delay for `op`, before clamping. +fn configured_delay_for(op: GcOp) -> Duration { + parse_secs(op.env_var()).or_else(|| parse_secs(DEFAULT_DELAY_VAR)).unwrap_or(Duration::ZERO) +} + +/// Clamp a configured delay to the op's safe maximum, warning once per call when it is truncated. +fn clamp_delay(op: GcOp, configured: Duration) -> Duration { + match op.max_delay() { + Some(max) if configured > max => { + tracing::warn!( + event = "mock_gc_delay", + op = op.label(), + stage = "clamped", + configured_secs = configured.as_secs_f64(), + max_secs = max.as_secs_f64(), + "mock GC delay exceeds the safe maximum for an inline dispatch op; clamping so it \ + cannot stall the inbox worker past the lease or race the dispute timelock" + ); + max + } + _ => configured, + } +} + +fn delay_for(op: GcOp) -> Duration { + clamp_delay(op, configured_delay_for(op)) +} + +/// Await the configured artificial delay for `op`, logging one structured line at start and end. +pub async fn simulate(op: GcOp) { + let delay = delay_for(op); + if delay.is_zero() { + return; + } + let delay_secs = delay.as_secs_f64(); + tracing::info!( + event = "mock_gc_delay", + op = op.label(), + stage = "started", + delay_secs, + "simulating mocked GC work" + ); + tokio::time::sleep(delay).await; + tracing::info!( + event = "mock_gc_delay", + op = op.label(), + stage = "finished", + delay_secs, + "mocked GC work complete" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + // The three heavy ops are spawned with lease renewal, so their delay is unbounded. + const HEAVY_OPS: [GcOp; 4] = [GcOp::BuilderInit, GcOp::Setup, GcOp::Solder, GcOp::Verify]; + // These two run inline on the dispatch worker with no lease renewal, so they are capped. + const INLINE_OPS: [GcOp; 2] = [GcOp::Challenge, GcOp::Eval]; + + #[test] + fn inline_dispatch_ops_are_capped() { + for op in INLINE_OPS { + assert_eq!(op.max_delay(), Some(LATENCY_CRITICAL_MAX_DELAY)); + } + for op in HEAVY_OPS { + assert_eq!(op.max_delay(), None); + } + } + + #[test] + fn clamp_truncates_only_inline_ops_over_the_cap() { + let over = LATENCY_CRITICAL_MAX_DELAY + Duration::from_secs(600); + // An oversized inline delay is clamped to the safe maximum (well under the inbox lease). + for op in INLINE_OPS { + assert_eq!(clamp_delay(op, over), LATENCY_CRITICAL_MAX_DELAY); + } + // Heavy ops keep whatever was configured, even minutes. + for op in HEAVY_OPS { + assert_eq!(clamp_delay(op, over), over); + } + } + + #[test] + fn clamp_preserves_values_within_the_cap() { + let within = Duration::from_secs(3); + for op in INLINE_OPS.into_iter().chain(HEAVY_OPS) { + assert_eq!(clamp_delay(op, within), within); + } + // Exactly the cap is not truncated. + for op in INLINE_OPS { + assert_eq!(clamp_delay(op, LATENCY_CRITICAL_MAX_DELAY), LATENCY_CRITICAL_MAX_DELAY); + } + } + + #[test] + fn cap_is_well_under_the_p2p_inbox_lease() { + // P2P_INBOX_LEASE_SECS is 5 minutes (node/src/action.rs); the cap must leave ample margin + // so a clamped inline op cannot outlive the (never-renewed) lease and get re-dispatched. + assert!(LATENCY_CRITICAL_MAX_DELAY <= Duration::from_secs(60)); + assert!(LATENCY_CRITICAL_MAX_DELAY * 4 < Duration::from_secs(5 * 60)); + } +} diff --git a/node/src/p2p_admission.rs b/node/src/p2p_admission.rs deleted file mode 100644 index 1e34b3c34..000000000 --- a/node/src/p2p_admission.rs +++ /dev/null @@ -1,4087 +0,0 @@ -//! Inbound gossip admission: envelope, identity, replay, rate and storage checks. -//! Durable messages are admitted before persistence and forwarding. -//! Rejected messages use gossipsub `Ignore`. - -use std::collections::{HashMap, HashSet}; -use std::str::FromStr; -use std::sync::{Arc, LazyLock, Mutex}; -use std::time::{Duration, Instant}; - -use anyhow::Result; -use client::goat_chain::GOATClient; -use libp2p::PeerId; -use sha2::{Digest, Sha256}; -use store::localdb::LocalDB; -use store::{P2pInboxAdmissionClass, P2pInboxClassUsage}; - -use crate::action::{GOATMessage, GOATMessageContent, P2PMessageDelivery}; -use crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE; - -/// Pending retention for unregistered senders; registered senders use the normal retention window. -pub const UNREGISTERED_PENDING_TTL_SECS: i64 = 2 * 60 * 60; - -const VERIFIER_SET_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); -const VERIFIER_SET_RETRY_INTERVAL: Duration = Duration::from_secs(30); -const COMMITTEE_POSITIVE_TTL: Duration = Duration::from_secs(10 * 60); -const COMMITTEE_NEGATIVE_TTL: Duration = Duration::from_secs(2 * 60); -/// Maximum stale age for retaining a registered classification during refresh. -const COMMITTEE_STALE_LIMIT: Duration = Duration::from_secs(30 * 60); -const COMMITTEE_CACHE_MAX_ENTRIES: usize = 4096; -/// First-time lookup rate; known-member refreshes are exempt. -const COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE: u32 = 60; -const COMMITTEE_MAX_IN_FLIGHT_LOOKUPS: usize = 16; -/// Maximum unknown-identity lookups in flight. -const COMMITTEE_MAX_UNKNOWN_IN_FLIGHT: usize = 12; -const REGISTRY_RPC_TIMEOUT: Duration = Duration::from_secs(10); - -/// Separate discovery budget for operator stake lookups. -const OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE: u32 = 60; -const OPERATOR_MAX_IN_FLIGHT_LOOKUPS: usize = 16; -const OPERATOR_MAX_UNKNOWN_IN_FLIGHT: usize = 12; -const REFUSED_PEER_LOOKUPS_PER_MINUTE: u32 = 30; - -/// A confirmed operator binding keeps its class this long before re-validation; -/// a shorter window applies while the stake is still unknown. -const OPERATOR_STAKE_TTL: Duration = Duration::from_secs(10 * 60); -const OPERATOR_STAKE_STALE_LIMIT: Duration = Duration::from_secs(30 * 60); -const OPERATOR_CACHE_MAX_ENTRIES: usize = 4096; -/// Maximum persisted operator bindings loaded at startup. -pub const OPERATOR_BINDING_LOAD_LIMIT: i64 = OPERATOR_CACHE_MAX_ENTRIES as i64; - -const IMMEDIATE_BURST: f64 = 240.0; -const IMMEDIATE_REFILL_PER_SEC: f64 = 2.0; -const IMMEDIATE_MAX_TRACKED_PEERS: usize = 4096; - -/// NodeInfo broadcast response cooldown. -const NODE_INFO_RESPONSE_COOLDOWN: Duration = Duration::from_secs(10); - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct Quota { - pub rows: i64, - pub bytes: i64, -} - -impl Quota { - fn admits(&self, rows: i64, bytes: i64) -> bool { - rows <= self.rows && bytes <= self.bytes - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct InboundLimits { - pub max_json_bytes: usize, - pub max_binary_bytes: usize, - /// Every queued row, whatever its class. - pub global: Quota, - /// Global queue capacity reserved for committee. - pub committee_reserve: Quota, - /// Every queued row from unregistered senders. - pub unregistered_class: Quota, - pub registered_peer: Quota, - pub unregistered_peer: Quota, -} - -impl InboundLimits { - /// Derive byte quotas from the global ceiling; row limits remain fixed. - pub fn with_queued_bytes(max_json_bytes: usize, max_queued_bytes: i64) -> Self { - Self { - max_json_bytes, - max_binary_bytes: MAX_GOSSIPSUB_TRANSMIT_SIZE, - global: Quota { rows: 16_384, bytes: max_queued_bytes }, - committee_reserve: Quota { rows: 4_096, bytes: max_queued_bytes / 4 }, - unregistered_class: Quota { rows: 4_096, bytes: max_queued_bytes / 4 }, - registered_peer: Quota { rows: 1_024, bytes: max_queued_bytes / 8 }, - unregistered_peer: Quota { rows: 512, bytes: max_queued_bytes / 32 }, - } - } - - pub fn from_env() -> Self { - Self::with_queued_bytes( - crate::env::get_p2p_max_json_message_bytes(), - crate::env::get_p2p_inbox_max_queued_bytes(), - ) - } -} - -static INBOUND_LIMITS: LazyLock = LazyLock::new(InboundLimits::from_env); - -pub fn inbound_limits() -> &'static InboundLimits { - &INBOUND_LIMITS -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum DropReason { - OversizedJson, - OversizedBinary, - BinaryFromNonVerifier, - /// A registered author's message whose sequence number was already seen, or - /// is older than the replay window: a verbatim re-publish by someone else. - ReplayedSequence, - /// Sequence number exceeds the accepted clock lead. - FutureSequence, - /// The chain has said the author does not hold the role this kind of - /// message can only come from. - SenderRoleDenied, - /// A message addressed to another role, from an unregistered author, beyond - /// what this node relays for such authors. - ForwardRate, - DirectPeerRate, - AuthorRate, - UnregisteredRate, - GlobalRate, - Undecodable, - UnexpectedBinaryKind, - ImmediateRateLimited, - /// The bounded queue in front of the control worker (NodeInfo, ACKs) had no - /// room, by rows or by bytes. Distinct from `ImmediateRateLimited`: that one - /// is a sender exceeding its rate, this one is the node falling behind. - ControlQueueFull, - SenderQuota, - UnregisteredClassQuota, - GlobalQuota, - /// Not a message drop: a `NodeInfo` from a sender that is not registered was - /// not stored because the `node` table is at its ceiling for such senders. - NodeTableFull, - /// NodeInfo rejected for invalid identity, fields or binding. - NodeInfoRejected, - /// Not a drop: a message from an unregistered sender that was queued. Only - /// counted for the log summary, never exported as a drop metric. - AdmittedUnregistered, - /// Duplicate queued payload, recorded in the periodic summary. - DuplicatePayload, -} - -impl DropReason { - pub const fn as_str(self) -> &'static str { - match self { - Self::OversizedJson => "oversized_json", - Self::OversizedBinary => "oversized_binary", - Self::BinaryFromNonVerifier => "binary_from_non_verifier", - Self::ReplayedSequence => "replayed_sequence", - Self::FutureSequence => "future_sequence", - Self::SenderRoleDenied => "sender_role_denied", - Self::ForwardRate => "forward_rate", - Self::DirectPeerRate => "direct_peer_rate", - Self::AuthorRate => "author_rate", - Self::UnregisteredRate => "unregistered_rate", - Self::GlobalRate => "global_rate", - Self::Undecodable => "undecodable", - Self::UnexpectedBinaryKind => "unexpected_binary_kind", - Self::ImmediateRateLimited => "immediate_rate_limited", - Self::ControlQueueFull => "control_queue_full", - Self::SenderQuota => "sender_quota", - Self::UnregisteredClassQuota => "unregistered_class_quota", - Self::GlobalQuota => "global_quota", - Self::NodeTableFull => "node_table_full", - Self::NodeInfoRejected => "node_info_rejected", - Self::AdmittedUnregistered => "admitted_unregistered", - Self::DuplicatePayload => "duplicate_payload", - } - } - - /// Whether a drop is attributable to the direct peer, independent of local state and configuration. - pub const fn blames_direct_peer(self) -> bool { - matches!(self, Self::OversizedBinary | Self::Undecodable | Self::UnexpectedBinaryKind) - } - - /// Whether the payload was decoded before it was dropped. - pub const fn after_decode(self) -> bool { - matches!( - self, - Self::UnexpectedBinaryKind - | Self::SenderRoleDenied - | Self::ForwardRate - | Self::ImmediateRateLimited - | Self::ControlQueueFull - | Self::SenderQuota - | Self::UnregisteredClassQuota - | Self::GlobalQuota - ) - } -} - -/// Checks that need nothing but the raw bytes and the sender's identity. -pub fn check_envelope( - data: &[u8], - sender_is_verifier: bool, - limits: &InboundLimits, -) -> Result<(), DropReason> { - if GOATMessage::is_binary_envelope(data) { - if !sender_is_verifier { - return Err(DropReason::BinaryFromNonVerifier); - } - if data.len() > limits.max_binary_bytes { - return Err(DropReason::OversizedBinary); - } - } else if data.len() > limits.max_json_bytes { - return Err(DropReason::OversizedJson); - } - Ok(()) -} - -/// Whether one more queued row of `content_len` bytes from a sender of `class` -/// fits. `usage` is the queued state *before* the row is added. -pub fn check_inbox_quota( - class: P2pInboxAdmissionClass, - content_len: usize, - usage: &[P2pInboxClassUsage], - limits: &InboundLimits, -) -> Result<(), DropReason> { - let content_len = content_len as i64; - let total_rows: i64 = usage.iter().map(|usage| usage.rows).sum(); - let total_bytes: i64 = usage.iter().map(|usage| usage.bytes).sum(); - if !limits.global.admits(total_rows + 1, total_bytes + content_len) { - return Err(DropReason::GlobalQuota); - } - if class != P2pInboxAdmissionClass::Committee { - let committee = P2pInboxAdmissionClass::Committee.to_string(); - let (rows, bytes) = usage - .iter() - .filter(|usage| usage.admission_class != committee) - .fold((0, 0), |(rows, bytes), usage| (rows + usage.rows, bytes + usage.bytes)); - let open = Quota { - rows: limits.global.rows - limits.committee_reserve.rows, - bytes: limits.global.bytes - limits.committee_reserve.bytes, - }; - if !open.admits(rows + 1, bytes + content_len) { - return Err(DropReason::GlobalQuota); - } - } - - // A sender can hold rows in more than one class: it is classified when each - // row arrives, and a registration lookup may complete in between. - let peer_rows: i64 = usage.iter().map(|usage| usage.peer_rows).sum(); - let peer_bytes: i64 = usage.iter().map(|usage| usage.peer_bytes).sum(); - let peer_quota = - if class.is_registered() { limits.registered_peer } else { limits.unregistered_peer }; - if !peer_quota.admits(peer_rows + 1, peer_bytes + content_len) { - return Err(DropReason::SenderQuota); - } - - if !class.is_registered() { - let class_name = P2pInboxAdmissionClass::Unregistered.to_string(); - let (class_rows, class_bytes) = usage - .iter() - .find(|usage| usage.admission_class == class_name) - .map_or((0, 0), |usage| (usage.rows, usage.bytes)); - if !limits.unregistered_class.admits(class_rows + 1, class_bytes + content_len) { - return Err(DropReason::UnregisteredClassQuota); - } - } - Ok(()) -} - -/// Persistent C/R/C/U rotation; empty classes yield their turn. -#[derive(Debug, Default)] -pub struct InboxSchedule { - position: usize, -} - -/// Indices into the per-class queues of [`InboxSchedule::next`]. -pub const SCHEDULE_COMMITTEE: usize = 0; -pub const SCHEDULE_REGISTERED: usize = 1; -pub const SCHEDULE_UNREGISTERED: usize = 2; - -const SCHEDULE_ROTA: [usize; 4] = - [SCHEDULE_COMMITTEE, SCHEDULE_REGISTERED, SCHEDULE_COMMITTEE, SCHEDULE_UNREGISTERED]; - -impl InboxSchedule { - /// The queue a row of `admission_class` belongs to. - pub fn queue_of(admission_class: &str) -> usize { - if admission_class == P2pInboxAdmissionClass::Committee.to_string() { - SCHEDULE_COMMITTEE - } else if admission_class == P2pInboxAdmissionClass::Registered.to_string() { - SCHEDULE_REGISTERED - } else { - SCHEDULE_UNREGISTERED - } - } - - /// Advance the rotation only for a row about to be dispatched. - pub fn next(&mut self, queues: &mut [std::collections::VecDeque; 3]) -> Option { - for step in 0..SCHEDULE_ROTA.len() { - let position = (self.position + step) % SCHEDULE_ROTA.len(); - if let Some(row) = queues[SCHEDULE_ROTA[position]].pop_front() { - self.position = (position + 1) % SCHEDULE_ROTA.len(); - return Some(row); - } - } - None - } -} - -static INBOX_SCHEDULE: LazyLock> = - LazyLock::new(|| Mutex::new(InboxSchedule::default())); - -/// The process-wide rota. Only the inbox worker, on the swarm loop, touches it. -pub fn inbox_schedule() -> std::sync::MutexGuard<'static, InboxSchedule> { - INBOX_SCHEDULE.lock().unwrap_or_else(std::sync::PoisonError::into_inner) -} - -pub fn content_hash(data: &[u8]) -> [u8; 32] { - Sha256::digest(data).into() -} - -#[derive(Clone, Copy, Debug)] -struct CommitteeEntry { - registered: bool, - fetched_at: Instant, -} - -impl CommitteeEntry { - fn ttl(&self) -> Duration { - if self.registered { COMMITTEE_POSITIVE_TTL } else { COMMITTEE_NEGATIVE_TTL } - } - - fn needs_refresh(&self, now: Instant) -> bool { - now.saturating_duration_since(self.fetched_at) >= self.ttl() - } - - fn is_registered(&self, now: Instant) -> bool { - self.registered && now.saturating_duration_since(self.fetched_at) < COMMITTEE_STALE_LIMIT - } -} - -/// Canonical operator identity: the x-only master public key. -pub type OperatorKey = [u8; 32]; - -pub fn operator_key(pubkey: &bitcoin::XOnlyPublicKey) -> OperatorKey { - pubkey.serialize() -} - -/// A peer that proved (via a NodeInfo binding) it owns a master key, plus what -/// the chain says about that key's operator stake. -#[derive(Clone, Debug)] -struct OperatorBinding { - pubkey: OperatorKey, - issued_at: i64, - /// `Some(true)` once the key is confirmed to be a sufficiently staked - /// operator on chain; `None` until the lookup completes. - staked: Option, - fetched_at: Option, - /// Configured binding; bypasses discovery budget but still requires a positive stake verdict. - trusted: bool, -} - -impl OperatorBinding { - fn needs_refresh(&self, now: Instant) -> bool { - match self.fetched_at { - None => true, - Some(fetched_at) => now.saturating_duration_since(fetched_at) >= OPERATOR_STAKE_TTL, - } - } - - fn is_registered(&self, now: Instant) -> bool { - self.staked == Some(true) - && self - .fetched_at - .is_some_and(|at| now.saturating_duration_since(at) < OPERATOR_STAKE_STALE_LIMIT) - } -} - -#[derive(Debug, Default)] -struct RegistryState { - verifiers: HashSet>, - verifiers_fetched_at: Option, - verifiers_attempted_at: Option, - verifiers_in_flight: bool, - committee: HashMap, - committee_in_flight: HashSet, - /// Operator bindings keyed by the peer that proved them. - operators: HashMap, - /// Reverse index from master key to its current peer ID. - operator_owner: HashMap, - operators_in_flight: HashSet, - committee_discovery: DiscoveryBudget, - /// Separate lookup budget for connected neighbours. - neighbour_discovery: DiscoveryBudget, - /// For peers the connection gate refused; see [`LookupBudget::Refused`]. - refused_discovery: DiscoveryBudget, - operator_discovery: DiscoveryBudget, -} - -/// Per-minute budget for first-time (unknown-identity) registry lookups. -#[derive(Debug, Default)] -struct DiscoveryBudget { - window_started_at: Option, - lookups_in_window: u32, -} - -impl DiscoveryBudget { - /// Charge one lookup. Returns false when the budget is exhausted. - fn take(&mut self, per_minute: u32, now: Instant) -> bool { - let window_open = self.window_started_at.is_some_and(|started_at| { - now.saturating_duration_since(started_at) < Duration::from_secs(60) - }); - if !window_open { - self.window_started_at = Some(now); - self.lookups_in_window = 0; - } - if self.lookups_in_window >= per_minute { - return false; - } - self.lookups_in_window += 1; - true - } -} - -/// Whose budget a first-time committee lookup is charged to. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum LookupBudget { - /// Discovery budget for signed message authors. - Author, - /// A peer this node holds a connection with; bounded by the connection - /// ceiling. Local bans depend on these lookups coming back. - Neighbour, - /// A peer whose connection was refused for lack of an open slot. - Refused, -} - -impl RegistryState { - /// Claim the verifier set refresh if it is due. The caller must fetch it and - /// call `record_verifier_set`, or it stays marked as in flight. - fn claim_verifier_set_refresh(&mut self, now: Instant) -> bool { - let due = match (self.verifiers_fetched_at, self.verifiers_attempted_at) { - (_, Some(attempted_at)) - if now.saturating_duration_since(attempted_at) < VERIFIER_SET_RETRY_INTERVAL => - { - false - } - (Some(fetched_at), _) => { - now.saturating_duration_since(fetched_at) >= VERIFIER_SET_REFRESH_INTERVAL - } - (None, _) => true, - }; - if !due || self.verifiers_in_flight { - return false; - } - self.verifiers_in_flight = true; - self.verifiers_attempted_at = Some(now); - true - } - - /// Claim a committee lookup for `peer` if one is due and a slot is free. The - /// caller must resolve it with `record_committee_peer`. - fn claim_committee_lookup( - &mut self, - peer: &PeerId, - budget: LookupBudget, - now: Instant, - ) -> bool { - if self.verifiers.contains(&peer.to_bytes()) || self.committee_in_flight.contains(peer) { - return false; - } - let known_registered = match self.committee.get(peer) { - Some(entry) if !entry.needs_refresh(now) => return false, - Some(entry) => entry.registered, - None => false, - }; - // Known-member refreshes may use every slot without discovery charges. - let in_flight = self.committee_in_flight.len(); - let admitted = if known_registered { - in_flight < COMMITTEE_MAX_IN_FLIGHT_LOOKUPS - } else { - match budget { - LookupBudget::Author => { - in_flight < COMMITTEE_MAX_UNKNOWN_IN_FLIGHT - && self - .committee_discovery - .take(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, now) - } - LookupBudget::Neighbour => { - in_flight < COMMITTEE_MAX_IN_FLIGHT_LOOKUPS - && self - .neighbour_discovery - .take(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, now) - } - LookupBudget::Refused => { - in_flight < COMMITTEE_MAX_UNKNOWN_IN_FLIGHT - && self.refused_discovery.take(REFUSED_PEER_LOOKUPS_PER_MINUTE, now) - } - } - }; - if admitted { - self.committee_in_flight.insert(*peer); - } - admitted - } - - /// Claim a due operator lookup; confirmed operators bypass discovery limits. - fn claim_operator_refresh( - &mut self, - peer: &PeerId, - pubkey: &OperatorKey, - now: Instant, - ) -> bool { - if self.operators_in_flight.contains(peer) { - return false; - } - let known_registered = match self.operators.get(peer) { - // Only the current owner of the key is worth validating. - Some(binding) if binding.pubkey != *pubkey => return false, - Some(binding) if !binding.needs_refresh(now) => return false, - Some(binding) => binding.staked == Some(true) || binding.trusted, - None => return false, - }; - let in_flight = self.operators_in_flight.len(); - if known_registered { - if in_flight >= OPERATOR_MAX_IN_FLIGHT_LOOKUPS { - return false; - } - } else if in_flight >= OPERATOR_MAX_UNKNOWN_IN_FLIGHT - || !self.operator_discovery.take(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE, now) - { - return false; - } - self.operators_in_flight.insert(*peer); - true - } -} - -/// Trust window for restored registrations awaiting chain refresh. -const SEEDED_TRUST: Duration = Duration::from_secs(5 * 60); - -/// How far back a seeded entry is dated: due for re-validation right away, and -/// `SEEDED_TRUST` short of the stale limit at which it stops being trusted. -fn seeded_fetched_at(now: Instant, stale_limit: Duration) -> Instant { - now.checked_sub(stale_limit.saturating_sub(SEEDED_TRUST)).unwrap_or(now) -} - -/// Why a configured operator binding was not applied. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum TrustedBindingConflict { - /// The peer already holds a binding, signed by another key. - PeerBoundToAnotherKey(OperatorKey), - /// The key is already bound, by its own signature, to another peer id. - KeyBoundToAnotherPeer(PeerId), -} - -/// The role a kind of message can only legitimately come from. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum SenderRole { - /// Chain observations and requests any node may publish. - Any, - Committee, - Verifier, - Operator, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum RoleVerdict { - Confirmed, - Denied, - Unknown, -} - -/// What the caller should fetch from the chain, decided under the cache lock so -/// concurrent messages from one peer start a single lookup. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct RefreshPlan { - pub verifier_set: bool, - pub committee_peer: bool, -} - -/// Cached chain registration; misses are Unregistered and trigger background lookups. -#[derive(Debug, Default)] -pub struct PeerRegistry { - state: Mutex, -} - -impl PeerRegistry { - fn lock(&self) -> std::sync::MutexGuard<'_, RegistryState> { - self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner) - } - - pub fn is_verifier(&self, peer: &PeerId) -> bool { - self.lock().verifiers.contains(&peer.to_bytes()) - } - - pub fn sender_class(&self, peer: &PeerId, now: Instant) -> P2pInboxAdmissionClass { - let state = self.lock(); - if state.committee.get(peer).is_some_and(|entry| entry.is_registered(now)) { - P2pInboxAdmissionClass::Committee - } else if state.verifiers.contains(&peer.to_bytes()) - || state.operators.get(peer).is_some_and(|binding| binding.is_registered(now)) - { - P2pInboxAdmissionClass::Registered - } else { - P2pInboxAdmissionClass::Unregistered - } - } - - /// Whether `peer` is a committee member, from the cache. For handlers whose - /// authorisation is the committee specifically, not any registered sender. - pub fn is_committee(&self, peer: &PeerId, now: Instant) -> bool { - self.sender_class(peer, now) == P2pInboxAdmissionClass::Committee - } - - /// Cached role verdict. Only fresh negative answers are Denied; missing or stale answers are Unknown. - pub fn role_verdict(&self, peer: &PeerId, role: SenderRole, now: Instant) -> RoleVerdict { - let state = self.lock(); - match role { - SenderRole::Any => RoleVerdict::Confirmed, - SenderRole::Committee => match state.committee.get(peer) { - Some(entry) if entry.is_registered(now) => RoleVerdict::Confirmed, - Some(entry) if !entry.registered && !entry.needs_refresh(now) => { - RoleVerdict::Denied - } - _ => RoleVerdict::Unknown, - }, - SenderRole::Verifier => { - if state.verifiers.contains(&peer.to_bytes()) { - RoleVerdict::Confirmed - } else if state.verifiers_fetched_at.is_some_and(|at| { - now.saturating_duration_since(at) < VERIFIER_SET_REFRESH_INTERVAL * 2 - }) { - RoleVerdict::Denied - } else { - RoleVerdict::Unknown - } - } - SenderRole::Operator => match state.operators.get(peer) { - Some(binding) if binding.is_registered(now) => RoleVerdict::Confirmed, - Some(binding) if binding.staked == Some(false) && !binding.needs_refresh(now) => { - RoleVerdict::Denied - } - _ => RoleVerdict::Unknown, - }, - } - } - - /// Whether fresh chain results identify the peer as unregistered. - pub fn is_known_unregistered(&self, peer: &PeerId, now: Instant) -> bool { - // A stale "no" does not count: the peer may have registered since. - let looked_up = self - .lock() - .committee - .get(peer) - .is_some_and(|entry| !entry.registered && !entry.needs_refresh(now)); - looked_up && !self.sender_class(peer, now).is_registered() - } - - /// Registered verifier peer IDs; `None` means not yet fetched. - pub fn verifier_peer_ids(&self) -> Option> { - let state = self.lock(); - state.verifiers_fetched_at?; - Some( - state - .verifiers - .iter() - .filter_map(|bytes| PeerId::from_bytes(bytes).ok()) - .map(|peer| peer.to_string()) - .collect(), - ) - } - - /// Record a verified binding; newer issuance wins and each key maps to one peer. - /// Returns whether the peer's previous persisted registration must be revoked. - pub fn observe_operator_binding( - &self, - peer: &PeerId, - pubkey: &OperatorKey, - issued_at: i64, - ) -> bool { - let mut state = self.lock(); - let mut released_confirmed_key = false; - // Move the key to the peer with the newer binding. - if let Some(current_owner) = state.operator_owner.get(pubkey).copied() - && current_owner != *peer - { - let newer = state - .operators - .get(¤t_owner) - // Require a strictly newer issuance time for ownership transfer. - .is_none_or(|existing| issued_at > existing.issued_at); - if !newer { - return false; - } - state.operators.remove(¤t_owner); - } - if let Some(previous) = state.operators.get_mut(peer) { - if previous.pubkey == *pubkey { - // Preserve the stake verdict when the peer re-announces the same key. - previous.issued_at = previous.issued_at.max(issued_at); - state.operator_owner.insert(*pubkey, *peer); - return false; - } - // A peer that rebinds to a different key releases its old key's - // ownership, and whatever was confirmed for that key with it. - released_confirmed_key = previous.staked == Some(true); - let previous_pubkey = previous.pubkey; - if state.operator_owner.get(&previous_pubkey) == Some(peer) { - state.operator_owner.remove(&previous_pubkey); - } - } - if state.operators.len() >= OPERATOR_CACHE_MAX_ENTRIES - && !state.operators.contains_key(peer) - { - let now = Instant::now(); - state.operators.retain(|_, binding| binding.is_registered(now) || binding.trusted); - let live: HashSet = state.operators.keys().copied().collect(); - state.operator_owner.retain(|_, owner| live.contains(owner)); - } - state.operator_owner.insert(*pubkey, *peer); - state.operators.insert( - *peer, - OperatorBinding { - pubkey: *pubkey, - issued_at, - staked: None, - fetched_at: None, - trusted: false, - }, - ); - released_confirmed_key - } - - /// Record a configured binding without overriding a signed binding. - /// See `P2P_TRUSTED_OPERATOR_BINDINGS`. - pub fn trust_operator_binding( - &self, - peer: &PeerId, - pubkey: &OperatorKey, - ) -> Result<(), TrustedBindingConflict> { - let mut state = self.lock(); - if let Some(existing) = state.operators.get(peer) - && existing.pubkey != *pubkey - { - return Err(TrustedBindingConflict::PeerBoundToAnotherKey(existing.pubkey)); - } - if let Some(owner) = state.operator_owner.get(pubkey) - && owner != peer - { - return Err(TrustedBindingConflict::KeyBoundToAnotherPeer(*owner)); - } - state.operator_owner.insert(*pubkey, *peer); - state.operators.entry(*peer).and_modify(|binding| binding.trusted = true).or_insert( - OperatorBinding { - pubkey: *pubkey, - issued_at: 0, - staked: None, - fetched_at: None, - trusted: true, - }, - ); - Ok(()) - } - - /// Whether an operator stake lookup for `peer`/`pubkey` should start now. - /// Claimed lookups must be resolved with [`Self::record_operator_stake`]. - pub fn plan_operator_refresh(&self, peer: &PeerId, pubkey: &OperatorKey, now: Instant) -> bool { - self.lock().claim_operator_refresh(peer, pubkey, now) - } - - /// Claim due operator refreshes, confirmed operators first. - pub fn plan_due_operator_refreshes(&self, now: Instant) -> Vec<(PeerId, OperatorKey)> { - let mut state = self.lock(); - let mut due: Vec<(PeerId, OperatorKey, bool)> = state - .operators - .iter() - // Retry a negative stake verdict only after another announcement. - .filter(|(peer, binding)| { - binding.needs_refresh(now) - && (binding.staked != Some(false) || binding.trusted) - && !state.operators_in_flight.contains(*peer) - }) - .map(|(peer, binding)| { - (*peer, binding.pubkey, binding.staked == Some(true) || binding.trusted) - }) - .collect(); - due.sort_by_key(|(_, _, known)| !*known); - due.into_iter() - .filter(|(peer, pubkey, _)| state.claim_operator_refresh(peer, pubkey, now)) - .map(|(peer, pubkey, _)| (peer, pubkey)) - .collect() - } - - /// Record the verdict for the peer's current binding and return the persistence update. - pub fn record_operator_stake( - &self, - peer: &PeerId, - pubkey: &OperatorKey, - staked: Option, - now: Instant, - ) -> Option { - let mut state = self.lock(); - state.operators_in_flight.remove(peer); - let staked = staked?; - // Ignore a result for a binding that was superseded while in flight. - let binding = state.operators.get_mut(peer).filter(|binding| binding.pubkey == *pubkey)?; - binding.staked = Some(staked); - binding.fetched_at = Some(now); - Some(staked) - } - - /// Restore a confirmed operator only when its current binding matches the stored key. - /// Schedule immediate revalidation. - pub fn seed_verified_operator( - &self, - peer: &PeerId, - pubkey: &OperatorKey, - issued_at: i64, - now: Instant, - ) { - self.observe_operator_binding(peer, pubkey, issued_at); - let mut state = self.lock(); - if let Some(binding) = state.operators.get_mut(peer) - && binding.pubkey == *pubkey - && binding.staked.is_none() - { - binding.staked = Some(true); - binding.fetched_at = Some(seeded_fetched_at(now, OPERATOR_STAKE_STALE_LIMIT)); - } - } - - /// Re-instate a committee member confirmed in an earlier session; see - /// [`Self::seed_verified_operator`]. - pub fn seed_verified_committee_peer(&self, peer: &PeerId, now: Instant) { - self.lock().committee.entry(*peer).or_insert(CommitteeEntry { - registered: true, - fetched_at: seeded_fetched_at(now, COMMITTEE_STALE_LIMIT), - }); - } - - /// Claim due committee-member refreshes. - pub fn plan_due_committee_refreshes(&self, now: Instant) -> Vec { - let mut state = self.lock(); - let due: Vec = state - .committee - .iter() - .filter(|(peer, entry)| { - entry.registered - && entry.needs_refresh(now) - && !state.committee_in_flight.contains(*peer) - }) - .map(|(peer, _)| *peer) - .collect(); - let free = COMMITTEE_MAX_IN_FLIGHT_LOOKUPS.saturating_sub(state.committee_in_flight.len()); - let claimed: Vec = due.into_iter().take(free).collect(); - state.committee_in_flight.extend(claimed.iter().copied()); - claimed - } - - /// Claim the lookups that are due. Whatever is claimed must be resolved with - /// the matching `record_*` call, or it stays marked as in flight. - pub fn plan_refresh(&self, peer: Option<&PeerId>, now: Instant) -> RefreshPlan { - let mut state = self.lock(); - RefreshPlan { - verifier_set: state.claim_verifier_set_refresh(now), - committee_peer: peer - .is_some_and(|peer| state.claim_committee_lookup(peer, LookupBudget::Author, now)), - } - } - - /// Claim only the neighbour's committee lookup, using the neighbour budget. - pub fn plan_neighbour_lookup(&self, peer: &PeerId, now: Instant) -> bool { - self.lock().claim_committee_lookup(peer, LookupBudget::Neighbour, now) - } - - /// Claim refused-peer lookups using their separate budget. - pub fn plan_refused_peer_lookup(&self, peer: &PeerId, now: Instant) -> bool { - self.lock().claim_committee_lookup(peer, LookupBudget::Refused, now) - } - - pub fn record_verifier_set(&self, verifiers: Option>>, now: Instant) { - let mut state = self.lock(); - state.verifiers_in_flight = false; - // Retain the previous verifier set on fetch failure. - if let Some(verifiers) = verifiers { - state.verifiers = verifiers.into_iter().collect(); - state.verifiers_fetched_at = Some(now); - } - } - - /// Record a committee lookup. Returns the definite verdict, for the caller - /// to persist; see [`Self::record_operator_stake`]. - pub fn record_committee_peer( - &self, - peer: &PeerId, - registered: Option, - now: Instant, - ) -> Option { - let mut state = self.lock(); - state.committee_in_flight.remove(peer); - let registered = registered?; - let changed = Some(registered); - if !state.committee.contains_key(peer) - && state.committee.len() >= COMMITTEE_CACHE_MAX_ENTRIES - { - state.committee.retain(|_, entry| !entry.needs_refresh(now)); - if state.committee.len() >= COMMITTEE_CACHE_MAX_ENTRIES { - if !registered { - return changed; - } - let evict = state - .committee - .iter() - .find(|(_, entry)| !entry.registered) - .map(|(peer, _)| *peer); - if let Some(evict) = evict { - state.committee.remove(&evict); - } - } - } - state.committee.insert(*peer, CommitteeEntry { registered, fetched_at: now }); - changed - } -} - -static PEER_REGISTRY: LazyLock = LazyLock::new(PeerRegistry::default); - -pub fn peer_registry() -> &'static PeerRegistry { - &PEER_REGISTRY -} - -/// Persisted registration kinds. -const REGISTERED_KIND_COMMITTEE: &str = "Committee"; -const REGISTERED_KIND_OPERATOR: &str = "Operator"; - -/// Persist a change in a peer's confirmed registration. Best effort: the cache -/// stays authoritative for this session, the table only warms the next one. -async fn persist_registration_change( - local_db: &LocalDB, - peer: &PeerId, - kind: &str, - pubkey: Option<&OperatorKey>, - change: Option, -) { - let Some(registered) = change else { - return; - }; - let peer_id = peer.to_string(); - let pubkey = pubkey.map(hex::encode).unwrap_or_default(); - let result = async { - let mut storage = local_db.acquire().await?; - if registered { - storage.upsert_p2p_registered_peer(&peer_id, kind, &pubkey).await - } else { - storage.delete_p2p_registered_peer(&peer_id, kind).await - } - } - .await; - if let Err(error) = result { - tracing::debug!( - event = "p2p_admission", - outcome = "registration_persist_failed", - peer_id = %peer_id, - kind, - error = %error, - "failed to persist a confirmed registration; it will be rediscovered after a restart" - ); - } -} - -/// Revoke persisted operator registration after a key change. -pub async fn revoke_persisted_operator(local_db: &LocalDB, peer: &PeerId) { - persist_registration_change(local_db, peer, REGISTERED_KIND_OPERATOR, None, Some(false)).await; -} - -/// Restore confirmed peers; operator bindings must be signature-checked and match stored keys. -pub async fn seed_registry_from_store( - local_db: &LocalDB, - registry: &PeerRegistry, - operator_bindings: &HashMap, - now: Instant, -) -> Result { - let persisted = local_db.acquire().await?.load_p2p_registered_peers().await?; - let mut seeded = 0; - for (peer_id, kind, confirmed_pubkey) in persisted { - let Ok(peer) = PeerId::from_str(&peer_id) else { - continue; - }; - match kind.as_str() { - REGISTERED_KIND_COMMITTEE => { - registry.seed_verified_committee_peer(&peer, now); - seeded += 1; - } - REGISTERED_KIND_OPERATOR => { - if let Some((pubkey, issued_at)) = operator_bindings.get(&peer) - && hex::encode(pubkey) == confirmed_pubkey - { - registry.seed_verified_operator(&peer, pubkey, *issued_at, now); - seeded += 1; - } - } - _ => {} - } - } - Ok(seeded) -} - -fn spawn_committee_lookup(local_db: &LocalDB, goat_client: &Arc, peer: PeerId) { - let registry = peer_registry(); - let (local_db, goat_client) = (local_db.clone(), goat_client.clone()); - tokio::spawn(async move { - let registered = match tokio::time::timeout( - REGISTRY_RPC_TIMEOUT, - goat_client.committee_mana_is_validate_peer_id(&peer.to_bytes()), - ) - .await - { - Ok(Ok(registered)) => Some(registered), - Ok(Err(error)) => { - tracing::debug!( - event = "p2p_admission", - outcome = "committee_lookup_failed", - peer_id = %peer, - error = %error, - "failed to look up a peer in the committee registry" - ); - None - } - Err(_) => None, - }; - let change = registry.record_committee_peer(&peer, registered, Instant::now()); - persist_registration_change(&local_db, &peer, REGISTERED_KIND_COMMITTEE, None, change) - .await; - }); -} - -fn spawn_operator_stake_lookup( - local_db: &LocalDB, - goat_client: &Arc, - peer: PeerId, - operator_key: OperatorKey, -) { - use crate::utils::OperatorStakeStatus; - - let registry = peer_registry(); - // The stake is registered under the x-only key; the parity chosen here is - // dropped again by the lookup. - let Ok(xonly) = bitcoin::XOnlyPublicKey::from_slice(&operator_key) else { - registry.record_operator_stake(&peer, &operator_key, Some(false), Instant::now()); - return; - }; - let pubkey = bitcoin::PublicKey::new(xonly.public_key(bitcoin::secp256k1::Parity::Even)); - let (local_db, goat_client) = (local_db.clone(), goat_client.clone()); - tokio::spawn(async move { - let staked = match tokio::time::timeout( - REGISTRY_RPC_TIMEOUT, - crate::utils::operator_stake_status(&goat_client, &pubkey), - ) - .await - { - Ok(Ok(OperatorStakeStatus::Staked)) => Some(true), - Ok(Ok( - OperatorStakeStatus::NotRegistered | OperatorStakeStatus::Insufficient { .. }, - )) => Some(false), - // Preserve the cached stake verdict on RPC failure. - Ok(Err(error)) => { - tracing::debug!( - event = "p2p_admission", - outcome = "operator_stake_lookup_failed", - peer_id = %peer, - error = %error, - "failed to confirm operator stake; leaving the binding unverified" - ); - None - } - Err(_) => None, - }; - let change = registry.record_operator_stake(&peer, &operator_key, staked, Instant::now()); - persist_registration_change( - &local_db, - &peer, - REGISTERED_KIND_OPERATOR, - Some(&operator_key), - change, - ) - .await; - }); -} - -/// Look up a peer this node is connected to; see -/// [`PeerRegistry::plan_neighbour_lookup`]. -pub fn refresh_neighbour_in_background( - local_db: &LocalDB, - goat_client: &Arc, - peer: PeerId, -) { - if peer_registry().plan_neighbour_lookup(&peer, Instant::now()) { - spawn_committee_lookup(local_db, goat_client, peer); - } -} - -/// Start whatever registry lookups are due for `peer` without waiting for them. -pub fn refresh_registry_in_background( - local_db: &LocalDB, - goat_client: &Arc, - peer: Option, -) { - let registry = peer_registry(); - let plan = registry.plan_refresh(peer.as_ref(), Instant::now()); - if plan.verifier_set { - let goat_client = goat_client.clone(); - tokio::spawn(async move { - let fetched = match tokio::time::timeout( - REGISTRY_RPC_TIMEOUT, - goat_client.committee_mana_get_verifiers(), - ) - .await - { - Ok(Ok(verifiers)) => Some(verifiers), - Ok(Err(error)) => { - tracing::warn!( - event = "p2p_admission", - outcome = "verifier_set_refresh_failed", - error = %error, - "failed to refresh the registered verifier set; keeping the previous one" - ); - None - } - Err(_) => { - tracing::warn!( - event = "p2p_admission", - outcome = "verifier_set_refresh_timeout", - "timed out refreshing the registered verifier set; keeping the previous one" - ); - None - } - }; - registry.record_verifier_set(fetched, Instant::now()); - }); - } - if let Some(peer) = peer.filter(|_| plan.committee_peer) { - spawn_committee_lookup(local_db, goat_client, peer); - } -} - -/// Schedule refreshes for known registrations that are due. -pub fn refresh_due_registrations_in_background(local_db: &LocalDB, goat_client: &Arc) { - let registry = peer_registry(); - let now = Instant::now(); - refresh_registry_in_background(local_db, goat_client, None); - for peer in registry.plan_due_committee_refreshes(now) { - spawn_committee_lookup(local_db, goat_client, peer); - } - for peer in take_refused_peers() { - if registry.plan_refused_peer_lookup(&peer, now) { - spawn_committee_lookup(local_db, goat_client, peer); - } - } - for (peer, operator_key) in registry.plan_due_operator_refreshes(now) { - spawn_operator_stake_lookup(local_db, goat_client, peer, operator_key); - } -} - -/// Confirm, off the event loop, that a proven operator binding belongs to a -/// sufficiently staked operator, and cache the verdict for classification. -pub fn refresh_operator_binding_in_background( - local_db: &LocalDB, - goat_client: &Arc, - peer: PeerId, - operator_key: OperatorKey, -) { - if !peer_registry().plan_operator_refresh(&peer, &operator_key, Instant::now()) { - return; - } - spawn_operator_stake_lookup(local_db, goat_client, peer, operator_key); -} - -/// Per-peer token bucket for messages that are dispatched without being queued. -#[derive(Debug)] -pub struct PeerRateLimiter { - buckets: Mutex>, - burst: f64, - refill_per_sec: f64, - max_peers: usize, -} - -impl PeerRateLimiter { - pub fn new(burst: f64, refill_per_sec: f64, max_peers: usize) -> Self { - Self { buckets: Mutex::new(HashMap::new()), burst, refill_per_sec, max_peers } - } - - pub fn allow(&self, peer: &PeerId, now: Instant) -> bool { - let mut buckets = self.buckets.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !buckets.contains_key(peer) && buckets.len() >= self.max_peers { - // A bucket that has refilled completely belongs to an idle peer and - // carries no state worth keeping. - let (burst, refill_per_sec) = (self.burst, self.refill_per_sec); - buckets.retain(|_, bucket| { - let elapsed = now.saturating_duration_since(bucket.refilled_at).as_secs_f64(); - bucket.tokens + elapsed * refill_per_sec < burst - }); - if buckets.len() >= self.max_peers { - return false; - } - } - let bucket = buckets.entry(*peer).or_insert_with(|| RateBucket::full(self.burst, now)); - bucket.refill(self.refill_per_sec, self.burst, now); - if bucket.tokens < 1.0 { - return false; - } - bucket.tokens -= 1.0; - true - } -} - -static IMMEDIATE_LIMITER: LazyLock = LazyLock::new(|| { - PeerRateLimiter::new(IMMEDIATE_BURST, IMMEDIATE_REFILL_PER_SEC, IMMEDIATE_MAX_TRACKED_PEERS) -}); - -pub fn immediate_limiter() -> &'static PeerRateLimiter { - &IMMEDIATE_LIMITER -} - -/// A token bucket carrying both a rate and a burst, refilled lazily on access. -#[derive(Clone, Copy, Debug)] -struct RateBucket { - tokens: f64, - refilled_at: Instant, -} - -impl RateBucket { - fn full(burst: f64, now: Instant) -> Self { - Self { tokens: burst, refilled_at: now } - } - - fn refill(&mut self, rate: f64, burst: f64, now: Instant) { - let elapsed = now.saturating_duration_since(self.refilled_at).as_secs_f64(); - self.tokens = (self.tokens + elapsed * rate).min(burst); - self.refilled_at = now; - } -} - -/// One message costs a whole message token plus `bytes` byte tokens. -#[derive(Clone, Copy, Debug)] -struct DualBucket { - msgs: RateBucket, - bytes: RateBucket, -} - -/// Rate + burst for one tier, in messages and bytes per second. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct TierRate { - pub msg_burst: u32, - pub msg_per_sec: u32, - pub byte_burst: i64, - pub byte_per_sec: i64, -} - -impl TierRate { - fn new_bucket(&self, now: Instant) -> DualBucket { - DualBucket { - msgs: RateBucket::full(self.msg_burst as f64, now), - bytes: RateBucket::full(self.byte_burst as f64, now), - } - } - - /// Refill then test whether one message of `bytes` fits, without spending. - fn peek(&self, bucket: &mut DualBucket, bytes: i64, now: Instant) -> bool { - self.peek_above(bucket, bytes, 0.0, now) - } - - /// Check capacity while retaining the specified fraction of burst tokens. - fn peek_above(&self, bucket: &mut DualBucket, bytes: i64, reserve: f64, now: Instant) -> bool { - bucket.msgs.refill(self.msg_per_sec as f64, self.msg_burst as f64, now); - bucket.bytes.refill(self.byte_per_sec as f64, self.byte_burst as f64, now); - bucket.msgs.tokens - 1.0 >= self.msg_burst as f64 * reserve - && bucket.bytes.tokens - bytes as f64 >= self.byte_burst as f64 * reserve - } - - fn spend(&self, bucket: &mut DualBucket, bytes: i64) { - bucket.msgs.tokens -= 1.0; - bucket.bytes.tokens -= bytes as f64; - } -} - -/// Pre-decode message and byte limits: direct peer, author, unregistered class and global. -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct RateLimits { - pub direct_peer: TierRate, - pub registered_author: TierRate, - pub unregistered_author: TierRate, - pub unregistered_total: TierRate, - /// Forward-only budget for unregistered authors. - pub unregistered_forward: TierRate, - pub global: TierRate, - /// Committee-only reserve in the global and direct-peer budgets. - pub committee_reserve: f64, - pub max_tracked_peers: usize, -} - -impl RateLimits { - pub const fn defaults() -> Self { - const MIB: i64 = 1024 * 1024; - Self { - // Direct-peer rate ceiling. - direct_peer: TierRate { - msg_burst: 16_384, - msg_per_sec: 4_096, - byte_burst: 512 * MIB, - byte_per_sec: 32 * MIB, - }, - // Generous: one GenCircuits is ~11.5 MiB, and an operator may send a - // few graph messages back to back. - registered_author: TierRate { - msg_burst: 8_192, - msg_per_sec: 512, - byte_burst: 256 * MIB, - byte_per_sec: 8 * MIB, - }, - // Room for one maximum-size JSON message and a node's heartbeats, - // not for a stream. - unregistered_author: TierRate { - msg_burst: 32, - msg_per_sec: 2, - byte_burst: 4 * MIB, - byte_per_sec: 128 * 1024, - }, - // Shared budget for all unregistered authors. - unregistered_total: TierRate { - msg_burst: 1_024, - msg_per_sec: 32, - byte_burst: 32 * MIB, - byte_per_sec: MIB, - }, - // Sized for the chain observations honest unregistered nodes - // (watchtowers, challengers) publish, not for bulk. - unregistered_forward: TierRate { - msg_burst: 256, - msg_per_sec: 16, - byte_burst: 8 * MIB, - byte_per_sec: 512 * 1024, - }, - global: TierRate { - msg_burst: 16_384, - msg_per_sec: 4_096, - byte_burst: 512 * MIB, - byte_per_sec: 32 * MIB, - }, - committee_reserve: 0.25, - max_tracked_peers: 8_192, - } - } -} - -/// Idle duration before forgetting a tracked bucket. -const RATE_BUCKET_IDLE_FORGET: Duration = Duration::from_secs(60); -/// Minimum interval between full-table sweeps. -const RATE_TABLE_SWEEP_INTERVAL: Duration = Duration::from_secs(5); - -/// Bounded per-key buckets; excess keys share one overflow bucket. -#[derive(Debug)] -struct BucketTable { - buckets: HashMap, - overflow: DualBucket, - swept_at: Option, -} - -/// Which bucket of a [`BucketTable`] a message is charged to. -#[derive(Clone, Copy, Debug)] -enum BucketSlot { - Tracked(PeerId), - Overflow, -} - -impl BucketTable { - fn new(overflow_tier: TierRate, now: Instant) -> Self { - Self { buckets: HashMap::new(), overflow: overflow_tier.new_bucket(now), swept_at: None } - } - - /// The slot `key` is charged to. `pinned` keys are always tracked: they are - /// bounded by an on-chain registry, not by whoever generates key pairs. - fn slot( - &mut self, - key: &PeerId, - tier: TierRate, - cap: usize, - pinned: bool, - now: Instant, - ) -> BucketSlot { - if !self.buckets.contains_key(key) { - if self.buckets.len() >= cap - && self - .swept_at - .is_none_or(|at| now.saturating_duration_since(at) >= RATE_TABLE_SWEEP_INTERVAL) - { - self.swept_at = Some(now); - self.buckets.retain(|_, bucket| { - now.saturating_duration_since(bucket.msgs.refilled_at) < RATE_BUCKET_IDLE_FORGET - }); - } - if self.buckets.len() >= cap && !pinned { - return BucketSlot::Overflow; - } - self.buckets.insert(*key, tier.new_bucket(now)); - } - BucketSlot::Tracked(*key) - } - - fn get(&self, slot: BucketSlot) -> DualBucket { - match slot { - BucketSlot::Tracked(key) => self.buckets[&key], - BucketSlot::Overflow => self.overflow, - } - } - - fn set(&mut self, slot: BucketSlot, bucket: DualBucket) { - match slot { - BucketSlot::Tracked(key) => { - self.buckets.insert(key, bucket); - } - BucketSlot::Overflow => self.overflow = bucket, - } - } -} - -#[derive(Debug)] -struct RateState { - direct: BucketTable, - authors: BucketTable, - unregistered_total: DualBucket, - unregistered_forward: DualBucket, - global: DualBucket, -} - -/// Multi-tier pre-decode rate budget. -#[derive(Debug)] -pub struct InboundRateLimiter { - state: Mutex, - limits: RateLimits, -} - -impl InboundRateLimiter { - pub fn new(limits: RateLimits) -> Self { - let now = Instant::now(); - Self { - state: Mutex::new(RateState { - direct: BucketTable::new(limits.direct_peer, now), - authors: BucketTable::new(limits.unregistered_author, now), - unregistered_total: limits.unregistered_total.new_bucket(now), - unregistered_forward: limits.unregistered_forward.new_bucket(now), - global: limits.global.new_bucket(now), - }), - limits, - } - } - - /// Charge the relay budget for a message from an unregistered author that - /// this node forwards without storing. - pub fn charge_forward(&self, bytes: i64, now: Instant) -> Result<(), DropReason> { - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let tier = self.limits.unregistered_forward; - let mut bucket = state.unregistered_forward; - let admitted = tier.peek(&mut bucket, bytes, now); - if admitted { - tier.spend(&mut bucket, bytes); - } - state.unregistered_forward = bucket; - admitted.then_some(()).ok_or(DropReason::ForwardRate) - } - - #[cfg(test)] - fn tracked_authors(&self) -> usize { - self.state.lock().unwrap().authors.buckets.len() - } - - /// Check all applicable rate tiers before charging any of them. - pub fn charge( - &self, - direct_peer: &PeerId, - author: &PeerId, - class: P2pInboxAdmissionClass, - bytes: i64, - now: Instant, - ) -> Result<(), DropReason> { - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let author_tier = if class.is_registered() { - self.limits.registered_author - } else { - self.limits.unregistered_author - }; - let cap = self.limits.max_tracked_peers; - let direct_tier = self.limits.direct_peer; - let registered = class.is_registered(); - let direct_slot = state.direct.slot(direct_peer, direct_tier, cap, false, now); - let author_slot = state.authors.slot(author, author_tier, cap, registered, now); - let mut direct = state.direct.get(direct_slot); - let mut author_bucket = state.authors.get(author_slot); - let mut unregistered = state.unregistered_total; - let mut global = state.global; - - let charge_unregistered = !class.is_registered(); - // The shared tiers keep a reserve only the committee may draw on. - let reserve = if class == P2pInboxAdmissionClass::Committee { - 0.0 - } else { - self.limits.committee_reserve - }; - let direct_ok = direct_tier.peek_above(&mut direct, bytes, reserve, now); - let author_ok = author_tier.peek(&mut author_bucket, bytes, now); - let unregistered_ok = !charge_unregistered - || self.limits.unregistered_total.peek(&mut unregistered, bytes, now); - let global_ok = self.limits.global.peek_above(&mut global, bytes, reserve, now); - - // Persist refilled buckets even when admission fails. - let outcome = if !global_ok { - Err(DropReason::GlobalRate) - } else if !unregistered_ok { - Err(DropReason::UnregisteredRate) - } else if !author_ok { - Err(DropReason::AuthorRate) - } else if !direct_ok { - Err(DropReason::DirectPeerRate) - } else { - direct_tier.spend(&mut direct, bytes); - author_tier.spend(&mut author_bucket, bytes); - if charge_unregistered { - self.limits.unregistered_total.spend(&mut unregistered, bytes); - } - self.limits.global.spend(&mut global, bytes); - Ok(()) - }; - state.direct.set(direct_slot, direct); - state.authors.set(author_slot, author_bucket); - state.unregistered_total = unregistered; - state.global = global; - outcome - } -} - -static INBOUND_RATE_LIMITER: LazyLock = - LazyLock::new(|| InboundRateLimiter::new(RateLimits::defaults())); - -pub fn inbound_rate_limiter() -> &'static InboundRateLimiter { - &INBOUND_RATE_LIMITER -} - -#[derive(Debug, Default)] -struct ResponseGateState { - sent_at: Option, - pending: bool, -} - -/// Coalesce NodeInfo requests into one response per cooldown. -#[derive(Debug)] -pub struct ResponseGate { - state: Mutex, - cooldown: Duration, -} - -impl ResponseGate { - pub fn new(cooldown: Duration) -> Self { - Self { state: Mutex::new(ResponseGateState::default()), cooldown } - } - - fn cooled_down(&self, state: &ResponseGateState, now: Instant) -> bool { - state.sent_at.is_none_or(|sent_at| now.saturating_duration_since(sent_at) >= self.cooldown) - } - - /// `true` when the caller should respond now; otherwise the request is - /// remembered for [`Self::take_pending`]. - pub fn try_respond(&self, now: Instant) -> bool { - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if self.cooled_down(&state, now) { - state.sent_at = Some(now); - state.pending = false; - true - } else { - state.pending = true; - false - } - } - - /// `true` when a deferred request is due; the caller then responds. - pub fn take_pending(&self, now: Instant) -> bool { - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if state.pending && self.cooled_down(&state, now) { - state.sent_at = Some(now); - state.pending = false; - true - } else { - false - } - } -} - -static NODE_INFO_RESPONSE_GATE: LazyLock = - LazyLock::new(|| ResponseGate::new(NODE_INFO_RESPONSE_COOLDOWN)); - -pub fn node_info_response_gate() -> &'static ResponseGate { - &NODE_INFO_RESPONSE_GATE -} - -/// Cooldown for duplicate graph-setup ACKs. -const DEDUP_ACK_COOLDOWN: Duration = Duration::from_secs(8); -/// Cooldown for peer liveness updates. -const PEER_TIMESTAMP_COOLDOWN: Duration = Duration::from_secs(30); -const COOLDOWN_GATE_MAX_ENTRIES: usize = 8_192; - -/// Minimum interval between full gate/set sweeps. -const GATE_SWEEP_INTERVAL: Duration = Duration::from_secs(5); - -/// Bounded keyed timestamps; reject new keys when all entries are live. -#[derive(Debug, Default)] -struct BoundedStamps { - stamps: HashMap, - swept_at: Option, -} - -impl BoundedStamps { - fn is_live(&self, key: &str, ttl: Duration, now: Instant) -> bool { - self.stamps.get(key).is_some_and(|at| now.saturating_duration_since(*at) < ttl) - } - - /// Stamp `key` with `now`. Returns false when the table is full of live - /// entries and `key` is not among them. - fn stamp(&mut self, key: &str, ttl: Duration, max_entries: usize, now: Instant) -> bool { - if !self.stamps.contains_key(key) && self.stamps.len() >= max_entries { - if self - .swept_at - .is_none_or(|at| now.saturating_duration_since(at) >= GATE_SWEEP_INTERVAL) - { - self.swept_at = Some(now); - self.stamps.retain(|_, at| now.saturating_duration_since(*at) < ttl); - } - if self.stamps.len() >= max_entries { - return false; - } - } - self.stamps.insert(key.to_string(), now); - true - } -} - -/// Allows an action for a key at most once per cooldown, within a hard bound on -/// its own memory. A key it has no room to track is denied: every action gated -/// here is one the peer retries, and none is worth an unbounded table. -#[derive(Debug)] -pub struct CooldownGate { - seen: Mutex, - cooldown: Duration, - max_entries: usize, -} - -impl CooldownGate { - fn new(cooldown: Duration, max_entries: usize) -> Self { - Self { seen: Mutex::new(BoundedStamps::default()), cooldown, max_entries } - } - - /// Whether `key` is inside its cooldown. Records nothing. - pub fn is_cooling(&self, key: &str, now: Instant) -> bool { - let seen = self.seen.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - seen.is_live(key, self.cooldown, now) - } - - /// `true` when the action for `key` may proceed, recording it as just done. - pub fn allow(&self, key: &str, now: Instant) -> bool { - let mut seen = self.seen.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - !seen.is_live(key, self.cooldown, now) - && seen.stamp(key, self.cooldown, self.max_entries, now) - } - - #[cfg(test)] - fn len(&self) -> usize { - self.seen.lock().unwrap().stamps.len() - } -} - -static DEDUP_ACK_GATE: LazyLock = - LazyLock::new(|| CooldownGate::new(DEDUP_ACK_COOLDOWN, COOLDOWN_GATE_MAX_ENTRIES)); - -pub fn dedup_ack_gate() -> &'static CooldownGate { - &DEDUP_ACK_GATE -} - -static PEER_TIMESTAMP_GATE: LazyLock = - LazyLock::new(|| CooldownGate::new(PEER_TIMESTAMP_COOLDOWN, COOLDOWN_GATE_MAX_ENTRIES)); - -pub fn peer_timestamp_gate() -> &'static CooldownGate { - &PEER_TIMESTAMP_GATE -} - -static NODE_TABLE_PURGED_AT: Mutex> = Mutex::new(None); - -/// `true` at most once per `interval`: when the `node` table purge should run. -pub fn node_table_purge_due(interval: Duration, now: Instant) -> bool { - let mut purged_at = - NODE_TABLE_PURGED_AT.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if purged_at.is_some_and(|at| now.saturating_duration_since(at) < interval) { - return false; - } - *purged_at = Some(now); - true -} - -const STRIKE_WINDOW: Duration = Duration::from_secs(60); -const STRIKE_LIMIT: u32 = 256; -const STRIKE_BAN: Duration = Duration::from_secs(10 * 60); -const STRIKE_MAX_TRACKED_PEERS: usize = 4_096; - -#[derive(Debug, Default)] -struct StrikeState { - /// Window start and strikes in it, per direct peer. - strikes: HashMap, - banned_until: HashMap, -} - -/// Temporary local bans based on attributable relay strikes; registered peers are exempt. -#[derive(Debug, Default)] -pub struct DirectPeerStrikes { - state: Mutex, -} - -impl DirectPeerStrikes { - /// Count one strike against `peer`. Returns `true` when it just reached the - /// limit and should be banned now. - pub fn strike(&self, peer: &PeerId, now: Instant) -> bool { - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if state.banned_until.contains_key(peer) { - return false; - } - if !state.strikes.contains_key(peer) && state.strikes.len() >= STRIKE_MAX_TRACKED_PEERS { - state.strikes.retain(|_, (started_at, _)| { - now.saturating_duration_since(*started_at) < STRIKE_WINDOW - }); - if state.strikes.len() >= STRIKE_MAX_TRACKED_PEERS { - return false; - } - } - let (started_at, count) = state.strikes.entry(*peer).or_insert((now, 0)); - if now.saturating_duration_since(*started_at) >= STRIKE_WINDOW { - (*started_at, *count) = (now, 0); - } - *count += 1; - if *count < STRIKE_LIMIT { - return false; - } - state.strikes.remove(peer); - state.banned_until.insert(*peer, now + STRIKE_BAN); - true - } - - /// Check the peer's current ban for each connection. - pub fn is_banned(&self, peer: &PeerId, now: Instant) -> bool { - let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - state.banned_until.get(peer).is_some_and(|until| now < *until) - } - - /// Peers whose ban has run out, removed from the ban list. - pub fn take_expired_bans(&self, now: Instant) -> Vec { - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let expired: Vec = state - .banned_until - .iter() - .filter(|(_, until)| now >= **until) - .map(|(peer, _)| *peer) - .collect(); - for peer in &expired { - state.banned_until.remove(peer); - } - expired - } -} - -static DIRECT_PEER_STRIKES: LazyLock = LazyLock::new(DirectPeerStrikes::default); - -pub fn direct_peer_strikes() -> &'static DirectPeerStrikes { - &DIRECT_PEER_STRIKES -} - -const REFUSED_PEERS_MAX: usize = 256; - -static REFUSED_PEERS: Mutex> = Mutex::new(Vec::new()); - -/// Queue refused peers for committee lookup on the next maintenance pass. -pub fn note_refused_peer(peer: PeerId) { - let mut refused = REFUSED_PEERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if refused.len() < REFUSED_PEERS_MAX && !refused.contains(&peer) { - refused.push(peer); - } -} - -fn take_refused_peers() -> Vec { - std::mem::take(&mut *REFUSED_PEERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner)) -} - -/// How often a protocol message this node has already stored is published again. -const PROTOCOL_REPUBLISH_COOLDOWN: Duration = Duration::from_secs(30); - -static PROTOCOL_REPUBLISH_GATE: LazyLock = - LazyLock::new(|| CooldownGate::new(PROTOCOL_REPUBLISH_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); - -/// Throttle stored-value recovery; stamp only on success. -/// A full gate leaves recovery open; live outbox rows retain their own retry schedule. -pub fn protocol_republish_gate() -> &'static CooldownGate { - &PROTOCOL_REPUBLISH_GATE -} - -/// How long a graph a relay just answered a sync request for stays on cooldown. -const SYNC_GRAPH_RESPONSE_COOLDOWN: Duration = Duration::from_secs(60); -/// TTL for locally requested graph responses. -const SYNC_GRAPH_REQUEST_TTL: Duration = Duration::from_secs(15 * 60); -const GRAPH_SET_MAX_ENTRIES: usize = 16_384; - -/// A global token bucket, for a resource no per-peer key bounds. -#[derive(Debug)] -pub struct GlobalRate { - bucket: Mutex, - rate: f64, - burst: f64, -} - -impl GlobalRate { - fn new(burst: f64, rate: f64) -> Self { - Self { bucket: Mutex::new(RateBucket::full(burst, Instant::now())), rate, burst } - } - - pub fn allow(&self, now: Instant) -> bool { - let mut bucket = self.bucket.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - bucket.refill(self.rate, self.burst, now); - if bucket.tokens < 1.0 { - return false; - } - bucket.tokens -= 1.0; - true - } -} - -/// A bounded set of keys, each valid until its TTL lapses. -#[derive(Debug)] -pub struct TtlSet { - entries: Mutex, - ttl: Duration, - max_entries: usize, -} - -impl TtlSet { - fn new(ttl: Duration, max_entries: usize) -> Self { - Self { entries: Mutex::new(BoundedStamps::default()), ttl, max_entries } - } - - /// Returns false when the set is full of unexpired keys and has no room. - pub fn insert(&self, key: &str, now: Instant) -> bool { - let mut entries = self.entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - entries.stamp(key, self.ttl, self.max_entries, now) - } - - /// Whether `key` is still present and unexpired. - pub fn contains(&self, key: &str, now: Instant) -> bool { - let entries = self.entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - entries.is_live(key, self.ttl, now) - } - - pub fn remove(&self, key: &str) { - let mut entries = self.entries.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - entries.stamps.remove(key); - } -} - -static SYNC_GRAPH_RESPONSE_GATE: LazyLock = - LazyLock::new(|| CooldownGate::new(SYNC_GRAPH_RESPONSE_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); - -/// Per-graph cooldown for answering `SyncGraphRequest` on the relay side. -pub fn sync_graph_response_gate() -> &'static CooldownGate { - &SYNC_GRAPH_RESPONSE_GATE -} - -// Separate aggregate response budgets for registered and unregistered requesters. -static SYNC_GRAPH_RESPONSE_BUDGET_REGISTERED: LazyLock = - LazyLock::new(|| GlobalRate::new(32.0, 2.0)); -static SYNC_GRAPH_RESPONSE_BUDGET_UNREGISTERED: LazyLock = - LazyLock::new(|| GlobalRate::new(8.0, 0.2)); - -/// Aggregate SyncGraph response budgets by registration class. -pub fn sync_graph_response_budget(requester_registered: bool) -> &'static GlobalRate { - if requester_registered { - &SYNC_GRAPH_RESPONSE_BUDGET_REGISTERED - } else { - &SYNC_GRAPH_RESPONSE_BUDGET_UNREGISTERED - } -} - -static REQUESTED_GRAPHS: LazyLock = - LazyLock::new(|| TtlSet::new(SYNC_GRAPH_REQUEST_TTL, GRAPH_SET_MAX_ENTRIES)); - -/// Outstanding local graph requests. -pub fn requested_graphs() -> &'static TtlSet { - &REQUESTED_GRAPHS -} - -/// How often one requested graph may be put through validation. A `SyncGraph` -/// costs chain queries, a graph rebuild and two signature sweeps; a sender that -/// keeps answering the same request with a bad graph gets one try per window. -const SYNC_GRAPH_VALIDATION_COOLDOWN: Duration = Duration::from_secs(30); - -static SYNC_GRAPH_VALIDATION_GATE: LazyLock = - LazyLock::new(|| CooldownGate::new(SYNC_GRAPH_VALIDATION_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); - -pub fn sync_graph_validation_gate() -> &'static CooldownGate { - &SYNC_GRAPH_VALIDATION_GATE -} - -/// Aggregate drop counters for periodic logging. -#[derive(Debug, Default)] -pub struct DropStats { - counts: Mutex>, -} - -impl DropStats { - pub fn record(&self, reason: DropReason) { - let mut counts = self.counts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - *counts.entry(reason).or_default() += 1; - } - - pub fn take(&self) -> Vec<(DropReason, u64)> { - let mut counts = self.counts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let mut taken: Vec<_> = counts.drain().collect(); - taken.sort_by_key(|(reason, _)| reason.as_str()); - taken - } -} - -static DROP_STATS: LazyLock = LazyLock::new(DropStats::default); - -pub fn record_drop(reason: DropReason) { - DROP_STATS.record(reason); -} - -/// Count an admitted message from an unregistered sender for the summary line. -/// Those are not logged one by one: whoever floods decides how many there are. -pub fn record_admitted_unregistered() { - DROP_STATS.record(DropReason::AdmittedUnregistered); -} - -/// Emit one line for everything dropped since the previous call. -pub fn log_drop_summary() { - let dropped = DROP_STATS.take(); - if dropped.is_empty() { - return; - } - let total: u64 = dropped.iter().map(|(_, count)| count).sum(); - let breakdown = dropped - .iter() - .map(|(reason, count)| format!("{}={count}", reason.as_str())) - .collect::>() - .join(" "); - tracing::warn!( - event = "p2p_admission", - outcome = "summary", - total, - breakdown, - "inbound P2P messages dropped, de-duplicated or admitted unlogged by admission control" - ); -} - -/// Number of sequence positions retained below the high-water mark. -const REPLAY_WINDOW: u64 = 1_024; -const REPLAY_MAX_TRACKED_AUTHORS: usize = 16_384; - -/// Maximum accepted sequence-number lead over local wall time. -const REPLAY_MAX_FUTURE: Duration = Duration::from_secs(10 * 60); -/// Below-floor rejections of one author per tick that are worth an operator's -/// attention; see [`ReplayGuard::take_lockout_suspects`]. -const REPLAY_LOCKOUT_SUSPECT_REJECTIONS: u32 = 8; - -#[derive(Debug, Default)] -struct AuthorSequence { - highest: u64, - /// Sequence numbers seen within `REPLAY_WINDOW` of `highest`. - recent: std::collections::BTreeSet, - /// Persisted rejection floor restored at startup. - floor: Option, - /// The mark the store is known to hold. `highest` above it is still owed. - persisted: u64, - /// Rejection count since the last report; diagnostic only. - rejected: u32, -} - -/// Per-registered-author sequence window with periodically persisted high-water marks. -/// Accept unseen numbers within the window; restored marks are rejection floors. -/// Authors must keep numbering monotonic across restarts. -/// `P2P_REPLAY_MARK_RESET_PEERS` clears selected persisted marks at startup. -#[derive(Debug, Default)] -pub struct ReplayGuard { - authors: Mutex>, - /// How far each registered author's numbering ran ahead of the local clock - /// since the last report; see [`Self::take_clock_leads`]. - clock_leads: Mutex>, -} - -/// Authors remembered per tick for the clock report. It is a diagnostic, and the -/// table must stay small whatever arrives. -const CLOCK_LEADS_MAX_AUTHORS: usize = 64; - -impl ReplayGuard { - /// Record `sequence_number` for `author`, or refuse it. `unix_nanos` is the - /// local wall clock. - pub fn admit( - &self, - author: &PeerId, - sequence_number: Option, - unix_nanos: u64, - ) -> Result<(), DropReason> { - // Require a signed sequence number. - let sequence_number = sequence_number.ok_or(DropReason::ReplayedSequence)?; - if sequence_number > unix_nanos { - self.note_clock_lead(author, sequence_number - unix_nanos); - } - if sequence_number > unix_nanos.saturating_add(REPLAY_MAX_FUTURE.as_nanos() as u64) { - return Err(DropReason::FutureSequence); - } - let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if !authors.contains_key(author) && authors.len() >= REPLAY_MAX_TRACKED_AUTHORS { - // Only reachable if the registries outgrow the table; forgetting an - // author merely re-opens its window once. - if let Some(evicted) = authors.keys().next().copied() { - authors.remove(&evicted); - } - } - let state = authors.entry(*author).or_default(); - if state.floor.is_some_and(|floor| sequence_number <= floor) { - state.rejected = state.rejected.saturating_add(1); - return Err(DropReason::ReplayedSequence); - } - if sequence_number > state.highest { - state.highest = sequence_number; - let floor = sequence_number.saturating_sub(REPLAY_WINDOW); - state.recent = state.recent.split_off(&floor); - } else if state.highest - sequence_number >= REPLAY_WINDOW - || state.recent.contains(&sequence_number) - { - state.rejected = state.rejected.saturating_add(1); - return Err(DropReason::ReplayedSequence); - } - state.recent.insert(sequence_number); - Ok(()) - } - - fn note_clock_lead(&self, author: &PeerId, lead_nanos: u64) { - let mut leads = self.clock_leads.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if leads.len() < CLOCK_LEADS_MAX_AUTHORS || leads.contains_key(author) { - let lead = leads.entry(*author).or_default(); - *lead = (*lead).max(lead_nanos); - } - } - - /// Return and clear each registered author's maximum clock lead since the previous call. - pub fn take_clock_leads(&self) -> Vec<(PeerId, Duration)> { - let mut leads = self.clock_leads.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - leads.drain().map(|(author, lead)| (author, Duration::from_nanos(lead))).collect() - } - - /// Restore an author's persisted mark. Call before any message is admitted. - pub fn restore_mark(&self, author: &PeerId, highest: u64) { - let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if authors.len() >= REPLAY_MAX_TRACKED_AUTHORS { - return; - } - let state = authors.entry(*author).or_default(); - state.highest = state.highest.max(highest); - state.persisted = state.persisted.max(highest); - state.floor = Some(state.floor.map_or(highest, |floor| floor.max(highest))); - } - - /// Marks awaiting successful persistence confirmation. - pub fn pending_marks(&self) -> Vec<(PeerId, u64)> { - let authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - authors - .iter() - .filter(|(_, state)| state.highest > state.persisted) - .map(|(author, state)| (*author, state.highest)) - .collect() - } - - /// Acknowledge marks that were written. A mark that has moved on since it - /// was read stays owed for the difference. - pub fn confirm_persisted(&self, marks: &[(PeerId, u64)]) { - let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - for (author, written) in marks { - if let Some(state) = authors.get_mut(author) { - state.persisted = state.persisted.max(*written); - } - } - } - - /// Return authors exceeding the rejection reporting threshold. - pub fn take_lockout_suspects(&self) -> Vec<(PeerId, u32)> { - let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - authors - .iter_mut() - .filter_map(|(author, state)| { - let rejected = std::mem::take(&mut state.rejected); - (rejected >= REPLAY_LOCKOUT_SUSPECT_REJECTIONS).then_some((*author, rejected)) - }) - .collect() - } -} - -/// The local wall clock in the unit gossipsub numbers its messages in. -pub fn unix_nanos_now() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_nanos().min(u128::from(u64::MAX)) as u64) -} - -pub fn replay_guard() -> &'static ReplayGuard { - &REPLAY_GUARD -} - -static REPLAY_GUARD: LazyLock = LazyLock::new(ReplayGuard::default); - -/// How long the per-class inbox totals are served from memory. -const INBOX_CLASS_TOTALS_TTL: Duration = Duration::from_secs(2); - -/// Cache per-class queue totals and increment them on admission. -/// Worker removals are reflected at the next cache refresh. -#[derive(Debug, Default)] -pub struct InboxUsageCache { - totals: Mutex)>>, -} - -impl InboxUsageCache { - fn fresh(&self, now: Instant) -> Option> { - let totals = self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - totals - .as_ref() - .filter(|(at, _)| now.saturating_duration_since(*at) < INBOX_CLASS_TOTALS_TTL) - .map(|(_, totals)| totals.clone()) - } - - fn store(&self, totals: Vec, now: Instant) { - *self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = - Some((now, totals)); - } - - /// Count a row that is about to be queued. - fn note_admitted(&self, class: P2pInboxAdmissionClass, bytes: i64) { - let mut totals = self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let Some((_, totals)) = totals.as_mut() else { - return; - }; - let class = class.to_string(); - match totals.iter_mut().find(|usage| usage.admission_class == class) { - Some(usage) => { - usage.rows += 1; - usage.bytes += bytes; - } - None => totals.push(P2pInboxClassUsage { - admission_class: class, - rows: 1, - bytes, - ..Default::default() - }), - } - } - - /// Combine cached class totals with fresh per-peer usage. - async fn usage( - &self, - storage: &mut store::localdb::StorageProcessor<'_>, - from_peer: &str, - now: Instant, - ) -> Result> { - let mut usage = match self.fresh(now) { - Some(totals) => totals, - None => { - let totals = storage.p2p_inbox_class_totals().await?; - self.store(totals.clone(), now); - totals - } - }; - for peer in storage.p2p_inbox_peer_usage(from_peer).await? { - match usage.iter_mut().find(|total| total.admission_class == peer.admission_class) { - Some(total) => { - total.peer_rows = peer.peer_rows; - total.peer_bytes = peer.peer_bytes; - } - None => usage.push(peer), - } - } - Ok(usage) - } -} - -static INBOX_USAGE_CACHE: LazyLock = LazyLock::new(InboxUsageCache::default); - -/// Injectable admission gates. -#[derive(Clone, Copy)] -pub struct AdmissionGates<'a> { - pub registry: &'a PeerRegistry, - pub replay_guard: &'a ReplayGuard, - pub rate_limiter: &'a InboundRateLimiter, - pub immediate_limiter: &'a PeerRateLimiter, - pub usage_cache: &'a InboxUsageCache, - pub limits: &'a InboundLimits, - /// This node's role. A durable message no handler of this role acts on is - /// relayed but not stored. - pub local_actor: &'a bitvm_lib::actors::Actor, - /// Graphs this node asked a relayer for. A `SyncGraph` for anything else is - /// somebody else's answer. - pub requested_graphs: &'a TtlSet, -} - -impl AdmissionGates<'_> { - /// Whether this node stores an admitted durable message of `kind`, or only - /// relays it. `synced_graph` is the graph a `SyncGraph` carries. - pub fn keeps( - &self, - kind: crate::action::MessageKind, - synced_graph: Option, - now: Instant, - ) -> bool { - kind.handled_by(self.local_actor) - && synced_graph - .is_none_or(|graph_id| self.requested_graphs.contains(&graph_id.to_string(), now)) - } -} - -impl<'a> AdmissionGates<'a> { - /// The process-wide gates the node runs with. - pub fn global(local_actor: &'a bitvm_lib::actors::Actor) -> Self { - Self { - local_actor, - requested_graphs: requested_graphs(), - registry: peer_registry(), - replay_guard: &REPLAY_GUARD, - rate_limiter: inbound_rate_limiter(), - immediate_limiter: immediate_limiter(), - usage_cache: &INBOX_USAGE_CACHE, - limits: inbound_limits(), - } - } -} - -/// One received gossipsub message, as admission sees it. -#[derive(Clone, Copy, Debug)] -pub struct InboundGossip<'a> { - /// The neighbour that relayed the message to this node. - pub direct_peer: &'a PeerId, - /// The signed author. - pub source: &'a PeerId, - /// The author's gossipsub sequence number, covered by its signature. - pub sequence_number: Option, - pub data: &'a [u8], -} - -/// What to do with one inbound gossipsub message. -pub enum InboundVerdict { - /// Neither processed nor forwarded. - Drop(DropReason), - /// Dispatch now; nothing is persisted. - Immediate(GOATMessage), - /// Forward without persistence, dispatch or ACK. - Forward, - /// Persist for the inbox worker. - Enqueue { message: GOATMessage, class: P2pInboxAdmissionClass, content_hash: [u8; 32] }, - /// The same sender already has this exact payload queued. - Duplicate(GOATMessage), -} - -/// Evaluate admission without swarm or chain access; propagate database errors. -pub async fn evaluate_inbound_message( - local_db: &LocalDB, - gates: &AdmissionGates<'_>, - inbound: &InboundGossip<'_>, - now: Instant, -) -> Result { - let InboundGossip { direct_peer, source, sequence_number, data } = *inbound; - if let Err(reason) = check_envelope(data, gates.registry.is_verifier(source), gates.limits) { - return Ok(InboundVerdict::Drop(reason)); - } - // Classification reads only the cache and never blocks. - let class = gates.registry.sender_class(source, now); - // Check replay before charging the registered author's rate budget. - if class.is_registered() - && let Err(reason) = gates.replay_guard.admit(source, sequence_number, unix_nanos_now()) - { - return Ok(InboundVerdict::Drop(reason)); - } - // Apply rate limits before decoding. - if let Err(reason) = - gates.rate_limiter.charge(direct_peer, source, class, data.len() as i64, now) - { - return Ok(InboundVerdict::Drop(reason)); - } - let Ok(message) = GOATMessage::deserialize_message(data).await else { - return Ok(InboundVerdict::Drop(DropReason::Undecodable)); - }; - // Require GenCircuits inside the binary envelope. - if GOATMessage::is_binary_envelope(data) - && !matches!(message.content, GOATMessageContent::GenCircuits(_)) - { - return Ok(InboundVerdict::Drop(DropReason::UnexpectedBinaryKind)); - } - - // Reject only definite sender-role denials; unknown identities remain subject to unregistered limits. - let sender_role = message.content.kind().sender_role(); - if gates.registry.role_verdict(source, sender_role, now) == RoleVerdict::Denied { - return Ok(InboundVerdict::Drop(DropReason::SenderRoleDenied)); - } - - if message.content.p2p_delivery() == P2PMessageDelivery::Immediate { - if !gates.immediate_limiter.allow(source, now) { - return Ok(InboundVerdict::Drop(DropReason::ImmediateRateLimited)); - } - return Ok(InboundVerdict::Immediate(message)); - } - - // Forward unsolicited SyncGraph responses without storing them locally. - let synced_graph = match &message.content { - GOATMessageContent::SyncGraph(sync) => Some(sync.graph_id), - _ => None, - }; - if !gates.keeps(message.content.kind(), synced_graph, now) { - // Apply the unregistered forwarding budget. - if !class.is_registered() - && let Err(reason) = gates.rate_limiter.charge_forward(data.len() as i64, now) - { - return Ok(InboundVerdict::Drop(reason)); - } - return Ok(InboundVerdict::Forward); - } - - let content_hash = content_hash(data); - let from_peer = source.to_string(); - let mut storage = local_db.acquire().await?; - if storage.has_queued_p2p_inbox_payload(&from_peer, &content_hash).await? { - return Ok(InboundVerdict::Duplicate(message)); - } - let usage = gates.usage_cache.usage(&mut storage, &from_peer, now).await?; - if let Err(reason) = check_inbox_quota(class, data.len(), &usage, gates.limits) { - return Ok(InboundVerdict::Drop(reason)); - } - gates.usage_cache.note_admitted(class, data.len() as i64); - Ok(InboundVerdict::Enqueue { message, class, content_hash }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::action::KickoffSent; - use bitvm_lib::actors::Actor; - use uuid::Uuid; - - const SMALL: InboundLimits = InboundLimits { - max_json_bytes: 1024, - max_binary_bytes: 4096, - global: Quota { rows: 6, bytes: 6000 }, - committee_reserve: Quota { rows: 0, bytes: 0 }, - unregistered_class: Quota { rows: 4, bytes: 3000 }, - registered_peer: Quota { rows: 3, bytes: 5000 }, - unregistered_peer: Quota { rows: 2, bytes: 2000 }, - }; - - fn usage( - class: P2pInboxAdmissionClass, - rows: i64, - bytes: i64, - peer: (i64, i64), - ) -> P2pInboxClassUsage { - P2pInboxClassUsage { - admission_class: class.to_string(), - rows, - bytes, - peer_rows: peer.0, - peer_bytes: peer.1, - } - } - - /// Use isolated gates in tests. - struct TestGates { - registry: PeerRegistry, - replay_guard: ReplayGuard, - rate_limiter: InboundRateLimiter, - immediate_limiter: PeerRateLimiter, - usage_cache: InboxUsageCache, - local_actor: Actor, - requested_graphs: TtlSet, - } - - /// A wall clock comfortably ahead of every sequence number the tests use. - const WALL_CLOCK: u64 = 1_800_000_000 * 1_000_000_000; - - /// A distinct operator key per `index`. The registry never parses it. - fn key(index: u32) -> OperatorKey { - let mut key = [0x11; 32]; - key[..4].copy_from_slice(&index.to_be_bytes()); - key - } - - impl TestGates { - fn new(immediate_limiter: PeerRateLimiter) -> Self { - Self { - requested_graphs: TtlSet::new(SYNC_GRAPH_REQUEST_TTL, 16), - local_actor: Actor::Committee, - registry: PeerRegistry::default(), - replay_guard: ReplayGuard::default(), - rate_limiter: InboundRateLimiter::new(RateLimits::defaults()), - immediate_limiter, - usage_cache: InboxUsageCache::default(), - } - } - - /// Evaluate `data` as authored and relayed by `source`. - async fn evaluate( - &self, - local_db: &LocalDB, - limits: &InboundLimits, - source: &PeerId, - sequence_number: u64, - data: &[u8], - now: Instant, - ) -> InboundVerdict { - let gates = AdmissionGates { - registry: &self.registry, - replay_guard: &self.replay_guard, - rate_limiter: &self.rate_limiter, - immediate_limiter: &self.immediate_limiter, - usage_cache: &self.usage_cache, - limits, - local_actor: &self.local_actor, - requested_graphs: &self.requested_graphs, - }; - let inbound = InboundGossip { - direct_peer: source, - source, - sequence_number: Some(sequence_number), - data, - }; - evaluate_inbound_message(local_db, &gates, &inbound, now).await.unwrap() - } - } - - async fn kickoff_sent_bytes() -> Vec { - GOATMessage::new( - Actor::Committee, - GOATMessageContent::KickoffSent(KickoffSent { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - }), - ) - .serialize_message() - .await - .unwrap() - } - - #[test] - fn envelope_limits_depend_on_the_encoding_and_the_sender() { - let json = vec![b'{'; 1025]; - assert_eq!(check_envelope(&json, true, &SMALL), Err(DropReason::OversizedJson)); - assert_eq!(check_envelope(&json[..1024], false, &SMALL), Ok(())); - - let mut binary = b"GOATBIN1".to_vec(); - binary.resize(2048, 0); - assert_eq!(check_envelope(&binary, false, &SMALL), Err(DropReason::BinaryFromNonVerifier)); - assert_eq!(check_envelope(&binary, true, &SMALL), Ok(()), "above the JSON ceiling"); - binary.resize(4097, 0); - assert_eq!(check_envelope(&binary, true, &SMALL), Err(DropReason::OversizedBinary)); - } - - /// Verify GenCircuits fits the binary transport ceiling. - #[tokio::test] - async fn gen_circuits_fits_the_binary_envelope() { - use crate::action::GenCircuits; - use bitvm_lib::babe_adapter::{BABE_N_CC, build_setup_package}; - - let secp = bitcoin::secp256k1::Secp256k1::new(); - let secret = bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(); - let message = GOATMessage::new( - Actor::Operator, - GOATMessageContent::GenCircuits(GenCircuits { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - verifier_pubkey: bitcoin::PublicKey::new(secret.public_key(&secp)), - setup_package: build_setup_package(BABE_N_CC).unwrap(), - }), - ); - let bytes = message.serialize_message().await.unwrap(); - assert!(GOATMessage::is_binary_envelope(&bytes)); - let limits = InboundLimits::with_queued_bytes(2 << 20, 2 << 30); - assert!(bytes.len() > limits.max_json_bytes, "GenCircuits is {} bytes", bytes.len()); - assert!( - bytes.len() * 5 <= limits.max_binary_bytes * 4, - "GenCircuits is {} bytes, within 20% of the {} byte ceiling", - bytes.len(), - limits.max_binary_bytes - ); - assert_eq!(check_envelope(&bytes, true, &limits), Ok(())); - assert!( - (bytes.len() as i64) * 8 <= limits.registered_peer.bytes, - "a verifier must be able to have several setups queued" - ); - } - - /// Verify graph-bearing JSON fits the configured verifier-slot capacity. - #[test] - fn default_json_ceiling_fits_graphs_with_several_verifier_slots() { - use bitvm_lib::types::BitvmGcCircuitData; - use goat::assert_scripts::WireHash; - - let secp = bitcoin::secp256k1::Secp256k1::new(); - let secret = bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(); - // 0xff is the widest byte in JSON's decimal array encoding. - let slot = BitvmGcCircuitData { - verifier_pubkey: bitcoin::PublicKey::new(secret.public_key(&secp)), - final_msg_hashlocks: vec![[0xff; 20]; bitvm_lib::babe_adapter::BABE_M_CC], - wire_hashes: std::array::from_fn(|_| WireHash { - true_label_hash: [0xff; 20], - false_label_hash: [0xff; 20], - }), - }; - let slot_len = serde_json::to_vec(&slot).unwrap().len(); - assert!( - slot_len * 8 <= crate::env::DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES, - "one GC slot is {slot_len} bytes of JSON" - ); - } - - #[test] - fn quota_is_charged_per_sender_per_class_and_globally() { - use P2pInboxAdmissionClass::{Registered, Unregistered}; - - assert_eq!(check_inbox_quota(Unregistered, 100, &[], &SMALL), Ok(())); - - // The sender's own rows, in whichever class they were admitted. - let own = [usage(Unregistered, 1, 100, (1, 100)), usage(Registered, 1, 100, (1, 100))]; - assert_eq!( - check_inbox_quota(Unregistered, 100, &own, &SMALL), - Err(DropReason::SenderQuota) - ); - assert_eq!(check_inbox_quota(Registered, 100, &own, &SMALL), Ok(())); - let own_bytes = [usage(Unregistered, 1, 1950, (1, 1950))]; - assert_eq!( - check_inbox_quota(Unregistered, 100, &own_bytes, &SMALL), - Err(DropReason::SenderQuota) - ); - - // Other unregistered senders exhaust the shared pool; a registered - // sender is not charged against it. - let crowded = [usage(Unregistered, 4, 400, (0, 0))]; - assert_eq!( - check_inbox_quota(Unregistered, 100, &crowded, &SMALL), - Err(DropReason::UnregisteredClassQuota) - ); - assert_eq!(check_inbox_quota(Registered, 100, &crowded, &SMALL), Ok(())); - - // The global ceiling counts every class, including pre-migration rows. - let full = [usage(Unregistered, 3, 300, (0, 0)), usage(Registered, 3, 300, (0, 0))]; - assert_eq!(check_inbox_quota(Registered, 100, &full, &SMALL), Err(DropReason::GlobalQuota)); - let heavy = [usage(Registered, 1, 5950, (0, 0))]; - assert_eq!( - check_inbox_quota(Registered, 100, &heavy, &SMALL), - Err(DropReason::GlobalQuota) - ); - } - - #[test] - fn derived_quotas_nest_inside_the_global_ceiling() { - let limits = InboundLimits::with_queued_bytes(2 << 20, 2 << 30); - assert!(limits.unregistered_peer.bytes < limits.unregistered_class.bytes); - assert!(limits.unregistered_class.bytes < limits.global.bytes); - assert!(limits.registered_peer.bytes < limits.global.bytes); - assert!(limits.unregistered_peer.rows < limits.unregistered_class.rows); - assert!(limits.unregistered_class.rows < limits.global.rows); - assert!(limits.max_json_bytes as i64 <= limits.unregistered_peer.bytes); - } - - #[test] - fn registry_classifies_from_cache_and_never_blocks() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - let (verifier, member, stranger) = (PeerId::random(), PeerId::random(), PeerId::random()); - - // Nothing is known yet: everyone is unregistered, and lookups are due. - assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Unregistered); - assert_eq!( - registry.plan_refresh(Some(&member), now), - RefreshPlan { verifier_set: true, committee_peer: true } - ); - assert_eq!( - registry.plan_refresh(Some(&member), now), - RefreshPlan::default(), - "a lookup already in flight is not started twice" - ); - - registry.record_verifier_set(Some(vec![verifier.to_bytes()]), now); - registry.record_committee_peer(&member, Some(true), now); - assert!(registry.is_verifier(&verifier)); - assert_eq!(registry.sender_class(&verifier, now), P2pInboxAdmissionClass::Registered); - assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Committee); - assert!(registry.is_committee(&member, now)); - assert!(!registry.is_committee(&verifier, now), "registered, but not the committee"); - assert_eq!(registry.sender_class(&stranger, now), P2pInboxAdmissionClass::Unregistered); - assert_eq!( - registry.plan_refresh(Some(&verifier), now), - RefreshPlan::default(), - "a verifier needs no committee lookup" - ); - - // Past its TTL the member is re-validated but keeps its class meanwhile. - // The verifier set has aged out by then as well. - let later = now + COMMITTEE_POSITIVE_TTL; - assert_eq!(registry.sender_class(&member, later), P2pInboxAdmissionClass::Committee); - assert_eq!( - registry.plan_refresh(Some(&member), later), - RefreshPlan { verifier_set: true, committee_peer: true } - ); - registry.record_committee_peer(&member, None, later); - assert_eq!( - registry.sender_class(&member, later), - P2pInboxAdmissionClass::Committee, - "a failed lookup leaves the cached answer in place" - ); - assert_eq!( - registry.sender_class(&member, now + COMMITTEE_STALE_LIMIT), - P2pInboxAdmissionClass::Unregistered, - "but it is not trusted forever" - ); - - // A failed verifier refresh keeps the previous set and is retried soon, - // but not on every message. - registry.record_verifier_set(None, later); - assert!(registry.is_verifier(&verifier)); - assert!(!registry.plan_refresh(None, later + Duration::from_secs(1)).verifier_set); - assert!(registry.plan_refresh(None, later + VERIFIER_SET_RETRY_INTERVAL).verifier_set); - } - - #[test] - fn unknown_identities_cannot_starve_revalidation_of_known_members() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - registry.record_verifier_set(Some(vec![]), now); - let member = PeerId::random(); - registry.record_committee_peer(&member, Some(true), now); - - let later = now + COMMITTEE_POSITIVE_TTL; - let mut started = 0; - for _ in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { - let stranger = PeerId::random(); - if registry.plan_refresh(Some(&stranger), later).committee_peer { - started += 1; - registry.record_committee_peer(&stranger, Some(false), later); - } - } - assert_eq!(started, COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE); - assert!( - registry.plan_refresh(Some(&member), later).committee_peer, - "re-validating a registered peer is not charged to the discovery budget" - ); - } - - /// Item 5: even with the unknown-lookup slots saturated and in flight, a - /// known member's re-check must still find a reserved slot. - #[test] - fn known_member_recheck_survives_saturated_unknown_lookups() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - registry.record_verifier_set(Some(vec![]), now); - let member = PeerId::random(); - registry.record_committee_peer(&member, Some(true), now); - - // Fill and hold the unknown-lookup slots (do not resolve them). - let mut unknown_in_flight = 0; - for _ in 0..COMMITTEE_MAX_IN_FLIGHT_LOOKUPS * 2 { - if registry.plan_refresh(Some(&PeerId::random()), now).committee_peer { - unknown_in_flight += 1; - } - } - assert_eq!( - unknown_in_flight, COMMITTEE_MAX_UNKNOWN_IN_FLIGHT, - "unknown lookups are capped below the total, leaving slots reserved" - ); - let later = now + COMMITTEE_POSITIVE_TTL; - assert!( - registry.plan_refresh(Some(&member), later).committee_peer, - "a known member re-check uses a reserved slot despite the unknown flood" - ); - } - - /// An operator becomes registered only once its proven binding is confirmed - /// to be a staked operator, and a key maps to one peer id at a time. - #[test] - fn operator_binding_grants_registered_only_after_stake_confirmation() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - let peer = PeerId::random(); - let pubkey = &key(1); - - registry.observe_operator_binding(&peer, pubkey, 1); - assert_eq!( - registry.sender_class(&peer, now), - P2pInboxAdmissionClass::Unregistered, - "a proven binding alone is not enough; stake is unconfirmed" - ); - assert!(registry.plan_operator_refresh(&peer, pubkey, now)); - assert!( - !registry.plan_operator_refresh(&peer, pubkey, now), - "a lookup already in flight is not started twice" - ); - registry.record_operator_stake(&peer, pubkey, Some(true), now); - assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); - - // Verify one key registers only one peer. - let new_peer = PeerId::random(); - registry.observe_operator_binding(&new_peer, pubkey, 2); - registry.record_operator_stake(&new_peer, pubkey, Some(true), now); - assert_eq!(registry.sender_class(&new_peer, now), P2pInboxAdmissionClass::Registered); - assert_eq!( - registry.sender_class(&peer, now), - P2pInboxAdmissionClass::Unregistered, - "the superseded peer id loses the class" - ); - - // An older binding for a key already owned by another peer is ignored. - registry.observe_operator_binding(&peer, pubkey, 1); - assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Unregistered); - } - - /// The rate budget rejects a flood before decode, charges every tier, and - /// only debits when all tiers pass. - #[test] - fn inbound_rate_budget_bounds_bytes_and_messages_per_tier() { - const MIB: i64 = 1024 * 1024; - let limits = RateLimits { - direct_peer: TierRate { - msg_burst: 100, - msg_per_sec: 0, - byte_burst: 100 * MIB, - byte_per_sec: 0, - }, - registered_author: TierRate { - msg_burst: 100, - msg_per_sec: 0, - byte_burst: 100 * MIB, - byte_per_sec: 0, - }, - unregistered_author: TierRate { - msg_burst: 100, - msg_per_sec: 0, - byte_burst: 2 * MIB, - byte_per_sec: 0, - }, - unregistered_total: TierRate { - msg_burst: 100, - msg_per_sec: 0, - byte_burst: 3 * MIB, - byte_per_sec: 0, - }, - unregistered_forward: TierRate { - msg_burst: 100, - msg_per_sec: 0, - byte_burst: 100 * MIB, - byte_per_sec: 0, - }, - global: TierRate { - msg_burst: 100, - msg_per_sec: 0, - byte_burst: 100 * MIB, - byte_per_sec: 0, - }, - committee_reserve: 0.0, - max_tracked_peers: 16, - }; - let limiter = InboundRateLimiter::new(limits); - let now = Instant::now(); - let relay = PeerId::random(); - use P2pInboxAdmissionClass::{Registered, Unregistered}; - - // One unregistered author is capped at its own 2 MiB byte burst. - let a = PeerId::random(); - assert!(limiter.charge(&relay, &a, Unregistered, MIB, now).is_ok()); - assert_eq!( - limiter.charge(&relay, &a, Unregistered, 2 * MIB, now), - Err(DropReason::AuthorRate), - "the author's own bucket has only 1 MiB left" - ); - - // Verify distinct unregistered authors share the class byte budget. - let (b, c, d) = (PeerId::random(), PeerId::random(), PeerId::random()); - assert!(limiter.charge(&relay, &b, Unregistered, MIB, now).is_ok()); - assert!(limiter.charge(&relay, &c, Unregistered, MIB, now).is_ok()); - assert_eq!( - limiter.charge(&relay, &d, Unregistered, MIB, now), - Err(DropReason::UnregisteredRate) - ); - - // A registered author is not charged against the unregistered pool, and - // the messages rejected above did not drain the global tier: a fresh - // registered author is still admitted up to the global burst. - let op = PeerId::random(); - assert!(limiter.charge(&relay, &op, Registered, 50 * MIB, now).is_ok()); - } - - #[test] - fn rate_limiter_refills_and_bounds_its_own_memory() { - let limiter = PeerRateLimiter::new(2.0, 1.0, 2); - let now = Instant::now(); - let (a, b, c) = (PeerId::random(), PeerId::random(), PeerId::random()); - assert!(limiter.allow(&a, now)); - assert!(limiter.allow(&a, now)); - assert!(!limiter.allow(&a, now), "the burst is spent"); - assert!(limiter.allow(&a, now + Duration::from_secs(1)), "one token per second"); - - assert!(limiter.allow(&b, now + Duration::from_secs(1))); - assert!( - !limiter.allow(&c, now + Duration::from_secs(1)), - "no room to track a new peer while the tracked ones are active" - ); - assert!( - limiter.allow(&c, now + Duration::from_secs(60)), - "idle peers are forgotten to make room" - ); - } - - #[test] - fn response_gate_coalesces_requests_inside_the_cooldown() { - let gate = ResponseGate::new(Duration::from_secs(10)); - let now = Instant::now(); - assert!(gate.try_respond(now)); - assert!(!gate.take_pending(now + Duration::from_secs(20)), "nothing was deferred"); - - assert!(!gate.try_respond(now + Duration::from_secs(1))); - assert!(!gate.try_respond(now + Duration::from_secs(2))); - assert!(!gate.take_pending(now + Duration::from_secs(5)), "still cooling down"); - assert!(gate.take_pending(now + Duration::from_secs(10)), "one response for both"); - assert!(!gate.take_pending(now + Duration::from_secs(30))); - assert!(gate.try_respond(now + Duration::from_secs(30))); - } - - #[tokio::test] - async fn verdicts_follow_the_queue_state() { - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let now = Instant::now(); - let source = PeerId::random(); - let limits = InboundLimits { unregistered_peer: Quota { rows: 1, bytes: 4096 }, ..SMALL }; - - let verdict = gates.evaluate(&local_db, &limits, &source, 1, b"not a message", now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::Undecodable))); - - let first = kickoff_sent_bytes().await; - let verdict = gates.evaluate(&local_db, &limits, &source, 2, &first, now).await; - let InboundVerdict::Enqueue { class, content_hash, .. } = verdict else { - panic!("an empty inbox admits the first message"); - }; - assert_eq!(class, P2pInboxAdmissionClass::Unregistered); - let mut storage = local_db.acquire().await.unwrap(); - storage - .insert_p2p_inbox_message(&store::P2pInboxMessage { - message_id: "first".to_string(), - actor: "Committee".to_string(), - from_peer: source.to_string(), - msg_type: "KickoffSent".to_string(), - content: first.clone(), - content_size: first.len() as i64, - admission_class: class.to_string(), - content_hash: Some(content_hash.to_vec()), - ..Default::default() - }) - .await - .unwrap(); - drop(storage); - - // Verify duplicate detection precedes quota enforcement. - let verdict = gates.evaluate(&local_db, &limits, &source, 3, &first, now).await; - assert!(matches!(verdict, InboundVerdict::Duplicate(_))); - - // A different payload from the same sender is over its row quota ... - let second = kickoff_sent_bytes().await; - let verdict = gates.evaluate(&local_db, &limits, &source, 4, &second, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::SenderQuota))); - - // ... unless the sender turns out to be registered. - gates.registry.record_committee_peer(&source, Some(true), now); - let verdict = gates.evaluate(&local_db, &limits, &source, 5, &second, now).await; - assert!(matches!( - verdict, - InboundVerdict::Enqueue { class: P2pInboxAdmissionClass::Committee, .. } - )); - } - - /// Verify replay neither passes admission nor consumes rate budget. - #[tokio::test] - async fn replayed_registered_messages_are_dropped_before_the_rate_charge() { - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let now = Instant::now(); - let member = PeerId::random(); - gates.registry.record_committee_peer(&member, Some(true), now); - - let message = kickoff_sent_bytes().await; - let verdict = gates.evaluate(&local_db, &SMALL, &member, 100, &message, now).await; - assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); - for _ in 0..3 { - let verdict = gates.evaluate(&local_db, &SMALL, &member, 100, &message, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ReplayedSequence))); - } - - // Unregistered authors use rate limits rather than replay tracking. - let stranger = PeerId::random(); - let other = kickoff_sent_bytes().await; - let verdict = gates.evaluate(&local_db, &SMALL, &stranger, 7, &other, now).await; - assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); - } - - #[test] - fn replay_window_tolerates_reordering_but_not_reuse() { - let guard = ReplayGuard::default(); - let author = PeerId::random(); - let base = 1_000_000; - assert_eq!(guard.admit(&author, Some(base), WALL_CLOCK), Ok(())); - assert_eq!(guard.admit(&author, Some(base + 5), WALL_CLOCK), Ok(())); - assert_eq!(guard.admit(&author, Some(base + 3), WALL_CLOCK), Ok(()), "reordered, but new"); - assert_eq!( - guard.admit(&author, Some(base + 3), WALL_CLOCK), - Err(DropReason::ReplayedSequence) - ); - assert_eq!( - guard.admit(&author, Some(base + 5), WALL_CLOCK), - Err(DropReason::ReplayedSequence) - ); - assert_eq!(guard.admit(&author, None, WALL_CLOCK), Err(DropReason::ReplayedSequence)); - - // The author restarts: numbering jumps to the new start-up time, and - // everything from the previous session falls out of the window. - let restarted = base + 10 * REPLAY_WINDOW; - assert_eq!(guard.admit(&author, Some(restarted), WALL_CLOCK), Ok(())); - assert_eq!( - guard.admit(&author, Some(base + 4), WALL_CLOCK), - Err(DropReason::ReplayedSequence) - ); - assert_eq!(guard.admit(&author, Some(restarted - REPLAY_WINDOW + 1), WALL_CLOCK), Ok(())); - assert_eq!( - guard.admit(&author, Some(restarted - REPLAY_WINDOW), WALL_CLOCK), - Err(DropReason::ReplayedSequence) - ); - - // Windows are per author. - assert_eq!(guard.admit(&PeerId::random(), Some(base), WALL_CLOCK), Ok(())); - } - - /// Verify re-announcement preserves stake classification. - #[test] - fn reannounced_operator_binding_keeps_its_stake_verdict() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - let peer = PeerId::random(); - let pubkey = &key(1); - - registry.observe_operator_binding(&peer, pubkey, 100); - assert!(registry.plan_operator_refresh(&peer, pubkey, now)); - assert_eq!(registry.record_operator_stake(&peer, pubkey, Some(true), now), Some(true)); - assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); - - registry.observe_operator_binding(&peer, pubkey, 400); - assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); - assert!( - !registry.plan_operator_refresh(&peer, pubkey, now), - "the verdict is still fresh, so no lookup is due" - ); - - // Keep the newest binding timestamp. - let usurper = PeerId::random(); - registry.observe_operator_binding(&usurper, pubkey, 300); - assert_eq!(registry.sender_class(&peer, now), P2pInboxAdmissionClass::Registered); - assert!(!registry.plan_operator_refresh(&usurper, pubkey, now)); - - // Every definite verdict is handed back for persisting, changed or not. - let later = now + OPERATOR_STAKE_TTL; - assert!(registry.plan_operator_refresh(&peer, pubkey, later)); - assert_eq!(registry.record_operator_stake(&peer, pubkey, Some(true), later), Some(true)); - assert!(registry.plan_operator_refresh(&peer, pubkey, later + OPERATOR_STAKE_TTL)); - assert_eq!( - registry.record_operator_stake(&peer, pubkey, Some(false), later + OPERATOR_STAKE_TTL), - Some(false) - ); - } - - /// Self-signed operator bindings are free to mint. They may exhaust the - /// operator discovery budget, but not the committee's, and not the - /// re-validation of operators already confirmed. - #[test] - fn operator_binding_flood_spends_only_the_operator_discovery_budget() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - registry.record_verifier_set(Some(vec![]), now); - let operator = PeerId::random(); - let real = key(0); - registry.observe_operator_binding(&operator, &real, 1); - assert!(registry.plan_operator_refresh(&operator, &real, now)); - registry.record_operator_stake(&operator, &real, Some(true), now); - - let mut started = 0; - for index in 0..(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { - let (sybil, sybil_key) = (PeerId::random(), key(index + 1)); - registry.observe_operator_binding(&sybil, &sybil_key, 1); - if registry.plan_operator_refresh(&sybil, &sybil_key, now) { - started += 1; - registry.record_operator_stake(&sybil, &sybil_key, Some(false), now); - } - } - // The real operator's own first lookup took one token of the same budget. - assert_eq!(started + 1, OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE); - - assert!( - registry.plan_refresh(Some(&PeerId::random()), now).committee_peer, - "committee discovery has its own budget" - ); - let due = registry.plan_due_operator_refreshes(now + OPERATOR_STAKE_TTL); - assert!( - due.first().is_some_and(|(peer, _)| *peer == operator), - "the confirmed operator is re-validated first, outside the spent budget" - ); - } - - /// Registrations confirmed in an earlier session are trusted at once and are - /// re-validated as known members, not rediscovered through the budget. - #[test] - fn seeded_registrations_are_trusted_and_revalidated_as_known() { - let registry = PeerRegistry::default(); - // `Instant` cannot be dated before the machine booted; keep clear of it. - let now = Instant::now() + COMMITTEE_STALE_LIMIT; - let (member, operator) = (PeerId::random(), PeerId::random()); - registry.seed_verified_committee_peer(&member, now); - registry.seed_verified_operator(&operator, &key(0), 1, now); - assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Committee); - assert_eq!(registry.sender_class(&operator, now), P2pInboxAdmissionClass::Registered); - - // Spend both discovery budgets on strangers. - registry.record_verifier_set(Some(vec![]), now); - for index in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { - let stranger = PeerId::random(); - if registry.plan_refresh(Some(&stranger), now).committee_peer { - registry.record_committee_peer(&stranger, Some(false), now); - } - let sybil_key = key(index + 1); - registry.observe_operator_binding(&stranger, &sybil_key, 1); - if registry.plan_operator_refresh(&stranger, &sybil_key, now) { - registry.record_operator_stake(&stranger, &sybil_key, Some(false), now); - } - } - - assert_eq!(registry.plan_due_committee_refreshes(now), vec![member]); - assert!( - registry.plan_due_operator_refreshes(now).iter().any(|(peer, _)| *peer == operator) - ); - // Revoke restored registrations after a negative chain answer. - assert_eq!(registry.record_committee_peer(&member, Some(false), now), Some(false)); - assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Unregistered); - } - - /// A flood of fresh identities must neither grow the bucket table nor evict - /// the peers it already tracks: past the cap they share one bucket. - #[test] - fn rate_table_is_bounded_and_overflow_identities_share_one_bucket() { - let tier = TierRate { msg_burst: 2, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; - let roomy = - TierRate { msg_burst: 1_000, msg_per_sec: 0, byte_burst: 1 << 30, byte_per_sec: 0 }; - let limiter = InboundRateLimiter::new(RateLimits { - direct_peer: roomy, - registered_author: roomy, - unregistered_author: tier, - unregistered_total: roomy, - unregistered_forward: roomy, - global: roomy, - committee_reserve: 0.0, - max_tracked_peers: 4, - }); - let now = Instant::now(); - let relay = PeerId::random(); - use P2pInboxAdmissionClass::{Registered, Unregistered}; - - for _ in 0..4 { - assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); - } - // New authors share overflow capacity when the table is full. - assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); - assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); - for _ in 0..64 { - assert_eq!( - limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now), - Err(DropReason::AuthorRate) - ); - } - assert_eq!(limiter.tracked_authors(), 4, "fresh identities do not grow the table"); - - // A registered author is bounded by the chain registry and always tracked. - let member = PeerId::random(); - assert!(limiter.charge(&relay, &member, Registered, 1, now).is_ok()); - assert_eq!(limiter.tracked_authors(), 5); - - // Idle buckets are forgotten, making room for new authors again. - let later = now + RATE_BUCKET_IDLE_FORGET; - assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, later).is_ok()); - assert_eq!(limiter.tracked_authors(), 1); - } - - /// Forgetting an idle bucket must not hand its owner tokens it had spent. - #[test] - fn default_tiers_refill_within_the_idle_forget_window() { - let limits = RateLimits::defaults(); - for tier in [ - limits.direct_peer, - limits.registered_author, - limits.unregistered_author, - limits.unregistered_total, - limits.unregistered_forward, - limits.global, - ] { - let window = RATE_BUCKET_IDLE_FORGET.as_secs() as i64; - assert!(i64::from(tier.msg_burst) <= i64::from(tier.msg_per_sec) * window); - assert!(tier.byte_burst <= tier.byte_per_sec * window); - } - } - - /// Verify admissions immediately update cached class totals. - #[tokio::test] - async fn cached_class_totals_count_admissions_immediately() { - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let now = Instant::now(); - let limits = InboundLimits { - unregistered_class: Quota { rows: 2, bytes: 3000 }, - unregistered_peer: Quota { rows: 2, bytes: 3000 }, - ..SMALL - }; - - // Nothing is inserted between evaluations: only the cache can know that - // the class already has two rows on their way into the inbox. - for sequence_number in 0..2 { - let message = kickoff_sent_bytes().await; - let verdict = gates - .evaluate(&local_db, &limits, &PeerId::random(), sequence_number, &message, now) - .await; - assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); - } - let message = kickoff_sent_bytes().await; - let verdict = gates.evaluate(&local_db, &limits, &PeerId::random(), 9, &message, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::UnregisteredClassQuota))); - - // Reload totals after cache expiry. - let later = now + INBOX_CLASS_TOTALS_TTL; - let verdict = - gates.evaluate(&local_db, &limits, &PeerId::random(), 10, &message, later).await; - assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); - } - - #[tokio::test] - async fn binary_envelope_is_reserved_for_gen_circuits_from_verifiers() { - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let now = Instant::now(); - let source = PeerId::random(); - - // A small message wrapped in the binary envelope: well-formed, but it - // is not the payload the envelope exists for. - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::KickoffSent(KickoffSent { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - }), - ); - let mut wrapped = b"GOATBIN1".to_vec(); - wrapped.extend(bincode::serialize(&message).unwrap()); - - let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &wrapped, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::BinaryFromNonVerifier))); - - gates.registry.record_verifier_set(Some(vec![source.to_bytes()]), now); - let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &wrapped, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::UnexpectedBinaryKind))); - } - - #[tokio::test] - async fn immediate_messages_are_rate_limited_per_sender() { - use crate::action::NodeInfo; - - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(1.0, 0.0, 16)); - let now = Instant::now(); - let source = PeerId::random(); - let request = GOATMessage::new( - Actor::All, - GOATMessageContent::RequestNodeInfo(NodeInfo { - peer_id: source.to_string(), - actor: "Operator".to_string(), - goat_addr: String::new(), - btc_pub_key: String::new(), - socket_addr: String::new(), - node_name: String::new(), - service_fee_rate: 0.0, - available_peg_btc: "0".to_string(), - ..Default::default() - }), - ) - .serialize_message() - .await - .unwrap(); - - let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &request, now).await; - assert!(matches!(verdict, InboundVerdict::Immediate(_))); - let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &request, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ImmediateRateLimited))); - let other = PeerId::random(); - let verdict = gates.evaluate(&local_db, &SMALL, &other, 1, &request, now).await; - assert!(matches!(verdict, InboundVerdict::Immediate(_)), "limits are per sender"); - } - - /// Verify equivalent key encodings share one operator identity. - #[test] - fn operator_identity_ignores_how_the_key_is_spelled() { - use crate::action::{NodeInfo, sign_node_info_binding, verify_node_info_binding}; - use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; - - let secp = Secp256k1::new(); - let keypair = Keypair::from_secret_key(&secp, &SecretKey::from_slice(&[9; 32]).unwrap()); - let compressed = bitcoin::PublicKey::new(keypair.public_key()); - let mut other_parity = compressed.to_string(); - other_parity.replace_range(..2, if other_parity.starts_with("02") { "03" } else { "02" }); - let mut uncompressed = compressed; - uncompressed.compressed = false; - let spellings = [ - compressed.to_string(), - compressed.to_string().to_uppercase(), - other_parity, - uncompressed.to_string(), - ]; - - let registry = PeerRegistry::default(); - let now = Instant::now(); - let mut keys = HashSet::new(); - let mut peers = Vec::new(); - for (index, spelling) in spellings.iter().enumerate() { - let peer = PeerId::random(); - let issued_at = 100 + index as i64; - let node_info = NodeInfo { - peer_id: peer.to_string(), - btc_pub_key: spelling.clone(), - binding_sig: sign_node_info_binding( - &peer.to_string(), - spelling, - issued_at, - &keypair, - ), - binding_issued_at: issued_at, - ..Default::default() - }; - let verified = verify_node_info_binding(&node_info) - .unwrap_or_else(|| panic!("{spelling} is a valid spelling of the key")); - let operator = operator_key(&verified); - keys.insert(operator); - registry.observe_operator_binding(&peer, &operator, issued_at); - if registry.plan_operator_refresh(&peer, &operator, now) { - registry.record_operator_stake(&peer, &operator, Some(true), now); - } - peers.push(peer); - } - assert_eq!(keys.len(), 1, "every spelling is the same operator"); - let registered = - peers.iter().filter(|peer| registry.sender_class(peer, now).is_registered()).count(); - assert_eq!(registered, 1, "one stake backs one peer id, however the key is written"); - assert!(registry.sender_class(peers.last().unwrap(), now).is_registered()); - } - - /// Verify equal timestamps cannot transfer ownership and key changes revoke prior registration. - #[test] - fn operator_key_changes_hands_only_forward_and_rebinding_revokes() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - let (first, second) = (PeerId::random(), PeerId::random()); - let (staked, unstaked) = (key(1), key(2)); - - assert!(!registry.observe_operator_binding(&first, &staked, 100)); - assert!(registry.plan_operator_refresh(&first, &staked, now)); - assert_eq!(registry.record_operator_stake(&first, &staked, Some(true), now), Some(true)); - - registry.observe_operator_binding(&second, &staked, 100); - assert!(registry.sender_class(&first, now).is_registered(), "same issue time: no change"); - assert!(!registry.plan_operator_refresh(&second, &staked, now)); - - // Moving to an unstaked key gives up the confirmed one ... - assert!(registry.observe_operator_binding(&first, &unstaked, 200)); - assert!(!registry.sender_class(&first, now).is_registered()); - // ... and the negative verdict for the new key is still handed back, so - // the store is cleaned even though nothing "changed" for this binding. - assert!(registry.plan_operator_refresh(&first, &unstaked, now)); - assert_eq!( - registry.record_operator_stake(&first, &unstaked, Some(false), now), - Some(false) - ); - // A verdict for the key the peer no longer holds is ignored. - assert_eq!(registry.record_operator_stake(&first, &staked, Some(true), now), None); - } - - /// A stored confirmation is restored only for the key it was made for. - #[tokio::test] - async fn stored_operator_confirmation_is_bound_to_its_key() { - let local_db = store::create_local_db("sqlite::memory:").await; - let now = Instant::now() + OPERATOR_STAKE_STALE_LIMIT; - let (kept, rebound, member) = (PeerId::random(), PeerId::random(), PeerId::random()); - let (staked, unstaked) = (key(1), key(2)); - { - let mut storage = local_db.acquire().await.unwrap(); - for peer in [&kept, &rebound] { - storage - .upsert_p2p_registered_peer( - &peer.to_string(), - REGISTERED_KIND_OPERATOR, - &hex::encode(if peer == &kept { staked } else { key(3) }), - ) - .await - .unwrap(); - } - storage - .upsert_p2p_registered_peer(&member.to_string(), REGISTERED_KIND_COMMITTEE, "") - .await - .unwrap(); - } - // `rebound` was confirmed for key 3 but now presents a (validly signed) - // binding for an unstaked key. - let bindings = HashMap::from([(kept, (staked, 10)), (rebound, (unstaked, 20))]); - let registry = PeerRegistry::default(); - for (peer, (operator, issued_at)) in &bindings { - registry.observe_operator_binding(peer, operator, *issued_at); - } - let seeded = seed_registry_from_store(&local_db, ®istry, &bindings, now).await.unwrap(); - assert_eq!(seeded, 2, "the kept operator and the committee member"); - assert_eq!(registry.sender_class(&kept, now), P2pInboxAdmissionClass::Registered); - assert_eq!(registry.sender_class(&member, now), P2pInboxAdmissionClass::Committee); - assert_eq!(registry.sender_class(&rebound, now), P2pInboxAdmissionClass::Unregistered); - - // The revocation a re-binding triggers removes the stored row. - revoke_persisted_operator(&local_db, &rebound).await; - let stored = local_db.acquire().await.unwrap().load_p2p_registered_peers().await.unwrap(); - assert!(!stored.iter().any(|(peer_id, _, _)| *peer_id == rebound.to_string())); - } - - #[test] - fn committee_reserve_is_closed_to_other_senders() { - use P2pInboxAdmissionClass::{Committee, Registered, Unregistered}; - let limits = InboundLimits { committee_reserve: Quota { rows: 2, bytes: 2000 }, ..SMALL }; - - // Four rows from staked operators fill everything outside the reserve. - let crowded = [usage(Registered, 4, 400, (0, 0))]; - assert_eq!( - check_inbox_quota(Registered, 100, &crowded, &limits), - Err(DropReason::GlobalQuota) - ); - assert_eq!( - check_inbox_quota(Unregistered, 100, &crowded, &limits), - Err(DropReason::GlobalQuota) - ); - assert_eq!(check_inbox_quota(Committee, 100, &crowded, &limits), Ok(())); - // Committee rows do not count against the others' share. - let committee_heavy = [usage(Committee, 3, 300, (0, 0)), usage(Registered, 2, 200, (0, 0))]; - assert_eq!(check_inbox_quota(Registered, 100, &committee_heavy, &limits), Ok(())); - // The committee is still bounded by the global ceiling. - let full = [usage(Committee, 6, 600, (0, 0))]; - assert_eq!(check_inbox_quota(Committee, 100, &full, &limits), Err(DropReason::GlobalQuota)); - } - - /// Verify a full gate rejects new live keys. - #[test] - fn gates_and_sets_refuse_new_keys_when_full_of_live_ones() { - let now = Instant::now(); - let gate = CooldownGate::new(Duration::from_secs(60), 4); - for index in 0..4 { - assert!(gate.allow(&format!("key-{index}"), now)); - } - for index in 4..64 { - assert!(!gate.allow(&format!("key-{index}"), now + Duration::from_secs(1))); - } - assert_eq!(gate.len(), 4); - assert!(gate.is_cooling("key-0", now + Duration::from_secs(1))); - assert!(!gate.is_cooling("key-9", now + Duration::from_secs(1)), "never recorded"); - // Expired keys make room again, once the sweep interval has passed. - assert!(gate.allow("late", now + Duration::from_secs(61))); - assert_eq!(gate.len(), 1); - - let set = TtlSet::new(Duration::from_secs(60), 2); - assert!(set.insert("a", now)); - assert!(set.insert("b", now)); - assert!(!set.insert("c", now), "full of unexpired keys"); - assert!(set.insert("a", now), "refreshing a member needs no room"); - set.remove("a"); - assert!(!set.contains("a", now)); - assert!(set.insert("c", now)); - } - - /// Verify restored replay marks reject older sequence numbers. - #[test] - fn restored_replay_mark_is_a_floor() { - let guard = ReplayGuard::default(); - let author = PeerId::random(); - assert_eq!(guard.admit(&author, Some(500), WALL_CLOCK), Ok(())); - assert_eq!(guard.admit(&author, Some(498), WALL_CLOCK), Ok(())); - assert_eq!(guard.pending_marks(), vec![(author, 500)]); - - let restarted = ReplayGuard::default(); - restarted.restore_mark(&author, 500); - assert!(restarted.pending_marks().is_empty(), "a restored mark is already stored"); - for replayed in [497, 498, 499, 500] { - assert_eq!( - restarted.admit(&author, Some(replayed), WALL_CLOCK), - Err(DropReason::ReplayedSequence) - ); - } - assert_eq!(restarted.admit(&author, Some(501), WALL_CLOCK), Ok(())); - assert_eq!(restarted.pending_marks(), vec![(author, 501)]); - } - - /// Persisted confirmation must retain failed writes and newer pending marks. - #[test] - fn replay_marks_stay_owed_until_the_write_is_confirmed() { - let guard = ReplayGuard::default(); - let author = PeerId::random(); - assert_eq!(guard.admit(&author, Some(10), WALL_CLOCK), Ok(())); - - // Tick 1: the marks are read, the write fails, nothing is confirmed. - let owed = guard.pending_marks(); - assert_eq!(owed, vec![(author, 10)]); - assert_eq!(guard.pending_marks(), owed, "still owed on the next tick"); - - // Tick 2: the write is under way when a newer message is admitted. - let written = guard.pending_marks(); - assert_eq!(guard.admit(&author, Some(11), WALL_CLOCK), Ok(())); - guard.confirm_persisted(&written); - assert_eq!(guard.pending_marks(), vec![(author, 11)], "only 10 reached the store"); - - guard.confirm_persisted(&[(author, 11)]); - assert!(guard.pending_marks().is_empty()); - // A late confirmation of an older write never moves the record back. - guard.confirm_persisted(&[(author, 10)]); - assert!(guard.pending_marks().is_empty()); - } - - /// Verify future sequence numbers do not raise replay marks. - #[test] - fn future_sequence_numbers_cannot_raise_the_mark() { - let guard = ReplayGuard::default(); - let author = PeerId::random(); - let ahead = WALL_CLOCK + REPLAY_MAX_FUTURE.as_nanos() as u64; - assert_eq!( - guard.admit(&author, Some(ahead + 1), WALL_CLOCK), - Err(DropReason::FutureSequence) - ); - assert!(guard.pending_marks().is_empty(), "nothing was recorded"); - assert_eq!(guard.admit(&author, Some(ahead), WALL_CLOCK), Ok(()), "inside the tolerance"); - - // The corrected clock numbers lower than the mark: refused, and surfaced - // for the operator once it keeps happening. - assert!(guard.take_lockout_suspects().is_empty()); - for offset in 0..REPLAY_LOCKOUT_SUSPECT_REJECTIONS as u64 { - assert_eq!( - guard.admit(&author, Some(WALL_CLOCK - REPLAY_WINDOW * 2 + offset), WALL_CLOCK), - Err(DropReason::ReplayedSequence) - ); - } - assert_eq!( - guard.take_lockout_suspects(), - vec![(author, REPLAY_LOCKOUT_SUSPECT_REJECTIONS)] - ); - assert!(guard.take_lockout_suspects().is_empty(), "reported once per tick"); - } - - #[test] - fn direct_peer_is_banned_at_the_strike_limit_and_released_later() { - let strikes = DirectPeerStrikes::default(); - let now = Instant::now(); - let (flooder, bystander) = (PeerId::random(), PeerId::random()); - for _ in 0..(STRIKE_LIMIT - 1) { - assert!(!strikes.strike(&flooder, now)); - } - assert!(!strikes.strike(&bystander, now)); - assert!(strikes.strike(&flooder, now), "the limit is reached exactly once"); - assert!(!strikes.strike(&flooder, now), "already banned"); - assert!(strikes.take_expired_bans(now).is_empty()); - assert_eq!(strikes.take_expired_bans(now + STRIKE_BAN), vec![flooder]); - - // Strikes spread thinner than the window never add up. - let slow = PeerId::random(); - for round in 0..(STRIKE_LIMIT * 2) { - assert!(!strikes.strike(&slow, now + STRIKE_WINDOW * round)); - } - - // Only verdicts that do not depend on this node's state are blamed on - // the neighbour; and an unanswered lookup is not "known unregistered". - assert!(DropReason::Undecodable.blames_direct_peer()); - assert!(DropReason::UnexpectedBinaryKind.blames_direct_peer()); - for local in [ - DropReason::SenderQuota, - DropReason::ReplayedSequence, - // An honest relay earns this one just by being the only neighbour. - DropReason::DirectPeerRate, - // The JSON ceiling is configurable, so nodes may disagree on it. - DropReason::OversizedJson, - DropReason::BinaryFromNonVerifier, - ] { - assert!(!local.blames_direct_peer(), "{local:?} depends on this node"); - } - assert!(!strikes.is_banned(&bystander, now)); - let banned = PeerId::random(); - for _ in 0..STRIKE_LIMIT { - strikes.strike(&banned, now); - } - assert!(strikes.is_banned(&banned, now), "a ban holds against reconnecting"); - assert!(!strikes.is_banned(&banned, now + STRIKE_BAN)); - let registry = PeerRegistry::default(); - assert!(!registry.is_known_unregistered(&flooder, now)); - registry.record_committee_peer(&flooder, Some(false), now); - assert!(registry.is_known_unregistered(&flooder, now)); - registry.record_committee_peer(&bystander, Some(true), now); - assert!(!registry.is_known_unregistered(&bystander, now)); - } - - /// A durable message no handler of this node's role acts on is relayed, but - /// takes no inbox row and no quota. - #[tokio::test] - async fn messages_for_another_role_are_forwarded_without_being_stored() { - use crate::action::NackReady; - - let local_db = store::create_local_db("sqlite::memory:").await; - let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let now = Instant::now(); - let source = PeerId::random(); - let nack_ready = GOATMessage::new( - Actor::Verifier, - GOATMessageContent::NackReady(NackReady { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - }), - ) - .serialize_message() - .await - .unwrap(); - - let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &nack_ready, now).await; - assert!(matches!(verdict, InboundVerdict::Forward), "a committee node has no handler"); - gates.local_actor = Actor::Verifier; - let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &nack_ready, now).await; - assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); - } - - /// Verify every dispatch role arm is covered by `MessageKind::handled_by`. - #[test] - fn dispatch_arms_are_covered_by_the_role_table() { - use crate::action::MessageKind; - use std::str::FromStr; - - let source = include_str!("handle.rs"); - let dispatcher = { - let start = source.find("pub(crate) fn heavy_task_from_content(").unwrap(); - let end = source.find("fn make_message(").unwrap(); - &source[start..end] - }; - let roles = [Actor::Committee, Actor::Operator, Actor::Verifier, Actor::Watchtower]; - let mut arms = 0; - for (at, _) in dispatcher.match_indices("GOATMessageContent::") { - // Match dispatch patterns ending in `=>`, excluding expressions in arm bodies. - let rest = &dispatcher[at + "GOATMessageContent::".len()..]; - let name: String = rest.chars().take_while(|c| c.is_alphanumeric()).collect(); - let Ok(kind) = MessageKind::from_str(&name) else { - continue; - }; - let Some(arrow) = rest.find("=>") else { - continue; - }; - let pattern = &rest[..arrow]; - if pattern.contains(';') || pattern.contains(".await") || pattern.contains("Some(") { - continue; - } - arms += 1; - let named: Vec<&Actor> = - roles.iter().filter(|role| pattern.contains(&format!("Actor::{role}"))).collect(); - // No role named: the `_` arm, which every role reaches. - let reached: Vec<&Actor> = - if named.is_empty() { roles.iter().collect() } else { named }; - for role in reached { - assert!( - kind.handled_by(role), - "dispatch has an arm for ({name}, {role}) but handled_by says it is not \ - handled: the message would be relayed without ever being processed" - ); - } - } - assert!(arms >= 40, "the scan found only {arms} arms; has dispatch moved?"); - } - - /// Verify committee-only reserves in shared rate tiers. - #[test] - fn shared_rate_tiers_keep_a_reserve_for_the_committee() { - let tier = TierRate { msg_burst: 8, msg_per_sec: 0, byte_burst: 8_000, byte_per_sec: 0 }; - let roomy = - TierRate { msg_burst: 1_000, msg_per_sec: 0, byte_burst: 1 << 30, byte_per_sec: 0 }; - let limiter = InboundRateLimiter::new(RateLimits { - direct_peer: roomy, - registered_author: roomy, - unregistered_author: roomy, - unregistered_total: roomy, - unregistered_forward: roomy, - global: tier, - committee_reserve: 0.25, - max_tracked_peers: 16, - }); - let now = Instant::now(); - let relay = PeerId::random(); - use P2pInboxAdmissionClass::{Committee, Registered, Unregistered}; - - // Six of eight messages are open to everyone ... - for _ in 0..6 { - assert!(limiter.charge(&relay, &PeerId::random(), Registered, 100, now).is_ok()); - } - assert_eq!( - limiter.charge(&relay, &PeerId::random(), Registered, 100, now), - Err(DropReason::GlobalRate) - ); - assert_eq!( - limiter.charge(&relay, &PeerId::random(), Unregistered, 100, now), - Err(DropReason::GlobalRate) - ); - // ... and the last two only to the committee. - for _ in 0..2 { - assert!(limiter.charge(&relay, &PeerId::random(), Committee, 100, now).is_ok()); - } - assert_eq!( - limiter.charge(&relay, &PeerId::random(), Committee, 100, now), - Err(DropReason::GlobalRate) - ); - } - - /// Verify unregistered forward-only traffic is rate-limited. - #[tokio::test] - async fn forwarding_for_unregistered_authors_is_budgeted() { - use crate::action::NackReady; - - let local_db = store::create_local_db("sqlite::memory:").await; - let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let tier = TierRate { msg_burst: 2, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; - gates.rate_limiter = InboundRateLimiter::new(RateLimits { - unregistered_forward: tier, - ..RateLimits::defaults() - }); - let now = Instant::now(); - let nack_ready = |graph_id| { - GOATMessage::new( - Actor::Verifier, - GOATMessageContent::NackReady(NackReady { instance_id: Uuid::new_v4(), graph_id }), - ) - }; - - for sequence_number in 0..2 { - let data = nack_ready(Uuid::new_v4()).serialize_message().await.unwrap(); - let verdict = gates - .evaluate(&local_db, &SMALL, &PeerId::random(), sequence_number, &data, now) - .await; - assert!(matches!(verdict, InboundVerdict::Forward)); - } - let data = nack_ready(Uuid::new_v4()).serialize_message().await.unwrap(); - let verdict = gates.evaluate(&local_db, &SMALL, &PeerId::random(), 9, &data, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ForwardRate))); - - let member = PeerId::random(); - gates.registry.record_committee_peer(&member, Some(true), now); - let verdict = gates.evaluate(&local_db, &SMALL, &member, 1, &data, now).await; - assert!(matches!(verdict, InboundVerdict::Forward), "registered authors are not charged"); - } - - /// A kind only one role can send is refused once the chain has said the - /// author does not hold that role — and only then. - #[tokio::test] - async fn sender_role_is_refused_only_on_a_definite_answer() { - use crate::action::{AggNonceConsensus, MessageKind}; - - assert_eq!(MessageKind::NonceGeneration.sender_role(), SenderRole::Committee); - assert_eq!(MessageKind::GenCircuits.sender_role(), SenderRole::Verifier); - assert_eq!(MessageKind::InitGraph.sender_role(), SenderRole::Operator); - assert_eq!(MessageKind::KickoffSent.sender_role(), SenderRole::Any); - // Keep the prefilter no stricter than handler authorization. - assert_eq!(MessageKind::SolderingProofReady.sender_role(), SenderRole::Any); - - let registry = PeerRegistry::default(); - let now = Instant::now(); - let (unknown, outsider, member) = (PeerId::random(), PeerId::random(), PeerId::random()); - registry.record_committee_peer(&outsider, Some(false), now); - registry.record_committee_peer(&member, Some(true), now); - let verdict = |peer, role, at| registry.role_verdict(peer, role, at); - assert_eq!(verdict(&unknown, SenderRole::Committee, now), RoleVerdict::Unknown); - assert_eq!(verdict(&outsider, SenderRole::Committee, now), RoleVerdict::Denied); - assert_eq!(verdict(&member, SenderRole::Committee, now), RoleVerdict::Confirmed); - assert_eq!(verdict(&outsider, SenderRole::Any, now), RoleVerdict::Confirmed); - assert_eq!( - verdict(&outsider, SenderRole::Committee, now + COMMITTEE_NEGATIVE_TTL), - RoleVerdict::Unknown, - "a stale \"no\" is not a \"no\": the peer may have registered since" - ); - // The verifier set is definite once fetched; before that nobody is denied. - assert_eq!(verdict(&unknown, SenderRole::Verifier, now), RoleVerdict::Unknown); - registry.record_verifier_set(Some(vec![member.to_bytes()]), now); - assert_eq!(verdict(&unknown, SenderRole::Verifier, now), RoleVerdict::Denied); - assert_eq!(verdict(&member, SenderRole::Verifier, now), RoleVerdict::Confirmed); - // An operator is denied only by a negative stake verdict for its binding. - assert_eq!(verdict(&unknown, SenderRole::Operator, now), RoleVerdict::Unknown); - registry.observe_operator_binding(&outsider, &key(1), 1); - assert_eq!(verdict(&outsider, SenderRole::Operator, now), RoleVerdict::Unknown); - assert!(registry.plan_operator_refresh(&outsider, &key(1), now)); - registry.record_operator_stake(&outsider, &key(1), Some(false), now); - assert_eq!(verdict(&outsider, SenderRole::Operator, now), RoleVerdict::Denied); - - // End to end: the same committee-only message, from an unknown author - // and from one the chain has turned down. - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - gates.registry.record_committee_peer(&outsider, Some(false), now); - let secp = bitcoin::secp256k1::Secp256k1::new(); - let keypair = bitcoin::secp256k1::Keypair::from_secret_key( - &secp, - &bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), - ); - let digest = bitcoin::secp256k1::Message::from_digest([3; 32]); - let data = GOATMessage::new( - Actor::Committee, - GOATMessageContent::AggNonceConsensus(AggNonceConsensus { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - committee_pubkey: bitcoin::PublicKey::new(keypair.public_key()), - consensus_hash: [3; 32], - signature: secp.sign_schnorr(&digest, &keypair), - }), - ) - .serialize_message() - .await - .unwrap(); - let verdict = gates.evaluate(&local_db, &SMALL, &unknown, 1, &data, now).await; - assert!( - matches!(verdict, InboundVerdict::Enqueue { .. }), - "unknown is not denied: the handler stays the authority" - ); - let verdict = gates.evaluate(&local_db, &SMALL, &outsider, 1, &data, now).await; - assert!(matches!(verdict, InboundVerdict::Drop(DropReason::SenderRoleDenied))); - } - - /// Verify neighbour discovery is independent of author discovery. - #[test] - fn neighbour_lookups_have_their_own_budget() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - registry.record_verifier_set(Some(vec![]), now); - for _ in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { - let author = PeerId::random(); - if registry.plan_refresh(Some(&author), now).committee_peer { - registry.record_committee_peer(&author, Some(false), now); - } - } - let neighbour = PeerId::random(); - assert!(!registry.plan_refresh(Some(&neighbour), now).committee_peer, "budget spent"); - assert!(registry.plan_neighbour_lookup(&neighbour, now)); - - // Verify refused peers cannot consume neighbour discovery budget. - let mut refused = 0; - for _ in 0..(REFUSED_PEER_LOOKUPS_PER_MINUTE * 2) { - let peer = PeerId::random(); - if registry.plan_refused_peer_lookup(&peer, now) { - refused += 1; - registry.record_committee_peer(&peer, Some(false), now); - } - } - assert_eq!(refused, REFUSED_PEER_LOOKUPS_PER_MINUTE); - assert!(registry.plan_neighbour_lookup(&PeerId::random(), now)); - } - - /// Verify committee-only lookup does not claim verifier-set refresh. - #[test] - fn neighbour_lookup_does_not_claim_the_verifier_set() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - // The verifier set has never been fetched, so it is due. - assert!(registry.plan_neighbour_lookup(&PeerId::random(), now)); - assert!(registry.plan_refused_peer_lookup(&PeerId::random(), now)); - assert!( - registry.plan_refresh(None, now).verifier_set, - "the verifier set is still there to be claimed by a caller that fetches it" - ); - } - - /// Verify weighted rotation with persistent backlog and one dispatch per tick. - #[test] - fn rota_serves_every_class_across_ticks_when_each_tick_runs_one_message() { - use std::collections::VecDeque; - - /// One tick as the worker runs it: list a batch by shares, then serve it - /// through the rota until the budget (in messages) is spent. The first - /// message always runs. - fn tick(schedule: &mut InboxSchedule, backlog: [usize; 3], budget: usize) -> Vec { - let shares = [8, 4, 4]; - let mut queues: [VecDeque; 3] = Default::default(); - for class in 0..3 { - queues[class].extend(std::iter::repeat_n(class, backlog[class].min(shares[class]))); - } - let mut served = Vec::new(); - loop { - if !served.is_empty() && served.len() >= budget { - break; - } - let Some(class) = schedule.next(&mut queues) else { - break; - }; - served.push(class); - } - served - } - - let mut schedule = InboxSchedule::default(); - let mut served = [0usize; 3]; - let mut last_served = [0usize; 3]; - let mut longest_wait = [0usize; 3]; - for tick_number in 1..=400 { - for class in tick(&mut schedule, [100, 100, 100], 1) { - served[class] += 1; - longest_wait[class] = longest_wait[class].max(tick_number - last_served[class]); - last_served[class] = tick_number; - } - } - assert_eq!(served, [200, 100, 100], "committee twice as often, nobody starved"); - assert!(longest_wait.iter().all(|wait| *wait <= 4), "{longest_wait:?}"); - - // Compare against restarting rotation at each tick. - let mut served = [0usize; 3]; - for _ in 0..400 { - for class in tick(&mut InboxSchedule::default(), [100, 100, 100], 1) { - served[class] += 1; - } - } - assert_eq!(served, [400, 0, 0]); - - // A class with nothing to claim passes its turn on instead of wasting it. - let mut schedule = InboxSchedule::default(); - let mut served = [0usize; 3]; - for _ in 0..300 { - for class in tick(&mut schedule, [100, 0, 100], 1) { - served[class] += 1; - } - } - assert_eq!(served, [200, 0, 100]); - - // With budget to spare a tick serves the batch interleaved, and the next - // tick picks up where this one stopped. - let mut schedule = InboxSchedule::default(); - assert_eq!(tick(&mut schedule, [100, 100, 100], 6), [0, 1, 0, 2, 0, 1]); - assert_eq!(tick(&mut schedule, [100, 100, 100], 2), [0, 2]); - assert_eq!(InboxSchedule::queue_of("Committee"), SCHEDULE_COMMITTEE); - assert_eq!(InboxSchedule::queue_of("Registered"), SCHEDULE_REGISTERED); - assert_eq!(InboxSchedule::queue_of("Unregistered"), SCHEDULE_UNREGISTERED); - assert_eq!(InboxSchedule::queue_of(""), SCHEDULE_UNREGISTERED, "unclassified is untrusted"); - } - - /// Verify configured bindings still require confirmed stake. - #[test] - fn trusted_operator_binding_needs_the_chain_but_not_the_budget() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - // Spend the operator discovery budget on self-signed bindings. - for index in 0..(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { - let (sybil, sybil_key) = (PeerId::random(), key(index + 10)); - registry.observe_operator_binding(&sybil, &sybil_key, 1); - if registry.plan_operator_refresh(&sybil, &sybil_key, now) { - registry.record_operator_stake(&sybil, &sybil_key, Some(false), now); - } - } - let (newcomer, newcomer_key) = (PeerId::random(), key(1)); - registry.observe_operator_binding(&newcomer, &newcomer_key, 1); - assert!(!registry.plan_operator_refresh(&newcomer, &newcomer_key, now), "budget spent"); - - let (trusted, trusted_key) = (PeerId::random(), key(2)); - assert_eq!(registry.trust_operator_binding(&trusted, &trusted_key), Ok(())); - assert!(!registry.sender_class(&trusted, now).is_registered(), "configured is not staked"); - let due = registry.plan_due_operator_refreshes(now); - assert!(due.contains(&(trusted, trusted_key)), "looked up despite the spent budget"); - assert!( - due.iter().all(|(_, operator)| *operator == trusted_key), - "keys the chain said are not staked are not swept again, and the newcomer still \ - has no budget: {} due", - due.len() - ); - registry.record_operator_stake(&trusted, &trusted_key, Some(true), now); - assert_eq!(registry.sender_class(&trusted, now), P2pInboxAdmissionClass::Registered); - - // The operator's own signed announcement later keeps the verdict ... - registry.observe_operator_binding(&trusted, &trusted_key, 500); - assert!(registry.sender_class(&trusted, now).is_registered()); - // ... and if it moves to another peer id, its signature outranks the - // static configuration. - let moved = PeerId::random(); - registry.observe_operator_binding(&moved, &trusted_key, 600); - assert!(!registry.sender_class(&trusted, now).is_registered()); - } - - /// Verify signed bindings take precedence over configuration. - #[test] - fn configured_binding_does_not_override_a_signed_one() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - let (operator, other_peer) = (PeerId::random(), PeerId::random()); - let (signed_key, configured_key) = (key(1), key(2)); - - // Restored from the store: the operator's own signed binding, confirmed. - registry.seed_verified_operator(&operator, &signed_key, 100, now); - let now = now + Duration::from_secs(1); - assert!(registry.sender_class(&operator, now).is_registered()); - - // The config still names an older key for that peer ... - assert_eq!( - registry.trust_operator_binding(&operator, &configured_key), - Err(TrustedBindingConflict::PeerBoundToAnotherKey(signed_key)) - ); - // ... or names the operator's key under a peer id it has moved away from. - assert_eq!( - registry.trust_operator_binding(&other_peer, &signed_key), - Err(TrustedBindingConflict::KeyBoundToAnotherPeer(operator)) - ); - assert!(registry.sender_class(&operator, now).is_registered(), "the signed binding stands"); - assert!(!registry.sender_class(&other_peer, now).is_registered()); - - // Agreeing with the signed binding keeps its verdict and adds the trust. - assert_eq!(registry.trust_operator_binding(&operator, &signed_key), Ok(())); - assert!(registry.sender_class(&operator, now).is_registered()); - - // The same peer configured twice with different keys: first one stands. - let fresh = PeerId::random(); - assert_eq!(registry.trust_operator_binding(&fresh, &key(3)), Ok(())); - assert_eq!( - registry.trust_operator_binding(&fresh, &key(4)), - Err(TrustedBindingConflict::PeerBoundToAnotherKey(key(3))) - ); - } - - /// Verify clock leads are recorded within and beyond tolerance. - #[test] - fn clock_leads_are_reported_per_author_and_then_forgotten() { - let guard = ReplayGuard::default(); - let (slightly_ahead, far_ahead, on_time) = - (PeerId::random(), PeerId::random(), PeerId::random()); - let minute = Duration::from_secs(60).as_nanos() as u64; - - assert_eq!(guard.admit(&on_time, Some(WALL_CLOCK - minute), WALL_CLOCK), Ok(())); - assert_eq!(guard.admit(&slightly_ahead, Some(WALL_CLOCK + 2 * minute), WALL_CLOCK), Ok(())); - assert_eq!(guard.admit(&slightly_ahead, Some(WALL_CLOCK + 3 * minute), WALL_CLOCK), Ok(())); - assert_eq!( - guard.admit(&far_ahead, Some(WALL_CLOCK + 30 * minute), WALL_CLOCK), - Err(DropReason::FutureSequence) - ); - - let mut leads = guard.take_clock_leads(); - leads.sort_by_key(|(_, lead)| *lead); - assert_eq!( - leads, - vec![ - (slightly_ahead, Duration::from_secs(3 * 60)), - (far_ahead, Duration::from_secs(30 * 60)) - ], - "the largest lead per author; an author on time is not listed" - ); - assert!(guard.take_clock_leads().is_empty()); - } - - /// Verify only successful recovery closes the gate. - #[test] - fn republish_gate_stays_open_until_a_publish_succeeds() { - let gate = CooldownGate::new(PROTOCOL_REPUBLISH_COOLDOWN, 16); - let now = Instant::now(); - let key = "committee-presign:graph"; - - // First run: the value is stored, the publish fails, nothing is stamped. - assert!(!gate.is_cooling(key, now)); - // The retry, seconds later, therefore publishes. - let retry = now + Duration::from_secs(10); - assert!(!gate.is_cooling(key, retry)); - assert!(gate.allow(key, retry), "stamped after the publish went through"); - // Later messages of the round do not publish it again ... - assert!(gate.is_cooling(key, retry + Duration::from_secs(1))); - // ... until the cooldown has passed, and other rounds are unaffected. - assert!(!gate.is_cooling(key, retry + PROTOCOL_REPUBLISH_COOLDOWN)); - assert!(!gate.is_cooling("committee-presign:another-graph", retry)); - } - - /// A "no" that has gone stale is not grounds for acting against a neighbour. - #[test] - fn a_stale_negative_is_not_known_unregistered() { - let registry = PeerRegistry::default(); - let now = Instant::now(); - let peer = PeerId::random(); - registry.record_committee_peer(&peer, Some(false), now); - assert!(registry.is_known_unregistered(&peer, now)); - assert!(!registry.is_known_unregistered(&peer, now + COMMITTEE_NEGATIVE_TTL)); - } - - #[test] - fn trusted_operator_bindings_are_parsed_strictly() { - use crate::env::parse_trusted_operator_bindings; - use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; - - let secp = Secp256k1::new(); - let keypair = Keypair::from_secret_key(&secp, &SecretKey::from_slice(&[9; 32]).unwrap()); - let pubkey = bitcoin::PublicKey::new(keypair.public_key()); - let xonly = bitcoin::XOnlyPublicKey::from(pubkey).serialize(); - let (first, second) = (PeerId::random(), PeerId::random()); - let value = format!( - " {first}={pubkey} , not-a-binding, {second} = {} ,{first}=zz,", - pubkey.to_string().to_uppercase() - ); - assert_eq!( - parse_trusted_operator_bindings(&value), - vec![(first, xonly), (second, xonly)], - "malformed entries are skipped, and every spelling is the same key" - ); - assert!(parse_trusted_operator_bindings("").is_empty()); - } - - /// Verify only requested SyncGraph responses are kept locally. - #[tokio::test] - async fn unsolicited_sync_graph_is_relayed_but_not_stored() { - use crate::action::{MessageKind, SyncGraphRequest}; - - let local_db = store::create_local_db("sqlite::memory:").await; - let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); - let now = Instant::now(); - let relayer = PeerId::random(); - gates.registry.record_committee_peer(&relayer, Some(true), now); - - // Exercise SyncGraph admission by message kind and graph ID. - for actor in [Actor::Committee, Actor::Operator, Actor::Verifier, Actor::Watchtower] { - assert!(MessageKind::SyncGraph.handled_by(&actor)); - } - // ... and the request for one is stored only where it can be answered. - let request = GOATMessage::new( - Actor::All, - GOATMessageContent::SyncGraphRequest(SyncGraphRequest { - instance_id: Uuid::new_v4(), - graph_id: Uuid::new_v4(), - }), - ) - .serialize_message() - .await - .unwrap(); - let verdict = gates.evaluate(&local_db, &SMALL, &relayer, 1, &request, now).await; - assert!(matches!(verdict, InboundVerdict::Enqueue { .. }), "a committee node answers"); - - // The decision itself: an answer is kept only by the node that asked, - // and stops being expected once the request has been served. - let (asked, unasked) = (Uuid::new_v4(), Uuid::new_v4()); - assert!(gates.requested_graphs.insert(&asked.to_string(), now)); - let admission = AdmissionGates { - registry: &gates.registry, - replay_guard: &gates.replay_guard, - rate_limiter: &gates.rate_limiter, - immediate_limiter: &gates.immediate_limiter, - usage_cache: &gates.usage_cache, - limits: &SMALL, - local_actor: &gates.local_actor, - requested_graphs: &gates.requested_graphs, - }; - assert!(admission.keeps(MessageKind::SyncGraph, Some(asked), now)); - assert!(!admission.keeps(MessageKind::SyncGraph, Some(unasked), now), "relayed only"); - assert!( - !admission.keeps(MessageKind::SyncGraph, Some(asked), now + SYNC_GRAPH_REQUEST_TTL), - "a request that has expired no longer expects an answer" - ); - gates.requested_graphs.remove(&asked.to_string()); - assert!(!admission.keeps(MessageKind::SyncGraph, Some(asked), now), "already served"); - // Other kinds are unaffected by the set. - assert!(admission.keeps(MessageKind::KickoffSent, None, now)); - assert!(!admission.keeps(MessageKind::NackReady, None, now), "not a committee message"); - assert_eq!(DropReason::ControlQueueFull.as_str(), "control_queue_full"); - } -} diff --git a/node/src/p2p_admission/gates.rs b/node/src/p2p_admission/gates.rs new file mode 100644 index 000000000..6d73b21cb --- /dev/null +++ b/node/src/p2p_admission/gates.rs @@ -0,0 +1,159 @@ +//! Cooldown gates: one response per key per window, and the throttled node +//! table refreshes. + +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +/// NodeInfo broadcast response cooldown. +const NODE_INFO_RESPONSE_COOLDOWN: Duration = Duration::from_secs(10); + +#[derive(Debug, Default)] +struct ResponseGateState { + sent_at: Option, + pending: bool, +} + +/// Coalesce NodeInfo requests into one response per cooldown. +#[derive(Debug)] +pub struct ResponseGate { + state: Mutex, + cooldown: Duration, +} + +impl ResponseGate { + pub fn new(cooldown: Duration) -> Self { + Self { state: Mutex::new(ResponseGateState::default()), cooldown } + } + + fn cooled_down(&self, state: &ResponseGateState, now: Instant) -> bool { + state.sent_at.is_none_or(|sent_at| now.saturating_duration_since(sent_at) >= self.cooldown) + } + + /// `true` when the caller should respond now; otherwise the request is + /// remembered for [`Self::take_pending`]. + pub fn try_respond(&self, now: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if self.cooled_down(&state, now) { + state.sent_at = Some(now); + state.pending = false; + true + } else { + state.pending = true; + false + } + } + + /// `true` when a deferred request is due; the caller then responds. + pub fn take_pending(&self, now: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.pending && self.cooled_down(&state, now) { + state.sent_at = Some(now); + state.pending = false; + true + } else { + false + } + } +} + +static NODE_INFO_RESPONSE_GATE: LazyLock = + LazyLock::new(|| ResponseGate::new(NODE_INFO_RESPONSE_COOLDOWN)); + +pub fn node_info_response_gate() -> &'static ResponseGate { + &NODE_INFO_RESPONSE_GATE +} + +/// Cooldown for peer liveness updates. +const PEER_TIMESTAMP_COOLDOWN: Duration = Duration::from_secs(30); +const COOLDOWN_GATE_MAX_ENTRIES: usize = 8_192; + +/// Minimum interval between full gate/set sweeps. +const GATE_SWEEP_INTERVAL: Duration = Duration::from_secs(5); + +/// Bounded keyed timestamps; reject new keys when all entries are live. +#[derive(Debug, Default)] +struct BoundedStamps { + stamps: HashMap, + swept_at: Option, +} + +impl BoundedStamps { + fn is_live(&self, key: &str, ttl: Duration, now: Instant) -> bool { + self.stamps.get(key).is_some_and(|at| now.saturating_duration_since(*at) < ttl) + } + + /// Stamp `key` with `now`. Returns false when the table is full of live + /// entries and `key` is not among them. + fn stamp(&mut self, key: &str, ttl: Duration, max_entries: usize, now: Instant) -> bool { + if !self.stamps.contains_key(key) && self.stamps.len() >= max_entries { + if self + .swept_at + .is_none_or(|at| now.saturating_duration_since(at) >= GATE_SWEEP_INTERVAL) + { + self.swept_at = Some(now); + self.stamps.retain(|_, at| now.saturating_duration_since(*at) < ttl); + } + if self.stamps.len() >= max_entries { + return false; + } + } + self.stamps.insert(key.to_string(), now); + true + } +} + +/// Allows an action for a key at most once per cooldown, within a hard bound on +/// its own memory. A key it has no room to track is denied: every action gated +/// here is one the peer retries, and none is worth an unbounded table. +#[derive(Debug)] +pub struct CooldownGate { + seen: Mutex, + cooldown: Duration, + max_entries: usize, +} + +impl CooldownGate { + pub(super) fn new(cooldown: Duration, max_entries: usize) -> Self { + Self { seen: Mutex::new(BoundedStamps::default()), cooldown, max_entries } + } + + /// Whether `key` is inside its cooldown. Records nothing. + #[cfg(test)] + pub fn is_cooling(&self, key: &str, now: Instant) -> bool { + let seen = self.seen.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + seen.is_live(key, self.cooldown, now) + } + + /// `true` when the action for `key` may proceed, recording it as just done. + pub fn allow(&self, key: &str, now: Instant) -> bool { + let mut seen = self.seen.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + !seen.is_live(key, self.cooldown, now) + && seen.stamp(key, self.cooldown, self.max_entries, now) + } + + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.seen.lock().unwrap().stamps.len() + } +} + +static PEER_TIMESTAMP_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(PEER_TIMESTAMP_COOLDOWN, COOLDOWN_GATE_MAX_ENTRIES)); + +pub fn peer_timestamp_gate() -> &'static CooldownGate { + &PEER_TIMESTAMP_GATE +} + +static NODE_TABLE_PURGED_AT: Mutex> = Mutex::new(None); + +/// `true` at most once per `interval`: when the `node` table purge should run. +pub fn node_table_purge_due(interval: Duration, now: Instant) -> bool { + let mut purged_at = + NODE_TABLE_PURGED_AT.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if purged_at.is_some_and(|at| now.saturating_duration_since(at) < interval) { + return false; + } + *purged_at = Some(now); + true +} diff --git a/node/src/p2p_admission/get_graph_budget.rs b/node/src/p2p_admission/get_graph_budget.rs new file mode 100644 index 000000000..3f135952b --- /dev/null +++ b/node/src/p2p_admission/get_graph_budget.rs @@ -0,0 +1,56 @@ +//! Budgets for serving `GetGraph` requests: per peer, per requester class +//! and concurrent reads. + +use std::sync::LazyLock; +use std::time::Duration; + +use super::gates::CooldownGate; +use super::rate::{GlobalRate, PeerRateLimiter}; + +/// How long one peer's identical `GetGraph` request stays on cooldown. +const GET_GRAPH_RESPONSE_COOLDOWN: Duration = Duration::from_secs(60); +const GRAPH_SET_MAX_ENTRIES: usize = 16_384; + +static GET_GRAPH_RESPONSE_GATE: LazyLock = + LazyLock::new(|| CooldownGate::new(GET_GRAPH_RESPONSE_COOLDOWN, GRAPH_SET_MAX_ENTRIES)); + +/// Cooldown for answering the same `GetGraph` request from the same peer. +pub fn get_graph_response_gate() -> &'static CooldownGate { + &GET_GRAPH_RESPONSE_GATE +} + +// Serving a definition reads and serializes up to tens of megabytes, so the +// total rate is bounded per requester class, whatever the number of peers. +static GET_GRAPH_RESPONSE_BUDGET_REGISTERED: LazyLock = + LazyLock::new(|| GlobalRate::new(32.0, 2.0)); +static GET_GRAPH_RESPONSE_BUDGET_UNREGISTERED: LazyLock = + LazyLock::new(|| GlobalRate::new(8.0, 0.2)); + +/// One requester's share: a registered peer flooding requests runs out of +/// its own tokens before it can drain the class budget other members need. +static GET_GRAPH_PEER_BUDGET: LazyLock = + LazyLock::new(|| PeerRateLimiter::new(8.0, 0.5, 1024)); + +pub fn get_graph_peer_budget() -> &'static PeerRateLimiter { + &GET_GRAPH_PEER_BUDGET +} + +/// Aggregate `GetGraph` response budgets by requester registration class. +pub fn get_graph_response_budget(requester_registered: bool) -> &'static GlobalRate { + if requester_registered { + &GET_GRAPH_RESPONSE_BUDGET_REGISTERED + } else { + &GET_GRAPH_RESPONSE_BUDGET_UNREGISTERED + } +} + +/// Definitions being read and serialized for `GetGraph` at once. A request +/// that finds no free slot is refused, never queued. +pub const GET_GRAPH_SERVE_SLOTS: usize = 4; + +static GET_GRAPH_SERVE_SLOTS_SEMAPHORE: LazyLock> = + LazyLock::new(|| std::sync::Arc::new(tokio::sync::Semaphore::new(GET_GRAPH_SERVE_SLOTS))); + +pub fn get_graph_serve_slots() -> &'static std::sync::Arc { + &GET_GRAPH_SERVE_SLOTS_SEMAPHORE +} diff --git a/node/src/p2p_admission/limits.rs b/node/src/p2p_admission/limits.rs new file mode 100644 index 000000000..56bd9656c --- /dev/null +++ b/node/src/p2p_admission/limits.rs @@ -0,0 +1,55 @@ +//! Message size limits and inbox quotas per sender class, from the +//! environment. + +use crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE; +use std::sync::LazyLock; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Quota { + pub rows: i64, + pub bytes: i64, +} + +impl Quota { + pub(super) fn admits(&self, rows: i64, bytes: i64) -> bool { + rows <= self.rows && bytes <= self.bytes + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct InboundLimits { + pub max_json_bytes: usize, + pub max_binary_bytes: usize, + /// Every stored network event, whatever its class. + pub global: Quota, + /// Part of the global budget only committee senders may fill. + pub committee_reserve: Quota, + /// Every stored network event of one registered sender, in whichever class. + pub registered_peer: Quota, +} + +impl InboundLimits { + /// Derive byte quotas from the global ceiling; row limits remain fixed. + pub fn with_inbox_bytes(max_json_bytes: usize, max_inbox_bytes: i64) -> Self { + Self { + max_json_bytes, + max_binary_bytes: MAX_GOSSIPSUB_TRANSMIT_SIZE, + global: Quota { rows: 16_384, bytes: max_inbox_bytes }, + committee_reserve: Quota { rows: 4_096, bytes: max_inbox_bytes / 4 }, + registered_peer: Quota { rows: 1_024, bytes: max_inbox_bytes / 8 }, + } + } + + pub fn from_env() -> Self { + Self::with_inbox_bytes( + crate::env::get_p2p_max_json_message_bytes(), + crate::env::get_p2p_inbox_max_bytes(), + ) + } +} + +static INBOUND_LIMITS: LazyLock = LazyLock::new(InboundLimits::from_env); + +pub fn inbound_limits() -> &'static InboundLimits { + &INBOUND_LIMITS +} diff --git a/node/src/p2p_admission/maintenance.rs b/node/src/p2p_admission/maintenance.rs new file mode 100644 index 000000000..5a508f1c6 --- /dev/null +++ b/node/src/p2p_admission/maintenance.rs @@ -0,0 +1,171 @@ +//! The admission maintenance pass: registry refreshes, drop and clock-skew +//! reports, bans that expired and stale node rows; and the persisted operator +//! bindings restored at boot. + +use crate::clock::current_time_secs; +use crate::messages::{NODE_INFO_BINDING_MAX_FUTURE_SECS, NodeInfo, verify_node_info_binding}; +use anyhow::Result; +use client::goat_chain::GOATClient; +use libp2p::PeerId; +use std::str::FromStr; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use store::localdb::LocalDB; + +/// A `node` row of an unregistered peer may be evicted to make room for a new +/// one once it has gone this long without a refresh: three missed heartbeats. +pub const NODE_TABLE_EVICTABLE_AFTER_SECS: i64 = 3 * crate::env::HEARTBEAT_INTERVAL_SECOND as i64; + +/// Maximum unregistered node rows. +pub const NODE_TABLE_MAX_UNREGISTERED_ROWS: i64 = 4_096; + +/// A `node` row of a peer that is not registered is deleted once it has not been +/// refreshed for this long; live nodes refresh theirs on every heartbeat. +const NODE_TABLE_STALE_SECS: i64 = 7 * 24 * 60 * 60; + +const NODE_TABLE_PURGE_INTERVAL: Duration = Duration::from_secs(60 * 60); + +/// Restore persisted operator bindings, re-verify signatures and refresh stake asynchronously. +pub async fn load_persisted_peer_bindings( + local_db: &LocalDB, + goat_client: &Arc, +) -> Result { + let bindings = local_db + .acquire() + .await? + .load_p2p_peer_bindings(crate::p2p_admission::OPERATOR_BINDING_LOAD_LIMIT) + .await?; + let registry = crate::p2p_admission::peer_registry(); + let mut loaded = 0; + let mut verified_bindings = std::collections::HashMap::new(); + for node in bindings { + let node_info = NodeInfo { + peer_id: node.peer_id.clone(), + btc_pub_key: node.btc_pub_key.clone(), + binding_sig: node.binding_sig.clone(), + binding_issued_at: node.binding_issued_at, + ..Default::default() + }; + let Some(operator_key) = verify_node_info_binding(&node_info) else { + continue; + }; + if node.binding_issued_at > current_time_secs() + NODE_INFO_BINDING_MAX_FUTURE_SECS { + continue; + } + if let Ok(peer_id) = PeerId::from_str(&node.peer_id) { + let operator_key = crate::p2p_admission::operator_key(&operator_key); + registry.observe_operator_binding(&peer_id, &operator_key, node.binding_issued_at); + verified_bindings.insert(peer_id, (operator_key, node.binding_issued_at)); + loaded += 1; + } + } + // Apply configured bindings only where no signed binding exists; stake still requires confirmation. + for (peer_id, operator_key) in crate::env::get_p2p_trusted_operator_bindings() { + match registry.trust_operator_binding(&peer_id, &operator_key) { + Ok(()) => loaded += 1, + Err(conflict) => tracing::warn!( + event = "p2p_admission", + outcome = "trusted_binding_ignored", + peer_id = %peer_id, + configured_key = %hex::encode(operator_key), + conflict = ?conflict, + env = crate::env::ENV_P2P_TRUSTED_OPERATOR_BINDINGS, + "configured operator binding disagrees with one the operator signed; keeping \ + the signed binding — update or remove the configured entry" + ), + } + } + // Peers confirmed on chain in an earlier session keep their class and are + // re-validated as known members, outside the discovery budget. + loaded += crate::p2p_admission::seed_registry_from_store( + local_db, + registry, + &verified_bindings, + Instant::now(), + ) + .await?; + // Everything loaded is due; the unconfirmed remainder is picked up by the + // regular tick as lookup slots free up. + crate::p2p_admission::refresh_due_registrations_in_background(local_db, goat_client); + Ok(loaded) +} + +/// A clock disagreement below this is not worth reporting. +const CLOCK_LEAD_REPORT_THRESHOLD: Duration = Duration::from_secs(60); + +/// Warn about registered authors whose message numbering is ahead of the local clock. +fn report_clock_leads(leads: Vec<(PeerId, Duration)>) { + let ahead: Vec<&(PeerId, Duration)> = + leads.iter().filter(|(_, lead)| *lead >= CLOCK_LEAD_REPORT_THRESHOLD).collect(); + let max_lead = ahead.iter().map(|(_, lead)| *lead).max().unwrap_or_default(); + match ahead.as_slice() { + [] => {} + [(peer, lead)] => tracing::warn!( + event = "p2p_admission", + outcome = "author_clock_ahead", + peer_id = %peer, + lead_secs = lead.as_secs(), + "a registered author's clock is ahead of this node's; past the tolerance its \ + messages are refused as coming from the future" + ), + several => tracing::error!( + event = "p2p_admission", + outcome = "local_clock_behind_suspected", + authors = several.len(), + max_lead_secs = max_lead.as_secs(), + "several registered authors' clocks are ahead of this node's: check this node's \ + clock (NTP). Past the tolerance their messages are refused as coming from the future" + ), + } +} + +/// Refresh registrations, summarize drops and purge stale node rows. +pub async fn run_p2p_admission_maintenance(local_db: &LocalDB, goat_client: &Arc) { + let now = Instant::now(); + crate::p2p_admission::refresh_due_registrations_in_background(local_db, goat_client); + crate::p2p_admission::revalidate_committee_candidates(local_db, goat_client).await; + crate::p2p_admission::log_drop_summary(); + // Expired bans only need forgetting: the gate compares against the clock. + crate::p2p_admission::direct_peer_strikes().take_expired_bans(now); + report_clock_leads(crate::p2p_admission::replay_guard().take_clock_leads()); + for (peer, rejected) in crate::p2p_admission::replay_guard().take_lockout_suspects() { + tracing::warn!( + event = "p2p_admission", + outcome = "replay_lockout_suspected", + peer_id = %peer, + rejected, + "a registered author keeps being refused as a replay; if it is not under \ + attack, its clock stepped back or another instance shares its key" + ); + } + + if !crate::p2p_admission::node_table_purge_due(NODE_TABLE_PURGE_INTERVAL, now) { + return; + } + let result = async { + local_db + .acquire() + .await? + .purge_stale_unregistered_nodes( + current_time_secs() - NODE_TABLE_STALE_SECS, + &crate::env::get_peer_id(), + ) + .await + } + .await; + match result { + Ok(0) => {} + Ok(purged) => tracing::info!( + event = "p2p_admission", + outcome = "stale_nodes_purged", + purged, + "removed node rows of unregistered peers that stopped announcing themselves" + ), + Err(error) => tracing::warn!( + event = "p2p_admission", + outcome = "maintenance_write_failed", + error = %error, + "failed to purge stale node rows; retried next tick" + ), + } +} diff --git a/node/src/p2p_admission/mod.rs b/node/src/p2p_admission/mod.rs new file mode 100644 index 000000000..dfee29623 --- /dev/null +++ b/node/src/p2p_admission/mod.rs @@ -0,0 +1,468 @@ +//! Inbound gossip admission: envelope, identity, replay, rate and storage checks. +//! Durable messages are admitted before persistence and forwarding. +//! Rejected messages use gossipsub `Ignore`. + +mod gates; +mod get_graph_budget; +mod limits; +mod maintenance; +mod rate; +mod registry; +mod registry_refresh; +mod replay; +mod strikes; +#[cfg(test)] +mod tests; + +use crate::messages::{GOATMessage, GOATMessageContent, P2PMessageDelivery}; +use anyhow::Result; +use libp2p::PeerId; +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; +use store::InboxClassUsage; +use store::localdb::LocalDB; + +pub use gates::{ + CooldownGate, ResponseGate, node_info_response_gate, node_table_purge_due, peer_timestamp_gate, +}; +pub use get_graph_budget::{ + GET_GRAPH_SERVE_SLOTS, get_graph_peer_budget, get_graph_response_budget, + get_graph_response_gate, get_graph_serve_slots, +}; +pub use limits::{InboundLimits, Quota, inbound_limits}; +pub use maintenance::{ + NODE_TABLE_EVICTABLE_AFTER_SECS, NODE_TABLE_MAX_UNREGISTERED_ROWS, + load_persisted_peer_bindings, run_p2p_admission_maintenance, +}; +pub use rate::{ + GlobalRate, InboundRateLimiter, PeerRateLimiter, RateLimits, TierRate, immediate_limiter, + inbound_rate_limiter, +}; +pub use registry::{ + OPERATOR_BINDING_LOAD_LIMIT, OperatorKey, PeerRegistry, RefreshPlan, SenderClass, + TrustedBindingConflict, operator_key, peer_registry, +}; +pub use registry_refresh::{ + refresh_due_registrations_in_background, refresh_neighbour_in_background, + refresh_operator_binding_in_background, refresh_registry_in_background, + revalidate_committee_candidates, revoke_persisted_operator, seed_registry_from_store, +}; +pub use replay::{ReplayGuard, replay_guard, unix_nanos_now}; +pub use strikes::{DirectPeerStrikes, direct_peer_strikes, note_refused_peer}; + +/// How long a rejected network event keeps its inbox row, and so its share +/// of the author's quota, before the maintenance purge removes it. +pub const REJECTED_EVENT_TTL_SECS: i64 = 2 * 60 * 60; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum DropReason { + OversizedJson, + OversizedBinary, + BinaryFromNonVerifier, + /// A graph parameter endorsement from a peer outside the loaded verifier + /// set: verifying one costs chain reads, and only verifiers send them. + EndorsementFromNonVerifier, + /// A registered author's message whose sequence number was already seen, or + /// is older than the replay window: a verbatim re-publish by someone else. + ReplayedSequence, + /// Sequence number exceeds the accepted clock lead. + FutureSequence, + /// A message addressed to another role, from an unregistered author, beyond + /// what this node relays for such authors. + ForwardRate, + DirectPeerRate, + AuthorRate, + UnregisteredRate, + GlobalRate, + Undecodable, + UnexpectedBinaryKind, + ImmediateRateLimited, + /// The bounded queue in front of the control worker (NodeInfo, ACKs) had no + /// room, by rows or by bytes. Distinct from `ImmediateRateLimited`: that one + /// is a sender exceeding its rate, this one is the node falling behind. + ControlQueueFull, + SenderQuota, + GlobalQuota, + /// Not a message drop: a `NodeInfo` from a sender that is not registered was + /// not stored because the `node` table is at its ceiling for such senders. + NodeTableFull, + /// NodeInfo rejected for invalid identity, fields or binding. + NodeInfoRejected, + /// Not a drop: the engine already held the event (same author and + /// content); recorded in the periodic summary. + DuplicatePayload, +} + +impl DropReason { + pub const fn as_str(self) -> &'static str { + match self { + Self::OversizedJson => "oversized_json", + Self::OversizedBinary => "oversized_binary", + Self::BinaryFromNonVerifier => "binary_from_non_verifier", + Self::EndorsementFromNonVerifier => "endorsement_from_non_verifier", + Self::ReplayedSequence => "replayed_sequence", + Self::FutureSequence => "future_sequence", + Self::ForwardRate => "forward_rate", + Self::DirectPeerRate => "direct_peer_rate", + Self::AuthorRate => "author_rate", + Self::UnregisteredRate => "unregistered_rate", + Self::GlobalRate => "global_rate", + Self::Undecodable => "undecodable", + Self::UnexpectedBinaryKind => "unexpected_binary_kind", + Self::ImmediateRateLimited => "immediate_rate_limited", + Self::ControlQueueFull => "control_queue_full", + Self::SenderQuota => "sender_quota", + Self::GlobalQuota => "global_quota", + Self::NodeTableFull => "node_table_full", + Self::NodeInfoRejected => "node_info_rejected", + Self::DuplicatePayload => "duplicate_payload", + } + } + + /// Whether a drop is attributable to the direct peer, independent of local state and configuration. + pub const fn blames_direct_peer(self) -> bool { + matches!(self, Self::OversizedBinary | Self::Undecodable | Self::UnexpectedBinaryKind) + } + + /// Whether the payload was decoded before it was dropped. + pub const fn after_decode(self) -> bool { + matches!( + self, + Self::UnexpectedBinaryKind + | Self::EndorsementFromNonVerifier + | Self::ForwardRate + | Self::ImmediateRateLimited + | Self::ControlQueueFull + | Self::SenderQuota + | Self::GlobalQuota + ) + } +} + +/// Checks that need nothing but the raw bytes and the sender's identity. +pub fn check_envelope( + data: &[u8], + sender_is_verifier: bool, + limits: &InboundLimits, +) -> Result<(), DropReason> { + if GOATMessage::is_binary_envelope(data) { + if !sender_is_verifier { + return Err(DropReason::BinaryFromNonVerifier); + } + if data.len() > limits.max_binary_bytes { + return Err(DropReason::OversizedBinary); + } + } else if data.len() > limits.max_json_bytes { + return Err(DropReason::OversizedJson); + } + Ok(()) +} + +/// Check whether a registered author's next inbox row fits its quotas. +/// Usage excludes the new row; unregistered authors are not persisted. +pub fn check_inbox_quota( + class: SenderClass, + content_len: usize, + usage: &[InboxClassUsage], + limits: &InboundLimits, +) -> Result<(), DropReason> { + let content_len = content_len as i64; + let total_rows: i64 = usage.iter().map(|usage| usage.rows).sum(); + let total_bytes: i64 = usage.iter().map(|usage| usage.bytes).sum(); + if !limits.global.admits(total_rows + 1, total_bytes + content_len) { + return Err(DropReason::GlobalQuota); + } + if class != SenderClass::Committee { + let committee = SenderClass::Committee.to_string(); + let (rows, bytes) = usage + .iter() + .filter(|usage| usage.admission_class != committee) + .fold((0, 0), |(rows, bytes), usage| (rows + usage.rows, bytes + usage.bytes)); + let open = Quota { + rows: limits.global.rows - limits.committee_reserve.rows, + bytes: limits.global.bytes - limits.committee_reserve.bytes, + }; + if !open.admits(rows + 1, bytes + content_len) { + return Err(DropReason::GlobalQuota); + } + } + + // A sender can hold rows in more than one class: it is classified when each + // row arrives, and a committee lookup may complete in between. + let peer_rows: i64 = usage.iter().map(|usage| usage.peer_rows).sum(); + let peer_bytes: i64 = usage.iter().map(|usage| usage.peer_bytes).sum(); + if !limits.registered_peer.admits(peer_rows + 1, peer_bytes + content_len) { + return Err(DropReason::SenderQuota); + } + Ok(()) +} + +/// Aggregate drop counters for periodic logging. +#[derive(Debug, Default)] +pub struct DropStats { + counts: Mutex>, +} + +impl DropStats { + pub fn record(&self, reason: DropReason) { + let mut counts = self.counts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + *counts.entry(reason).or_default() += 1; + } + + pub fn take(&self) -> Vec<(DropReason, u64)> { + let mut counts = self.counts.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut taken: Vec<_> = counts.drain().collect(); + taken.sort_by_key(|(reason, _)| reason.as_str()); + taken + } +} + +static DROP_STATS: LazyLock = LazyLock::new(DropStats::default); + +pub fn record_drop(reason: DropReason) { + DROP_STATS.record(reason); +} + +/// Emit one line for everything dropped since the previous call. +pub fn log_drop_summary() { + let dropped = DROP_STATS.take(); + if dropped.is_empty() { + return; + } + let total: u64 = dropped.iter().map(|(_, count)| count).sum(); + let breakdown = dropped + .iter() + .map(|(reason, count)| format!("{}={count}", reason.as_str())) + .collect::>() + .join(" "); + tracing::warn!( + event = "p2p_admission", + outcome = "summary", + total, + breakdown, + "inbound P2P messages dropped, de-duplicated or admitted unlogged by admission control" + ); +} + +/// How long the per-class inbox totals are served from memory. +const INBOX_CLASS_TOTALS_TTL: Duration = Duration::from_secs(2); + +/// Cache per-class engine inbox totals and increment them on admission. +/// Applies and purges are reflected at the next cache refresh. +#[derive(Debug, Default)] +pub struct InboxUsageCache { + totals: Mutex)>>, +} + +impl InboxUsageCache { + fn fresh(&self, now: Instant) -> Option> { + let totals = self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + totals + .as_ref() + .filter(|(at, _)| now.saturating_duration_since(*at) < INBOX_CLASS_TOTALS_TTL) + .map(|(_, totals)| totals.clone()) + } + + fn store(&self, totals: Vec, now: Instant) { + *self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some((now, totals)); + } + + /// Count a row that is about to be queued. + fn note_admitted(&self, class: SenderClass, bytes: i64) { + let mut totals = self.totals.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let Some((_, totals)) = totals.as_mut() else { + return; + }; + let class = class.to_string(); + match totals.iter_mut().find(|usage| usage.admission_class == class) { + Some(usage) => { + usage.rows += 1; + usage.bytes += bytes; + } + None => totals.push(InboxClassUsage { + admission_class: class, + rows: 1, + bytes, + ..Default::default() + }), + } + } + + /// Combine cached class totals with fresh per-peer usage. + async fn usage( + &self, + storage: &mut store::localdb::StorageProcessor<'_>, + from_peer: &str, + now: Instant, + ) -> Result> { + let mut usage = match self.fresh(now) { + Some(totals) => totals, + None => { + let totals = storage.engine_inbox_class_totals().await?; + self.store(totals.clone(), now); + totals + } + }; + for peer in storage.engine_inbox_peer_usage(from_peer).await? { + match usage.iter_mut().find(|total| total.admission_class == peer.admission_class) { + Some(total) => { + total.peer_rows = peer.peer_rows; + total.peer_bytes = peer.peer_bytes; + } + None => usage.push(peer), + } + } + Ok(usage) + } +} + +static INBOX_USAGE_CACHE: LazyLock = LazyLock::new(InboxUsageCache::default); + +/// Injectable admission gates. +#[derive(Clone, Copy)] +pub struct AdmissionGates<'a> { + pub registry: &'a PeerRegistry, + pub replay_guard: &'a ReplayGuard, + pub rate_limiter: &'a InboundRateLimiter, + pub immediate_limiter: &'a PeerRateLimiter, + pub usage_cache: &'a InboxUsageCache, + pub limits: &'a InboundLimits, + /// This node's role. A durable message no handler of this role acts on is + /// relayed but not stored. + pub local_actor: &'a bitvm_lib::actors::Actor, +} + +impl AdmissionGates<'_> { + /// Whether this node stores an admitted durable message of `kind`, or only + /// relays it. + pub fn keeps(&self, kind: crate::messages::MessageKind) -> bool { + kind.handled_by(self.local_actor) + } +} + +impl<'a> AdmissionGates<'a> { + /// The process-wide gates the node runs with. + pub fn global(local_actor: &'a bitvm_lib::actors::Actor) -> Self { + Self { + local_actor, + registry: peer_registry(), + replay_guard: replay_guard(), + rate_limiter: inbound_rate_limiter(), + immediate_limiter: immediate_limiter(), + usage_cache: &INBOX_USAGE_CACHE, + limits: inbound_limits(), + } + } +} + +/// One received gossipsub message, as admission sees it. +#[derive(Clone, Copy, Debug)] +pub struct InboundGossip<'a> { + /// The neighbour that relayed the message to this node. + pub direct_peer: &'a PeerId, + /// The signed author. + pub source: &'a PeerId, + /// The author's gossipsub sequence number, covered by its signature. + pub sequence_number: Option, + pub data: &'a [u8], +} + +/// What to do with one inbound gossipsub message. +pub enum InboundVerdict { + /// Neither processed nor forwarded. + Drop(DropReason), + /// Dispatch now; nothing is persisted. + Immediate(GOATMessage), + /// Forward without persistence, dispatch or ACK. + Forward, + /// Record as an engine inbox event of its aggregate; the engine + /// deduplicates by content id. + Enqueue { message: GOATMessage, class: SenderClass }, +} + +/// Drop the rejected network events the quota should stop charging for, +/// after `REJECTED_EVENT_TTL_SECS`. The sender's outbox re-sends what still +/// matters; the terminal retention of the engine is for audit, not quota. +pub async fn purge_stale_network_events(local_db: &LocalDB) -> Result { + let before = crate::clock::current_time_secs() - REJECTED_EVENT_TTL_SECS; + let purged = local_db.acquire().await?.purge_rejected_network_inbox_rows(before).await?; + if purged > 0 { + tracing::info!(event = "p2p_admission", purged, "rejected network events purged"); + } + Ok(purged) +} + +/// Evaluate admission without swarm or chain access; propagate database errors. +pub async fn evaluate_inbound_message( + local_db: &LocalDB, + gates: &AdmissionGates<'_>, + inbound: &InboundGossip<'_>, + now: Instant, +) -> Result { + let InboundGossip { direct_peer, source, sequence_number, data } = *inbound; + if let Err(reason) = check_envelope(data, gates.registry.is_verifier(source), gates.limits) { + return Ok(InboundVerdict::Drop(reason)); + } + // Classification reads only the cache and never blocks. + let class = gates.registry.sender_class(source, now); + // Check replay before charging the registered author's rate budget. + if class.is_registered() + && let Err(reason) = gates.replay_guard.admit(source, sequence_number, unix_nanos_now()) + { + return Ok(InboundVerdict::Drop(reason)); + } + // Apply rate limits before decoding. + if let Err(reason) = + gates.rate_limiter.charge(direct_peer, source, class, data.len() as i64, now) + { + return Ok(InboundVerdict::Drop(reason)); + } + let Ok(message) = GOATMessage::deserialize_message(data).await else { + return Ok(InboundVerdict::Drop(DropReason::Undecodable)); + }; + // Require GenCircuits inside the binary envelope. + if GOATMessage::is_binary_envelope(data) + && !matches!(message.content, GOATMessageContent::GenCircuits(_)) + { + return Ok(InboundVerdict::Drop(DropReason::UnexpectedBinaryKind)); + } + // A verifier's endorsement of the graph parameters comes from the closed + // on-chain verifier set; once that set is loaded nobody else gets a + // committee node to verify one, which costs chain reads. + if matches!(message.content, GOATMessageContent::VerifierGraphParamsEndorsement(_)) + && gates.registry.verifier_set_loaded() + && !gates.registry.is_verifier(source) + { + return Ok(InboundVerdict::Drop(DropReason::EndorsementFromNonVerifier)); + } + + if message.content.p2p_delivery() == P2PMessageDelivery::Immediate { + if !gates.immediate_limiter.allow(source, now) { + return Ok(InboundVerdict::Drop(DropReason::ImmediateRateLimited)); + } + return Ok(InboundVerdict::Immediate(message)); + } + + // Forward unhandled or unregistered messages within the forwarding budget. + // Only registered authors enter the durable inbox; registration is learned + // through NodeInfo and registry checks. + if !gates.keeps(message.content.kind()) || !class.is_registered() { + if !class.is_registered() + && let Err(reason) = gates.rate_limiter.charge_forward(data.len() as i64, now) + { + return Ok(InboundVerdict::Drop(reason)); + } + return Ok(InboundVerdict::Forward); + } + + let from_peer = source.to_string(); + let mut storage = local_db.acquire().await?; + let usage = gates.usage_cache.usage(&mut storage, &from_peer, now).await?; + if let Err(reason) = check_inbox_quota(class, data.len(), &usage, gates.limits) { + return Ok(InboundVerdict::Drop(reason)); + } + gates.usage_cache.note_admitted(class, data.len() as i64); + Ok(InboundVerdict::Enqueue { message, class }) +} diff --git a/node/src/p2p_admission/rate.rs b/node/src/p2p_admission/rate.rs new file mode 100644 index 000000000..142831f81 --- /dev/null +++ b/node/src/p2p_admission/rate.rs @@ -0,0 +1,405 @@ +//! Token buckets: per peer for immediate messages, per class and peer for the +//! inbox, and global for a resource no peer key bounds. + +use anyhow::Result; +use libp2p::PeerId; +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +use super::DropReason; +use super::registry::SenderClass; + +const IMMEDIATE_BURST: f64 = 240.0; +const IMMEDIATE_REFILL_PER_SEC: f64 = 2.0; +const IMMEDIATE_MAX_TRACKED_PEERS: usize = 4096; + +/// Per-peer token bucket for messages that are dispatched without being queued. +#[derive(Debug)] +pub struct PeerRateLimiter { + buckets: Mutex>, + burst: f64, + refill_per_sec: f64, + max_peers: usize, +} + +impl PeerRateLimiter { + pub fn new(burst: f64, refill_per_sec: f64, max_peers: usize) -> Self { + Self { buckets: Mutex::new(HashMap::new()), burst, refill_per_sec, max_peers } + } + + pub fn allow(&self, peer: &PeerId, now: Instant) -> bool { + let mut buckets = self.buckets.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !buckets.contains_key(peer) && buckets.len() >= self.max_peers { + // A bucket that has refilled completely belongs to an idle peer and + // carries no state worth keeping. + let (burst, refill_per_sec) = (self.burst, self.refill_per_sec); + buckets.retain(|_, bucket| { + let elapsed = now.saturating_duration_since(bucket.refilled_at).as_secs_f64(); + bucket.tokens + elapsed * refill_per_sec < burst + }); + if buckets.len() >= self.max_peers { + return false; + } + } + let bucket = buckets.entry(*peer).or_insert_with(|| RateBucket::full(self.burst, now)); + bucket.refill(self.refill_per_sec, self.burst, now); + if bucket.tokens < 1.0 { + return false; + } + bucket.tokens -= 1.0; + true + } +} + +static IMMEDIATE_LIMITER: LazyLock = LazyLock::new(|| { + PeerRateLimiter::new(IMMEDIATE_BURST, IMMEDIATE_REFILL_PER_SEC, IMMEDIATE_MAX_TRACKED_PEERS) +}); + +pub fn immediate_limiter() -> &'static PeerRateLimiter { + &IMMEDIATE_LIMITER +} + +/// A token bucket carrying both a rate and a burst, refilled lazily on access. +#[derive(Clone, Copy, Debug)] +struct RateBucket { + tokens: f64, + refilled_at: Instant, +} + +impl RateBucket { + fn full(burst: f64, now: Instant) -> Self { + Self { tokens: burst, refilled_at: now } + } + + fn refill(&mut self, rate: f64, burst: f64, now: Instant) { + let elapsed = now.saturating_duration_since(self.refilled_at).as_secs_f64(); + self.tokens = (self.tokens + elapsed * rate).min(burst); + self.refilled_at = now; + } +} + +/// One message costs a whole message token plus `bytes` byte tokens. +#[derive(Clone, Copy, Debug)] +struct DualBucket { + msgs: RateBucket, + bytes: RateBucket, +} + +/// Rate + burst for one tier, in messages and bytes per second. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TierRate { + pub msg_burst: u32, + pub msg_per_sec: u32, + pub byte_burst: i64, + pub byte_per_sec: i64, +} + +impl TierRate { + fn new_bucket(&self, now: Instant) -> DualBucket { + DualBucket { + msgs: RateBucket::full(self.msg_burst as f64, now), + bytes: RateBucket::full(self.byte_burst as f64, now), + } + } + + /// Refill then test whether one message of `bytes` fits, without spending. + fn peek(&self, bucket: &mut DualBucket, bytes: i64, now: Instant) -> bool { + self.peek_above(bucket, bytes, 0.0, now) + } + + /// Check capacity while retaining the specified fraction of burst tokens. + fn peek_above(&self, bucket: &mut DualBucket, bytes: i64, reserve: f64, now: Instant) -> bool { + bucket.msgs.refill(self.msg_per_sec as f64, self.msg_burst as f64, now); + bucket.bytes.refill(self.byte_per_sec as f64, self.byte_burst as f64, now); + bucket.msgs.tokens - 1.0 >= self.msg_burst as f64 * reserve + && bucket.bytes.tokens - bytes as f64 >= self.byte_burst as f64 * reserve + } + + fn spend(&self, bucket: &mut DualBucket, bytes: i64) { + bucket.msgs.tokens -= 1.0; + bucket.bytes.tokens -= bytes as f64; + } +} + +/// Pre-decode message and byte limits: direct peer, author, unregistered class and global. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct RateLimits { + pub direct_peer: TierRate, + pub registered_author: TierRate, + pub unregistered_author: TierRate, + pub unregistered_total: TierRate, + /// Forward-only budget for unregistered authors. + pub unregistered_forward: TierRate, + pub global: TierRate, + /// Committee-only reserve in the global and direct-peer budgets. + pub committee_reserve: f64, + pub max_tracked_peers: usize, +} + +impl RateLimits { + pub const fn defaults() -> Self { + const MIB: i64 = 1024 * 1024; + Self { + // Direct-peer rate ceiling. + direct_peer: TierRate { + msg_burst: 16_384, + msg_per_sec: 4_096, + byte_burst: 512 * MIB, + byte_per_sec: 32 * MIB, + }, + // Generous: one GenCircuits is ~11.5 MiB, and an operator may send a + // few graph messages back to back. + registered_author: TierRate { + msg_burst: 8_192, + msg_per_sec: 512, + byte_burst: 256 * MIB, + byte_per_sec: 8 * MIB, + }, + // Room for one maximum-size JSON message and a node's heartbeats, + // not for a stream. + unregistered_author: TierRate { + msg_burst: 32, + msg_per_sec: 2, + byte_burst: 4 * MIB, + byte_per_sec: 128 * 1024, + }, + // Shared budget for all unregistered authors. + unregistered_total: TierRate { + msg_burst: 1_024, + msg_per_sec: 32, + byte_burst: 32 * MIB, + byte_per_sec: MIB, + }, + // Sized for the chain observations honest unregistered nodes + // (watchtowers, challengers) publish, not for bulk. + unregistered_forward: TierRate { + msg_burst: 256, + msg_per_sec: 16, + byte_burst: 8 * MIB, + byte_per_sec: 512 * 1024, + }, + global: TierRate { + msg_burst: 16_384, + msg_per_sec: 4_096, + byte_burst: 512 * MIB, + byte_per_sec: 32 * MIB, + }, + committee_reserve: 0.25, + max_tracked_peers: 8_192, + } + } +} + +/// Idle duration before forgetting a tracked bucket. +pub(super) const RATE_BUCKET_IDLE_FORGET: Duration = Duration::from_secs(60); +/// Minimum interval between full-table sweeps. +const RATE_TABLE_SWEEP_INTERVAL: Duration = Duration::from_secs(5); + +/// Bounded per-key buckets; excess keys share one overflow bucket. +#[derive(Debug)] +struct BucketTable { + buckets: HashMap, + overflow: DualBucket, + swept_at: Option, +} + +/// Which bucket of a [`BucketTable`] a message is charged to. +#[derive(Clone, Copy, Debug)] +enum BucketSlot { + Tracked(PeerId), + Overflow, +} + +impl BucketTable { + fn new(overflow_tier: TierRate, now: Instant) -> Self { + Self { buckets: HashMap::new(), overflow: overflow_tier.new_bucket(now), swept_at: None } + } + + /// The slot `key` is charged to. `pinned` keys are always tracked: they are + /// bounded by an on-chain registry, not by whoever generates key pairs. + fn slot( + &mut self, + key: &PeerId, + tier: TierRate, + cap: usize, + pinned: bool, + now: Instant, + ) -> BucketSlot { + if !self.buckets.contains_key(key) { + if self.buckets.len() >= cap + && self + .swept_at + .is_none_or(|at| now.saturating_duration_since(at) >= RATE_TABLE_SWEEP_INTERVAL) + { + self.swept_at = Some(now); + self.buckets.retain(|_, bucket| { + now.saturating_duration_since(bucket.msgs.refilled_at) < RATE_BUCKET_IDLE_FORGET + }); + } + if self.buckets.len() >= cap && !pinned { + return BucketSlot::Overflow; + } + self.buckets.insert(*key, tier.new_bucket(now)); + } + BucketSlot::Tracked(*key) + } + + fn get(&self, slot: BucketSlot) -> DualBucket { + match slot { + BucketSlot::Tracked(key) => self.buckets[&key], + BucketSlot::Overflow => self.overflow, + } + } + + fn set(&mut self, slot: BucketSlot, bucket: DualBucket) { + match slot { + BucketSlot::Tracked(key) => { + self.buckets.insert(key, bucket); + } + BucketSlot::Overflow => self.overflow = bucket, + } + } +} + +#[derive(Debug)] +struct RateState { + direct: BucketTable, + authors: BucketTable, + unregistered_total: DualBucket, + unregistered_forward: DualBucket, + global: DualBucket, +} + +/// Multi-tier pre-decode rate budget. +#[derive(Debug)] +pub struct InboundRateLimiter { + state: Mutex, + limits: RateLimits, +} + +impl InboundRateLimiter { + pub fn new(limits: RateLimits) -> Self { + let now = Instant::now(); + Self { + state: Mutex::new(RateState { + direct: BucketTable::new(limits.direct_peer, now), + authors: BucketTable::new(limits.unregistered_author, now), + unregistered_total: limits.unregistered_total.new_bucket(now), + unregistered_forward: limits.unregistered_forward.new_bucket(now), + global: limits.global.new_bucket(now), + }), + limits, + } + } + + /// Charge the relay budget for a message from an unregistered author that + /// this node forwards without storing. + pub fn charge_forward(&self, bytes: i64, now: Instant) -> Result<(), DropReason> { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let tier = self.limits.unregistered_forward; + let mut bucket = state.unregistered_forward; + let admitted = tier.peek(&mut bucket, bytes, now); + if admitted { + tier.spend(&mut bucket, bytes); + } + state.unregistered_forward = bucket; + admitted.then_some(()).ok_or(DropReason::ForwardRate) + } + + #[cfg(test)] + pub(super) fn tracked_authors(&self) -> usize { + self.state.lock().unwrap().authors.buckets.len() + } + + /// Check all applicable rate tiers before charging any of them. + pub fn charge( + &self, + direct_peer: &PeerId, + author: &PeerId, + class: SenderClass, + bytes: i64, + now: Instant, + ) -> Result<(), DropReason> { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let author_tier = if class.is_registered() { + self.limits.registered_author + } else { + self.limits.unregistered_author + }; + let cap = self.limits.max_tracked_peers; + let direct_tier = self.limits.direct_peer; + let registered = class.is_registered(); + let direct_slot = state.direct.slot(direct_peer, direct_tier, cap, false, now); + let author_slot = state.authors.slot(author, author_tier, cap, registered, now); + let mut direct = state.direct.get(direct_slot); + let mut author_bucket = state.authors.get(author_slot); + let mut unregistered = state.unregistered_total; + let mut global = state.global; + + let charge_unregistered = !class.is_registered(); + // The shared tiers keep a reserve only the committee may draw on. + let reserve = + if class == SenderClass::Committee { 0.0 } else { self.limits.committee_reserve }; + let direct_ok = direct_tier.peek_above(&mut direct, bytes, reserve, now); + let author_ok = author_tier.peek(&mut author_bucket, bytes, now); + let unregistered_ok = !charge_unregistered + || self.limits.unregistered_total.peek(&mut unregistered, bytes, now); + let global_ok = self.limits.global.peek_above(&mut global, bytes, reserve, now); + + // Persist refilled buckets even when admission fails. + let outcome = if !global_ok { + Err(DropReason::GlobalRate) + } else if !unregistered_ok { + Err(DropReason::UnregisteredRate) + } else if !author_ok { + Err(DropReason::AuthorRate) + } else if !direct_ok { + Err(DropReason::DirectPeerRate) + } else { + direct_tier.spend(&mut direct, bytes); + author_tier.spend(&mut author_bucket, bytes); + if charge_unregistered { + self.limits.unregistered_total.spend(&mut unregistered, bytes); + } + self.limits.global.spend(&mut global, bytes); + Ok(()) + }; + state.direct.set(direct_slot, direct); + state.authors.set(author_slot, author_bucket); + state.unregistered_total = unregistered; + state.global = global; + outcome + } +} + +static INBOUND_RATE_LIMITER: LazyLock = + LazyLock::new(|| InboundRateLimiter::new(RateLimits::defaults())); + +pub fn inbound_rate_limiter() -> &'static InboundRateLimiter { + &INBOUND_RATE_LIMITER +} + +/// A global token bucket, for a resource no per-peer key bounds. +#[derive(Debug)] +pub struct GlobalRate { + bucket: Mutex, + rate: f64, + burst: f64, +} + +impl GlobalRate { + pub(super) fn new(burst: f64, rate: f64) -> Self { + Self { bucket: Mutex::new(RateBucket::full(burst, Instant::now())), rate, burst } + } + + pub fn allow(&self, now: Instant) -> bool { + let mut bucket = self.bucket.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + bucket.refill(self.rate, self.burst, now); + if bucket.tokens < 1.0 { + return false; + } + bucket.tokens -= 1.0; + true + } +} diff --git a/node/src/p2p_admission/registry.rs b/node/src/p2p_admission/registry.rs new file mode 100644 index 000000000..7ef4c8e50 --- /dev/null +++ b/node/src/p2p_admission/registry.rs @@ -0,0 +1,642 @@ +//! The peer registry: which peers are committee members, verifiers or +//! operators on chain, with the budgets of the lookups that keep it current. + +use anyhow::Result; +use libp2p::PeerId; +use std::collections::{HashMap, HashSet}; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; +use strum::{Display, EnumString}; + +/// How admission classifies the author of a message: a committee member +/// registered by peer id on chain, a verifier (on-chain peer id registry) or +/// an operator (staked master key bound to the peer id by a NodeInfo binding +/// proof), or anyone else. Rate limits and the storage quota are per class. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Display, EnumString)] +pub enum SenderClass { + Committee, + Registered, + Unregistered, +} + +impl SenderClass { + /// Whether the sender's identity is backed by an on-chain registration. + pub const fn is_registered(self) -> bool { + !matches!(self, Self::Unregistered) + } +} + +const VERIFIER_SET_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); +pub(super) const VERIFIER_SET_RETRY_INTERVAL: Duration = Duration::from_secs(30); +pub(super) const COMMITTEE_POSITIVE_TTL: Duration = Duration::from_secs(10 * 60); +pub(super) const COMMITTEE_NEGATIVE_TTL: Duration = Duration::from_secs(2 * 60); +/// Maximum stale age for retaining a registered classification during refresh. +pub(super) const COMMITTEE_STALE_LIMIT: Duration = Duration::from_secs(30 * 60); +const COMMITTEE_CACHE_MAX_ENTRIES: usize = 4096; +/// Lookup rate for committee candidates (peers announcing themselves as +/// committee by NodeInfo, or stored in the node table); known-member +/// refreshes are exempt. +pub(super) const COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE: u32 = 60; +pub(super) const COMMITTEE_MAX_IN_FLIGHT_LOOKUPS: usize = 16; +/// Maximum unknown-identity lookups in flight. +pub(super) const COMMITTEE_MAX_UNKNOWN_IN_FLIGHT: usize = 12; +pub(super) const REGISTRY_RPC_TIMEOUT: Duration = Duration::from_secs(10); + +/// Separate discovery budget for operator stake lookups. +pub(super) const OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE: u32 = 60; +const OPERATOR_MAX_IN_FLIGHT_LOOKUPS: usize = 16; +const OPERATOR_MAX_UNKNOWN_IN_FLIGHT: usize = 12; +pub(super) const REFUSED_PEER_LOOKUPS_PER_MINUTE: u32 = 30; + +/// A confirmed operator binding keeps its class this long before re-validation; +/// a shorter window applies while the stake is still unknown. +pub(super) const OPERATOR_STAKE_TTL: Duration = Duration::from_secs(10 * 60); +pub(super) const OPERATOR_STAKE_STALE_LIMIT: Duration = Duration::from_secs(30 * 60); +const OPERATOR_CACHE_MAX_ENTRIES: usize = 4096; +/// Maximum persisted operator bindings loaded at startup. +pub const OPERATOR_BINDING_LOAD_LIMIT: i64 = OPERATOR_CACHE_MAX_ENTRIES as i64; + +#[derive(Clone, Copy, Debug)] +struct CommitteeEntry { + registered: bool, + fetched_at: Instant, +} + +impl CommitteeEntry { + fn ttl(&self) -> Duration { + if self.registered { COMMITTEE_POSITIVE_TTL } else { COMMITTEE_NEGATIVE_TTL } + } + + fn needs_refresh(&self, now: Instant) -> bool { + now.saturating_duration_since(self.fetched_at) >= self.ttl() + } + + fn is_registered(&self, now: Instant) -> bool { + self.registered && now.saturating_duration_since(self.fetched_at) < COMMITTEE_STALE_LIMIT + } +} + +/// Canonical operator identity: the x-only master public key. +pub type OperatorKey = [u8; 32]; + +pub fn operator_key(pubkey: &bitcoin::XOnlyPublicKey) -> OperatorKey { + pubkey.serialize() +} + +/// A peer that proved (via a NodeInfo binding) it owns a master key, plus what +/// the chain says about that key's operator stake. +#[derive(Clone, Debug)] +struct OperatorBinding { + pubkey: OperatorKey, + issued_at: i64, + /// `Some(true)` once the key is confirmed to be a sufficiently staked + /// operator on chain; `None` until the lookup completes. + staked: Option, + fetched_at: Option, + /// Configured binding; bypasses discovery budget but still requires a positive stake verdict. + trusted: bool, +} + +impl OperatorBinding { + fn needs_refresh(&self, now: Instant) -> bool { + match self.fetched_at { + None => true, + Some(fetched_at) => now.saturating_duration_since(fetched_at) >= OPERATOR_STAKE_TTL, + } + } + + fn is_registered(&self, now: Instant) -> bool { + self.staked == Some(true) + && self + .fetched_at + .is_some_and(|at| now.saturating_duration_since(at) < OPERATOR_STAKE_STALE_LIMIT) + } +} + +#[derive(Debug, Default)] +struct RegistryState { + verifiers: HashSet>, + verifiers_fetched_at: Option, + verifiers_attempted_at: Option, + verifiers_in_flight: bool, + committee: HashMap, + committee_in_flight: HashSet, + /// Operator bindings keyed by the peer that proved them. + operators: HashMap, + /// Reverse index from master key to its current peer ID. + operator_owner: HashMap, + operators_in_flight: HashSet, + committee_discovery: DiscoveryBudget, + /// Separate lookup budget for connected neighbours. + neighbour_discovery: DiscoveryBudget, + /// For peers the connection gate refused; see [`LookupBudget::Refused`]. + refused_discovery: DiscoveryBudget, + operator_discovery: DiscoveryBudget, +} + +/// Per-minute budget for first-time (unknown-identity) registry lookups. +#[derive(Debug, Default)] +struct DiscoveryBudget { + window_started_at: Option, + lookups_in_window: u32, +} + +impl DiscoveryBudget { + /// Charge one lookup. Returns false when the budget is exhausted. + fn take(&mut self, per_minute: u32, now: Instant) -> bool { + let window_open = self.window_started_at.is_some_and(|started_at| { + now.saturating_duration_since(started_at) < Duration::from_secs(60) + }); + if !window_open { + self.window_started_at = Some(now); + self.lookups_in_window = 0; + } + if self.lookups_in_window >= per_minute { + return false; + } + self.lookups_in_window += 1; + true + } +} + +/// Whose budget a first-time committee lookup is charged to. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LookupBudget { + /// Discovery budget for peers identified as committee candidates by NodeInfo or + /// the node table. Durable messages do not trigger discovery. + Candidate, + /// A peer this node holds a connection with; bounded by the connection + /// ceiling. Local bans depend on these lookups coming back. + Neighbour, + /// A peer whose connection was refused for lack of an open slot. + Refused, +} + +impl RegistryState { + /// Claim the verifier set refresh if it is due. The caller must fetch it and + /// call `record_verifier_set`, or it stays marked as in flight. + fn claim_verifier_set_refresh(&mut self, now: Instant) -> bool { + let due = match (self.verifiers_fetched_at, self.verifiers_attempted_at) { + (_, Some(attempted_at)) + if now.saturating_duration_since(attempted_at) < VERIFIER_SET_RETRY_INTERVAL => + { + false + } + (Some(fetched_at), _) => { + now.saturating_duration_since(fetched_at) >= VERIFIER_SET_REFRESH_INTERVAL + } + (None, _) => true, + }; + if !due || self.verifiers_in_flight { + return false; + } + self.verifiers_in_flight = true; + self.verifiers_attempted_at = Some(now); + true + } + + /// Claim a committee lookup for `peer` if one is due and a slot is free. The + /// caller must resolve it with `record_committee_peer`. + fn claim_committee_lookup( + &mut self, + peer: &PeerId, + budget: LookupBudget, + now: Instant, + ) -> bool { + if self.verifiers.contains(&peer.to_bytes()) || self.committee_in_flight.contains(peer) { + return false; + } + let known_registered = match self.committee.get(peer) { + Some(entry) if !entry.needs_refresh(now) => return false, + Some(entry) => entry.registered, + None => false, + }; + // Known-member refreshes may use every slot without discovery charges. + let in_flight = self.committee_in_flight.len(); + let admitted = if known_registered { + in_flight < COMMITTEE_MAX_IN_FLIGHT_LOOKUPS + } else { + match budget { + LookupBudget::Candidate => { + in_flight < COMMITTEE_MAX_UNKNOWN_IN_FLIGHT + && self + .committee_discovery + .take(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, now) + } + LookupBudget::Neighbour => { + in_flight < COMMITTEE_MAX_IN_FLIGHT_LOOKUPS + && self + .neighbour_discovery + .take(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, now) + } + LookupBudget::Refused => { + in_flight < COMMITTEE_MAX_UNKNOWN_IN_FLIGHT + && self.refused_discovery.take(REFUSED_PEER_LOOKUPS_PER_MINUTE, now) + } + } + }; + if admitted { + self.committee_in_flight.insert(*peer); + } + admitted + } + + /// Claim a due operator lookup; confirmed operators bypass discovery limits. + fn claim_operator_refresh( + &mut self, + peer: &PeerId, + pubkey: &OperatorKey, + now: Instant, + ) -> bool { + if self.operators_in_flight.contains(peer) { + return false; + } + let known_registered = match self.operators.get(peer) { + // Only the current owner of the key is worth validating. + Some(binding) if binding.pubkey != *pubkey => return false, + Some(binding) if !binding.needs_refresh(now) => return false, + Some(binding) => binding.staked == Some(true) || binding.trusted, + None => return false, + }; + let in_flight = self.operators_in_flight.len(); + if known_registered { + if in_flight >= OPERATOR_MAX_IN_FLIGHT_LOOKUPS { + return false; + } + } else if in_flight >= OPERATOR_MAX_UNKNOWN_IN_FLIGHT + || !self.operator_discovery.take(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE, now) + { + return false; + } + self.operators_in_flight.insert(*peer); + true + } +} + +/// Trust window for restored registrations awaiting chain refresh. +const SEEDED_TRUST: Duration = Duration::from_secs(5 * 60); + +/// How far back a seeded entry is dated: due for re-validation right away, and +/// `SEEDED_TRUST` short of the stale limit at which it stops being trusted. +fn seeded_fetched_at(now: Instant, stale_limit: Duration) -> Instant { + now.checked_sub(stale_limit.saturating_sub(SEEDED_TRUST)).unwrap_or(now) +} + +/// Why a configured operator binding was not applied. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TrustedBindingConflict { + /// The peer already holds a binding, signed by another key. + PeerBoundToAnotherKey(OperatorKey), + /// The key is already bound, by its own signature, to another peer id. + KeyBoundToAnotherPeer(PeerId), +} + +/// What the caller should fetch from the chain, decided under the cache lock so +/// concurrent messages from one peer start a single lookup. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RefreshPlan { + pub verifier_set: bool, + pub committee_peer: bool, +} + +/// Cached chain registration; misses are Unregistered and trigger background lookups. +#[derive(Debug, Default)] +pub struct PeerRegistry { + state: Mutex, +} + +impl PeerRegistry { + fn lock(&self) -> std::sync::MutexGuard<'_, RegistryState> { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } + + pub fn is_verifier(&self, peer: &PeerId) -> bool { + self.lock().verifiers.contains(&peer.to_bytes()) + } + + /// Whether the verifier set was ever fetched; before that nobody is + /// turned away for not being in it. + pub fn verifier_set_loaded(&self) -> bool { + self.lock().verifiers_fetched_at.is_some() + } + + pub fn sender_class(&self, peer: &PeerId, now: Instant) -> SenderClass { + let state = self.lock(); + if state.committee.get(peer).is_some_and(|entry| entry.is_registered(now)) { + SenderClass::Committee + } else if state.verifiers.contains(&peer.to_bytes()) + || state.operators.get(peer).is_some_and(|binding| binding.is_registered(now)) + { + SenderClass::Registered + } else { + SenderClass::Unregistered + } + } + + /// Whether `peer` is a committee member, from the cache. For handlers whose + /// authorisation is the committee specifically, not any registered sender. + pub fn is_committee(&self, peer: &PeerId, now: Instant) -> bool { + self.sender_class(peer, now) == SenderClass::Committee + } + + /// Whether fresh chain results identify the peer as unregistered. + pub fn is_known_unregistered(&self, peer: &PeerId, now: Instant) -> bool { + // A stale "no" does not count: the peer may have registered since. + let looked_up = self + .lock() + .committee + .get(peer) + .is_some_and(|entry| !entry.registered && !entry.needs_refresh(now)); + looked_up && !self.sender_class(peer, now).is_registered() + } + + /// Committee peers the cache currently holds as registered, for ack sets. + pub fn committee_peer_ids(&self, now: Instant) -> Vec { + let state = self.lock(); + state + .committee + .iter() + .filter(|(_, entry)| entry.is_registered(now)) + .map(|(peer, _)| *peer) + .collect() + } + + /// The peer currently bound to an operator master key, if known. + pub fn operator_peer_for(&self, pubkey: &OperatorKey) -> Option { + self.lock().operator_owner.get(pubkey).copied() + } + + /// Record a verified binding; newer issuance wins and each key maps to one peer. + /// Returns whether the peer's previous persisted registration must be revoked. + pub fn observe_operator_binding( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + issued_at: i64, + ) -> bool { + let mut state = self.lock(); + let mut released_confirmed_key = false; + // Move the key to the peer with the newer binding. + if let Some(current_owner) = state.operator_owner.get(pubkey).copied() + && current_owner != *peer + { + let newer = state + .operators + .get(¤t_owner) + // Require a strictly newer issuance time for ownership transfer. + .is_none_or(|existing| issued_at > existing.issued_at); + if !newer { + return false; + } + state.operators.remove(¤t_owner); + } + if let Some(previous) = state.operators.get_mut(peer) { + if previous.pubkey == *pubkey { + // Preserve the stake verdict when the peer re-announces the same key. + previous.issued_at = previous.issued_at.max(issued_at); + state.operator_owner.insert(*pubkey, *peer); + return false; + } + // A peer that rebinds to a different key releases its old key's + // ownership, and whatever was confirmed for that key with it. + released_confirmed_key = previous.staked == Some(true); + let previous_pubkey = previous.pubkey; + if state.operator_owner.get(&previous_pubkey) == Some(peer) { + state.operator_owner.remove(&previous_pubkey); + } + } + if state.operators.len() >= OPERATOR_CACHE_MAX_ENTRIES + && !state.operators.contains_key(peer) + { + let now = Instant::now(); + state.operators.retain(|_, binding| binding.is_registered(now) || binding.trusted); + let live: HashSet = state.operators.keys().copied().collect(); + state.operator_owner.retain(|_, owner| live.contains(owner)); + } + state.operator_owner.insert(*pubkey, *peer); + state.operators.insert( + *peer, + OperatorBinding { + pubkey: *pubkey, + issued_at, + staked: None, + fetched_at: None, + trusted: false, + }, + ); + released_confirmed_key + } + + /// Record a configured binding without overriding a signed binding. + /// See `P2P_TRUSTED_OPERATOR_BINDINGS`. + pub fn trust_operator_binding( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + ) -> Result<(), TrustedBindingConflict> { + let mut state = self.lock(); + if let Some(existing) = state.operators.get(peer) + && existing.pubkey != *pubkey + { + return Err(TrustedBindingConflict::PeerBoundToAnotherKey(existing.pubkey)); + } + if let Some(owner) = state.operator_owner.get(pubkey) + && owner != peer + { + return Err(TrustedBindingConflict::KeyBoundToAnotherPeer(*owner)); + } + state.operator_owner.insert(*pubkey, *peer); + state.operators.entry(*peer).and_modify(|binding| binding.trusted = true).or_insert( + OperatorBinding { + pubkey: *pubkey, + issued_at: 0, + staked: None, + fetched_at: None, + trusted: true, + }, + ); + Ok(()) + } + + /// Whether an operator stake lookup for `peer`/`pubkey` should start now. + /// Claimed lookups must be resolved with [`Self::record_operator_stake`]. + pub fn plan_operator_refresh(&self, peer: &PeerId, pubkey: &OperatorKey, now: Instant) -> bool { + self.lock().claim_operator_refresh(peer, pubkey, now) + } + + /// Claim due operator refreshes, confirmed operators first. + pub fn plan_due_operator_refreshes(&self, now: Instant) -> Vec<(PeerId, OperatorKey)> { + let mut state = self.lock(); + let mut due: Vec<(PeerId, OperatorKey, bool)> = state + .operators + .iter() + // Retry a negative stake verdict only after another announcement. + .filter(|(peer, binding)| { + binding.needs_refresh(now) + && (binding.staked != Some(false) || binding.trusted) + && !state.operators_in_flight.contains(*peer) + }) + .map(|(peer, binding)| { + (*peer, binding.pubkey, binding.staked == Some(true) || binding.trusted) + }) + .collect(); + due.sort_by_key(|(_, _, known)| !*known); + due.into_iter() + .filter(|(peer, pubkey, _)| state.claim_operator_refresh(peer, pubkey, now)) + .map(|(peer, pubkey, _)| (peer, pubkey)) + .collect() + } + + /// Record the verdict for the peer's current binding and return the persistence update. + pub fn record_operator_stake( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + staked: Option, + now: Instant, + ) -> Option { + let mut state = self.lock(); + state.operators_in_flight.remove(peer); + let staked = staked?; + // Ignore a result for a binding that was superseded while in flight. + let binding = state.operators.get_mut(peer).filter(|binding| binding.pubkey == *pubkey)?; + binding.staked = Some(staked); + binding.fetched_at = Some(now); + Some(staked) + } + + /// Restore a confirmed operator only when its current binding matches the stored key. + /// Schedule immediate revalidation. + pub fn seed_verified_operator( + &self, + peer: &PeerId, + pubkey: &OperatorKey, + issued_at: i64, + now: Instant, + ) { + self.observe_operator_binding(peer, pubkey, issued_at); + let mut state = self.lock(); + if let Some(binding) = state.operators.get_mut(peer) + && binding.pubkey == *pubkey + && binding.staked.is_none() + { + binding.staked = Some(true); + binding.fetched_at = Some(seeded_fetched_at(now, OPERATOR_STAKE_STALE_LIMIT)); + } + } + + /// Re-instate a committee member confirmed in an earlier session; see + /// [`Self::seed_verified_operator`]. + pub fn seed_verified_committee_peer(&self, peer: &PeerId, now: Instant) { + self.lock().committee.entry(*peer).or_insert(CommitteeEntry { + registered: true, + fetched_at: seeded_fetched_at(now, COMMITTEE_STALE_LIMIT), + }); + } + + /// Claim due committee-member refreshes. + pub fn plan_due_committee_refreshes(&self, now: Instant) -> Vec { + let mut state = self.lock(); + let due: Vec = state + .committee + .iter() + .filter(|(peer, entry)| { + entry.registered + && entry.needs_refresh(now) + && !state.committee_in_flight.contains(*peer) + }) + .map(|(peer, _)| *peer) + .collect(); + let free = COMMITTEE_MAX_IN_FLIGHT_LOOKUPS.saturating_sub(state.committee_in_flight.len()); + let claimed: Vec = due.into_iter().take(free).collect(); + state.committee_in_flight.extend(claimed.iter().copied()); + claimed + } + + /// Claim the lookups that are due. Whatever is claimed must be resolved with + /// the matching `record_*` call, or it stays marked as in flight. + pub fn plan_refresh(&self, peer: Option<&PeerId>, now: Instant) -> RefreshPlan { + let mut state = self.lock(); + RefreshPlan { + verifier_set: state.claim_verifier_set_refresh(now), + committee_peer: peer.is_some_and(|peer| { + state.claim_committee_lookup(peer, LookupBudget::Candidate, now) + }), + } + } + + /// Claim only a candidate's committee lookup, using the discovery budget; + /// the verifier set is left to [`refresh_registry_in_background`](super::registry_refresh::refresh_registry_in_background). + pub fn plan_candidate_lookup(&self, peer: &PeerId, now: Instant) -> bool { + self.lock().claim_committee_lookup(peer, LookupBudget::Candidate, now) + } + + /// Whether the registry has never classified `peer`: no committee verdict, + /// not in the verifier set and no operator binding. + pub fn is_unclassified(&self, peer: &PeerId) -> bool { + let state = self.lock(); + !state.committee.contains_key(peer) + && !state.verifiers.contains(&peer.to_bytes()) + && !state.operators.contains_key(peer) + } + + /// Claim only the neighbour's committee lookup, using the neighbour budget. + pub fn plan_neighbour_lookup(&self, peer: &PeerId, now: Instant) -> bool { + self.lock().claim_committee_lookup(peer, LookupBudget::Neighbour, now) + } + + /// Claim refused-peer lookups using their separate budget. + pub fn plan_refused_peer_lookup(&self, peer: &PeerId, now: Instant) -> bool { + self.lock().claim_committee_lookup(peer, LookupBudget::Refused, now) + } + + pub fn record_verifier_set(&self, verifiers: Option>>, now: Instant) { + let mut state = self.lock(); + state.verifiers_in_flight = false; + // Retain the previous verifier set on fetch failure. + if let Some(verifiers) = verifiers { + state.verifiers = verifiers.into_iter().collect(); + state.verifiers_fetched_at = Some(now); + } + } + + /// Record a committee lookup. Returns the definite verdict, for the caller + /// to persist; see [`Self::record_operator_stake`]. + pub fn record_committee_peer( + &self, + peer: &PeerId, + registered: Option, + now: Instant, + ) -> Option { + let mut state = self.lock(); + state.committee_in_flight.remove(peer); + let registered = registered?; + let changed = Some(registered); + if !state.committee.contains_key(peer) + && state.committee.len() >= COMMITTEE_CACHE_MAX_ENTRIES + { + state.committee.retain(|_, entry| !entry.needs_refresh(now)); + if state.committee.len() >= COMMITTEE_CACHE_MAX_ENTRIES { + if !registered { + return changed; + } + let evict = state + .committee + .iter() + .find(|(_, entry)| !entry.registered) + .map(|(peer, _)| *peer); + if let Some(evict) = evict { + state.committee.remove(&evict); + } + } + } + state.committee.insert(*peer, CommitteeEntry { registered, fetched_at: now }); + changed + } +} + +static PEER_REGISTRY: LazyLock = LazyLock::new(PeerRegistry::default); + +pub fn peer_registry() -> &'static PeerRegistry { + &PEER_REGISTRY +} diff --git a/node/src/p2p_admission/registry_refresh.rs b/node/src/p2p_admission/registry_refresh.rs new file mode 100644 index 000000000..c2f125e1d --- /dev/null +++ b/node/src/p2p_admission/registry_refresh.rs @@ -0,0 +1,323 @@ +//! Background lookups that fill and refresh the peer registry from Goat and +//! the node table. + +use anyhow::Result; +use client::goat_chain::GOATClient; +use libp2p::PeerId; +use std::collections::HashMap; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use store::localdb::LocalDB; + +use super::registry::{OperatorKey, PeerRegistry, REGISTRY_RPC_TIMEOUT, peer_registry}; +use super::strikes::take_refused_peers; + +/// Persisted registration kinds. +pub(super) const REGISTERED_KIND_COMMITTEE: &str = "Committee"; +pub(super) const REGISTERED_KIND_OPERATOR: &str = "Operator"; + +/// Persist a change in a peer's confirmed registration. Best effort: the cache +/// stays authoritative for this session, the table only warms the next one. +async fn persist_registration_change( + local_db: &LocalDB, + peer: &PeerId, + kind: &str, + pubkey: Option<&OperatorKey>, + change: Option, +) { + let Some(registered) = change else { + return; + }; + let peer_id = peer.to_string(); + let pubkey = pubkey.map(hex::encode).unwrap_or_default(); + let result = async { + let mut storage = local_db.acquire().await?; + if registered { + storage.upsert_p2p_registered_peer(&peer_id, kind, &pubkey).await + } else { + storage.delete_p2p_registered_peer(&peer_id, kind).await + } + } + .await; + if let Err(error) = result { + tracing::debug!( + event = "p2p_admission", + outcome = "registration_persist_failed", + peer_id = %peer_id, + kind, + error = %error, + "failed to persist a confirmed registration; it will be rediscovered after a restart" + ); + } +} + +/// Revoke persisted operator registration after a key change. +pub async fn revoke_persisted_operator(local_db: &LocalDB, peer: &PeerId) { + persist_registration_change(local_db, peer, REGISTERED_KIND_OPERATOR, None, Some(false)).await; +} + +/// Restore confirmed peers; operator bindings must be signature-checked and match stored keys. +pub async fn seed_registry_from_store( + local_db: &LocalDB, + registry: &PeerRegistry, + operator_bindings: &HashMap, + now: Instant, +) -> Result { + let persisted = local_db.acquire().await?.load_p2p_registered_peers().await?; + let mut seeded = 0; + for (peer_id, kind, confirmed_pubkey) in persisted { + let Ok(peer) = PeerId::from_str(&peer_id) else { + continue; + }; + match kind.as_str() { + REGISTERED_KIND_COMMITTEE => { + registry.seed_verified_committee_peer(&peer, now); + seeded += 1; + } + REGISTERED_KIND_OPERATOR => { + if let Some((pubkey, issued_at)) = operator_bindings.get(&peer) + && hex::encode(pubkey) == confirmed_pubkey + { + registry.seed_verified_operator(&peer, pubkey, *issued_at, now); + seeded += 1; + } + } + _ => {} + } + } + Ok(seeded) +} + +fn spawn_committee_lookup(local_db: &LocalDB, goat_client: &Arc, peer: PeerId) { + let registry = peer_registry(); + let (local_db, goat_client) = (local_db.clone(), goat_client.clone()); + tokio::spawn(async move { + let registered = match tokio::time::timeout( + REGISTRY_RPC_TIMEOUT, + goat_client.committee_mana_is_validate_peer_id(&peer.to_bytes()), + ) + .await + { + Ok(Ok(registered)) => Some(registered), + Ok(Err(error)) => { + tracing::debug!( + event = "p2p_admission", + outcome = "committee_lookup_failed", + peer_id = %peer, + error = %error, + "failed to look up a peer in the committee registry" + ); + None + } + Err(_) => None, + }; + let change = registry.record_committee_peer(&peer, registered, Instant::now()); + persist_registration_change(&local_db, &peer, REGISTERED_KIND_COMMITTEE, None, change) + .await; + }); +} + +fn spawn_operator_stake_lookup( + local_db: &LocalDB, + goat_client: &Arc, + peer: PeerId, + operator_key: OperatorKey, +) { + use crate::utils::OperatorStakeStatus; + + let registry = peer_registry(); + // The stake is registered under the x-only key; the parity chosen here is + // dropped again by the lookup. + let Ok(xonly) = bitcoin::XOnlyPublicKey::from_slice(&operator_key) else { + registry.record_operator_stake(&peer, &operator_key, Some(false), Instant::now()); + return; + }; + let pubkey = bitcoin::PublicKey::new(xonly.public_key(bitcoin::secp256k1::Parity::Even)); + let (local_db, goat_client) = (local_db.clone(), goat_client.clone()); + tokio::spawn(async move { + let staked = match tokio::time::timeout( + REGISTRY_RPC_TIMEOUT, + crate::utils::operator_stake_status(&goat_client, &pubkey), + ) + .await + { + Ok(Ok(OperatorStakeStatus::Staked)) => Some(true), + Ok(Ok( + OperatorStakeStatus::NotRegistered | OperatorStakeStatus::Insufficient { .. }, + )) => Some(false), + // Preserve the cached stake verdict on RPC failure. + Ok(Err(error)) => { + tracing::debug!( + event = "p2p_admission", + outcome = "operator_stake_lookup_failed", + peer_id = %peer, + error = %error, + "failed to confirm operator stake; leaving the binding unverified" + ); + None + } + Err(_) => None, + }; + let change = registry.record_operator_stake(&peer, &operator_key, staked, Instant::now()); + persist_registration_change( + &local_db, + &peer, + REGISTERED_KIND_OPERATOR, + Some(&operator_key), + change, + ) + .await; + }); +} + +/// Look up a peer this node is connected to; see +/// [`PeerRegistry::plan_neighbour_lookup`]. +pub fn refresh_neighbour_in_background( + local_db: &LocalDB, + goat_client: &Arc, + peer: PeerId, +) { + if peer_registry().plan_neighbour_lookup(&peer, Instant::now()) { + spawn_committee_lookup(local_db, goat_client, peer); + } +} + +/// Schedule due verifier-set and candidate committee lookups without waiting. +/// Durable messages do not trigger these lookups. +pub fn refresh_registry_in_background( + local_db: &LocalDB, + goat_client: &Arc, + peer: Option, +) { + let registry = peer_registry(); + let plan = registry.plan_refresh(peer.as_ref(), Instant::now()); + if plan.verifier_set { + let goat_client = goat_client.clone(); + tokio::spawn(async move { + let fetched = match tokio::time::timeout( + REGISTRY_RPC_TIMEOUT, + goat_client.committee_mana_get_verifiers(), + ) + .await + { + Ok(Ok(verifiers)) => Some(verifiers), + Ok(Err(error)) => { + tracing::warn!( + event = "p2p_admission", + outcome = "verifier_set_refresh_failed", + error = %error, + "failed to refresh the registered verifier set; keeping the previous one" + ); + None + } + Err(_) => { + tracing::warn!( + event = "p2p_admission", + outcome = "verifier_set_refresh_timeout", + "timed out refreshing the registered verifier set; keeping the previous one" + ); + None + } + }; + registry.record_verifier_set(fetched, Instant::now()); + }); + } + if let Some(peer) = peer.filter(|_| plan.committee_peer) { + spawn_committee_lookup(local_db, goat_client, peer); + } +} + +/// Schedule refreshes for known registrations that are due. +pub fn refresh_due_registrations_in_background(local_db: &LocalDB, goat_client: &Arc) { + let registry = peer_registry(); + let now = Instant::now(); + refresh_registry_in_background(local_db, goat_client, None); + for peer in registry.plan_due_committee_refreshes(now) { + spawn_committee_lookup(local_db, goat_client, peer); + } + for peer in take_refused_peers() { + if registry.plan_refused_peer_lookup(&peer, now) { + spawn_committee_lookup(local_db, goat_client, peer); + } + } + for (peer, operator_key) in registry.plan_due_operator_refreshes(now) { + spawn_operator_stake_lookup(local_db, goat_client, peer, operator_key); + } +} + +/// Committee candidates the node table holds (self-declared by NodeInfo) +/// that the registry does not know as registered: looked up within the +/// candidate budget, so a member whose NodeInfo arrived while the budget was +/// spent, or before a restart, is classified by the maintenance tick instead +/// of by its first message. +const COMMITTEE_CANDIDATES_PER_TICK: u32 = 64; +const COMMITTEE_CANDIDATE_SCAN_INTERVAL: Duration = Duration::from_secs(60); +static COMMITTEE_CANDIDATES_SCANNED_AT: Mutex> = Mutex::new(None); + +fn committee_candidate_scan_due(now: Instant) -> bool { + let mut scanned_at = + COMMITTEE_CANDIDATES_SCANNED_AT.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if scanned_at + .is_some_and(|at| now.saturating_duration_since(at) < COMMITTEE_CANDIDATE_SCAN_INTERVAL) + { + return false; + } + *scanned_at = Some(now); + true +} + +/// Look up the self-declared committee peers the registry has never +/// classified. Peers with a verdict, registered or not, are refreshed by their +/// own messages; the scan only covers announcements that arrived while the +/// discovery budget was spent or before a restart. +pub async fn revalidate_committee_candidates(local_db: &LocalDB, goat_client: &Arc) { + let registry = peer_registry(); + let now = Instant::now(); + if !committee_candidate_scan_due(now) { + return; + } + let candidates = match local_db.acquire().await { + Ok(mut storage) => { + storage + .find_nodes( + &store::localdb::NodeQuery::default() + .with_actor(bitvm_lib::actors::Actor::Committee.to_string()) + .with_order("updated_at DESC".to_string()) + .with_limit(COMMITTEE_CANDIDATES_PER_TICK), + ) + .await + } + Err(error) => Err(error), + }; + let candidates = match candidates { + Ok((nodes, _)) => nodes, + Err(error) => { + tracing::debug!(event = "p2p_admission", outcome = "candidate_scan_failed", error = %error, "could not read committee candidates"); + return; + } + }; + for node in candidates { + let Ok(peer) = PeerId::from_str(&node.peer_id) else { continue }; + if !registry.is_unclassified(&peer) { + continue; + } + if registry.plan_candidate_lookup(&peer, now) { + spawn_committee_lookup(local_db, goat_client, peer); + } + } +} + +/// Confirm, off the event loop, that a proven operator binding belongs to a +/// sufficiently staked operator, and cache the verdict for classification. +pub fn refresh_operator_binding_in_background( + local_db: &LocalDB, + goat_client: &Arc, + peer: PeerId, + operator_key: OperatorKey, +) { + if !peer_registry().plan_operator_refresh(&peer, &operator_key, Instant::now()) { + return; + } + spawn_operator_stake_lookup(local_db, goat_client, peer, operator_key); +} diff --git a/node/src/p2p_admission/replay.rs b/node/src/p2p_admission/replay.rs new file mode 100644 index 000000000..b08d34b4c --- /dev/null +++ b/node/src/p2p_admission/replay.rs @@ -0,0 +1,125 @@ +//! Replay protection: one sequence window per author, with a lockout for +//! authors that keep sending stale or far-future messages. + +use anyhow::Result; +use libp2p::PeerId; +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex}; +use std::time::Duration; + +use super::DropReason; + +/// Number of sequence positions retained below the high-water mark. +pub(super) const REPLAY_WINDOW: u64 = 1_024; +const REPLAY_MAX_TRACKED_AUTHORS: usize = 16_384; + +/// Maximum accepted sequence-number lead over local wall time. +pub(super) const REPLAY_MAX_FUTURE: Duration = Duration::from_secs(10 * 60); +/// Below-floor rejections of one author per tick that are worth an operator's +/// attention; see [`ReplayGuard::take_lockout_suspects`]. +pub(super) const REPLAY_LOCKOUT_SUSPECT_REJECTIONS: u32 = 8; + +#[derive(Debug, Default)] +struct AuthorSequence { + highest: u64, + /// Sequence numbers seen within `REPLAY_WINDOW` of `highest`. + recent: std::collections::BTreeSet, + /// Rejection count since the last report; diagnostic only. + rejected: u32, +} + +/// Per-registered-author sequence window, in memory only: accept unseen numbers +/// within `REPLAY_WINDOW` of the highest one admitted. A restart re-opens every +/// window; what a replay can then re-inject is bounded by the engine inbox's +/// content-id dedup and, for a registered author, charged to its own rate budget. +#[derive(Debug, Default)] +pub struct ReplayGuard { + authors: Mutex>, + /// How far each registered author's numbering ran ahead of the local clock + /// since the last report; see [`Self::take_clock_leads`]. + clock_leads: Mutex>, +} + +/// Authors remembered per tick for the clock report. It is a diagnostic, and the +/// table must stay small whatever arrives. +const CLOCK_LEADS_MAX_AUTHORS: usize = 64; + +impl ReplayGuard { + /// Record `sequence_number` for `author`, or refuse it. `unix_nanos` is the + /// local wall clock. + pub fn admit( + &self, + author: &PeerId, + sequence_number: Option, + unix_nanos: u64, + ) -> Result<(), DropReason> { + // Require a signed sequence number. + let sequence_number = sequence_number.ok_or(DropReason::ReplayedSequence)?; + if sequence_number > unix_nanos { + self.note_clock_lead(author, sequence_number - unix_nanos); + } + if sequence_number > unix_nanos.saturating_add(REPLAY_MAX_FUTURE.as_nanos() as u64) { + return Err(DropReason::FutureSequence); + } + let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if !authors.contains_key(author) && authors.len() >= REPLAY_MAX_TRACKED_AUTHORS { + // Only reachable if the registries outgrow the table; forgetting an + // author merely re-opens its window once. + if let Some(evicted) = authors.keys().next().copied() { + authors.remove(&evicted); + } + } + let state = authors.entry(*author).or_default(); + if sequence_number > state.highest { + state.highest = sequence_number; + let floor = sequence_number.saturating_sub(REPLAY_WINDOW); + state.recent = state.recent.split_off(&floor); + } else if state.highest - sequence_number >= REPLAY_WINDOW + || state.recent.contains(&sequence_number) + { + state.rejected = state.rejected.saturating_add(1); + return Err(DropReason::ReplayedSequence); + } + state.recent.insert(sequence_number); + Ok(()) + } + + fn note_clock_lead(&self, author: &PeerId, lead_nanos: u64) { + let mut leads = self.clock_leads.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if leads.len() < CLOCK_LEADS_MAX_AUTHORS || leads.contains_key(author) { + let lead = leads.entry(*author).or_default(); + *lead = (*lead).max(lead_nanos); + } + } + + /// Return and clear each registered author's maximum clock lead since the previous call. + pub fn take_clock_leads(&self) -> Vec<(PeerId, Duration)> { + let mut leads = self.clock_leads.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + leads.drain().map(|(author, lead)| (author, Duration::from_nanos(lead))).collect() + } + + /// Return authors exceeding the rejection reporting threshold. + pub fn take_lockout_suspects(&self) -> Vec<(PeerId, u32)> { + let mut authors = self.authors.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + authors + .iter_mut() + .filter_map(|(author, state)| { + let rejected = std::mem::take(&mut state.rejected); + (rejected >= REPLAY_LOCKOUT_SUSPECT_REJECTIONS).then_some((*author, rejected)) + }) + .collect() + } +} + +/// The local wall clock in the unit gossipsub numbers its messages in. +pub fn unix_nanos_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos().min(u128::from(u64::MAX)) as u64) +} + +pub fn replay_guard() -> &'static ReplayGuard { + &REPLAY_GUARD +} + +static REPLAY_GUARD: LazyLock = LazyLock::new(ReplayGuard::default); diff --git a/node/src/p2p_admission/strikes.rs b/node/src/p2p_admission/strikes.rs new file mode 100644 index 000000000..4b4a35824 --- /dev/null +++ b/node/src/p2p_admission/strikes.rs @@ -0,0 +1,98 @@ +//! Strikes against peers that keep sending refused direct messages, and the +//! peers refused for an unknown identity. + +use libp2p::PeerId; +use std::collections::HashMap; +use std::sync::{LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +pub(super) const STRIKE_WINDOW: Duration = Duration::from_secs(60); +pub(super) const STRIKE_LIMIT: u32 = 256; +pub(super) const STRIKE_BAN: Duration = Duration::from_secs(10 * 60); +const STRIKE_MAX_TRACKED_PEERS: usize = 4_096; + +#[derive(Debug, Default)] +struct StrikeState { + /// Window start and strikes in it, per direct peer. + strikes: HashMap, + banned_until: HashMap, +} + +/// Temporary local bans based on attributable relay strikes; registered peers are exempt. +#[derive(Debug, Default)] +pub struct DirectPeerStrikes { + state: Mutex, +} + +impl DirectPeerStrikes { + /// Count one strike against `peer`. Returns `true` when it just reached the + /// limit and should be banned now. + pub fn strike(&self, peer: &PeerId, now: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.banned_until.contains_key(peer) { + return false; + } + if !state.strikes.contains_key(peer) && state.strikes.len() >= STRIKE_MAX_TRACKED_PEERS { + state.strikes.retain(|_, (started_at, _)| { + now.saturating_duration_since(*started_at) < STRIKE_WINDOW + }); + if state.strikes.len() >= STRIKE_MAX_TRACKED_PEERS { + return false; + } + } + let (started_at, count) = state.strikes.entry(*peer).or_insert((now, 0)); + if now.saturating_duration_since(*started_at) >= STRIKE_WINDOW { + (*started_at, *count) = (now, 0); + } + *count += 1; + if *count < STRIKE_LIMIT { + return false; + } + state.strikes.remove(peer); + state.banned_until.insert(*peer, now + STRIKE_BAN); + true + } + + /// Check the peer's current ban for each connection. + pub fn is_banned(&self, peer: &PeerId, now: Instant) -> bool { + let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.banned_until.get(peer).is_some_and(|until| now < *until) + } + + /// Peers whose ban has run out, removed from the ban list. + pub fn take_expired_bans(&self, now: Instant) -> Vec { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let expired: Vec = state + .banned_until + .iter() + .filter(|(_, until)| now >= **until) + .map(|(peer, _)| *peer) + .collect(); + for peer in &expired { + state.banned_until.remove(peer); + } + expired + } +} + +static DIRECT_PEER_STRIKES: LazyLock = LazyLock::new(DirectPeerStrikes::default); + +pub fn direct_peer_strikes() -> &'static DirectPeerStrikes { + &DIRECT_PEER_STRIKES +} + +const REFUSED_PEERS_MAX: usize = 256; + +static REFUSED_PEERS: Mutex> = Mutex::new(Vec::new()); + +/// Queue refused peers for committee lookup on the next maintenance pass. +pub fn note_refused_peer(peer: PeerId) { + let mut refused = REFUSED_PEERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if refused.len() < REFUSED_PEERS_MAX && !refused.contains(&peer) { + refused.push(peer); + } +} + +pub(super) fn take_refused_peers() -> Vec { + std::mem::take(&mut *REFUSED_PEERS.lock().unwrap_or_else(std::sync::PoisonError::into_inner)) +} diff --git a/node/src/p2p_admission/tests.rs b/node/src/p2p_admission/tests.rs new file mode 100644 index 000000000..aa6c83558 --- /dev/null +++ b/node/src/p2p_admission/tests.rs @@ -0,0 +1,1482 @@ +use super::rate::RATE_BUCKET_IDLE_FORGET; +use super::registry::{ + COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE, COMMITTEE_MAX_IN_FLIGHT_LOOKUPS, + COMMITTEE_MAX_UNKNOWN_IN_FLIGHT, COMMITTEE_NEGATIVE_TTL, COMMITTEE_POSITIVE_TTL, + COMMITTEE_STALE_LIMIT, OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE, OPERATOR_STAKE_STALE_LIMIT, + OPERATOR_STAKE_TTL, REFUSED_PEER_LOOKUPS_PER_MINUTE, VERIFIER_SET_RETRY_INTERVAL, +}; +use super::registry_refresh::{REGISTERED_KIND_COMMITTEE, REGISTERED_KIND_OPERATOR}; +use super::replay::{REPLAY_LOCKOUT_SUSPECT_REJECTIONS, REPLAY_MAX_FUTURE, REPLAY_WINDOW}; +use super::strikes::{STRIKE_BAN, STRIKE_LIMIT, STRIKE_WINDOW}; +use super::*; +use crate::messages::{AggNonceConsensus, InitGraph, sign_init_graph}; +use bitvm_lib::actors::Actor; +use std::collections::HashSet; +use uuid::Uuid; + +const SMALL: InboundLimits = InboundLimits { + max_json_bytes: 1024, + max_binary_bytes: 4096, + global: Quota { rows: 6, bytes: 6000 }, + committee_reserve: Quota { rows: 0, bytes: 0 }, + registered_peer: Quota { rows: 3, bytes: 5000 }, +}; + +fn usage(class: SenderClass, rows: i64, bytes: i64, peer: (i64, i64)) -> InboxClassUsage { + InboxClassUsage { + admission_class: class.to_string(), + rows, + bytes, + peer_rows: peer.0, + peer_bytes: peer.1, + } +} + +/// Use isolated gates in tests. +struct TestGates { + registry: PeerRegistry, + replay_guard: ReplayGuard, + rate_limiter: InboundRateLimiter, + immediate_limiter: PeerRateLimiter, + usage_cache: InboxUsageCache, + local_actor: Actor, +} + +/// A wall clock comfortably ahead of every sequence number the tests use. +const WALL_CLOCK: u64 = 1_800_000_000 * 1_000_000_000; + +/// A distinct operator key per `index`. The registry never parses it. +fn key(index: u32) -> OperatorKey { + let mut key = [0x11; 32]; + key[..4].copy_from_slice(&index.to_be_bytes()); + key +} + +impl TestGates { + fn new(immediate_limiter: PeerRateLimiter) -> Self { + Self { + local_actor: Actor::Committee, + registry: PeerRegistry::default(), + replay_guard: ReplayGuard::default(), + rate_limiter: InboundRateLimiter::new(RateLimits::defaults()), + immediate_limiter, + usage_cache: InboxUsageCache::default(), + } + } + + /// Evaluate `data` as authored and relayed by `source`. + async fn evaluate( + &self, + local_db: &LocalDB, + limits: &InboundLimits, + source: &PeerId, + sequence_number: u64, + data: &[u8], + now: Instant, + ) -> InboundVerdict { + let gates = AdmissionGates { + registry: &self.registry, + replay_guard: &self.replay_guard, + rate_limiter: &self.rate_limiter, + immediate_limiter: &self.immediate_limiter, + usage_cache: &self.usage_cache, + limits, + local_actor: &self.local_actor, + }; + let inbound = InboundGossip { + direct_peer: source, + source, + sequence_number: Some(sequence_number), + data, + }; + evaluate_inbound_message(local_db, &gates, &inbound, now).await.unwrap() + } +} + +fn test_keypair(seed: u8) -> bitcoin::secp256k1::Keypair { + bitcoin::secp256k1::Keypair::from_secret_key( + &bitcoin::secp256k1::Secp256k1::new(), + &bitcoin::secp256k1::SecretKey::from_slice(&[seed; 32]).unwrap(), + ) +} + +/// A committee round value: a durable message a committee node stores. +fn committee_vote() -> GOATMessage { + let keypair = test_keypair(7); + let digest = bitcoin::secp256k1::Message::from_digest([3; 32]); + GOATMessage::new( + Actor::Committee, + GOATMessageContent::AggNonceConsensus(AggNonceConsensus { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + committee_pubkey: bitcoin::PublicKey::new(keypair.public_key()), + consensus_hash: [3; 32], + signature: bitcoin::secp256k1::Secp256k1::new().sign_schnorr(&digest, &keypair), + }), + ) +} + +/// A durable message a committee node stores (the tests run as Committee). +async fn durable_bytes() -> Vec { + committee_vote().serialize_message().await.unwrap() +} + +/// An operator's recruitment: kept by verifiers, relayed by everyone else. +fn init_graph() -> GOATMessage { + let keypair = test_keypair(5); + let init: InitGraph = + sign_init_graph(Uuid::new_v4(), Uuid::new_v4(), &keypair, PeerId::random().to_bytes()); + GOATMessage::new(Actor::Verifier, GOATMessageContent::InitGraph(init)) +} + +#[test] +fn envelope_limits_depend_on_the_encoding_and_the_sender() { + let json = vec![b'{'; 1025]; + assert_eq!(check_envelope(&json, true, &SMALL), Err(DropReason::OversizedJson)); + assert_eq!(check_envelope(&json[..1024], false, &SMALL), Ok(())); + + let mut binary = b"GOATBIN1".to_vec(); + binary.resize(2048, 0); + assert_eq!(check_envelope(&binary, false, &SMALL), Err(DropReason::BinaryFromNonVerifier)); + assert_eq!(check_envelope(&binary, true, &SMALL), Ok(()), "above the JSON ceiling"); + binary.resize(4097, 0); + assert_eq!(check_envelope(&binary, true, &SMALL), Err(DropReason::OversizedBinary)); +} + +/// Verify GenCircuits fits the binary transport ceiling. +#[tokio::test] +async fn gen_circuits_fits_the_binary_envelope() { + use crate::messages::GenCircuits; + use bitvm_lib::babe_adapter::{BABE_N_CC, build_setup_package}; + + let secp = bitcoin::secp256k1::Secp256k1::new(); + let secret = bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(); + let message = GOATMessage::new( + Actor::Operator, + GOATMessageContent::GenCircuits(GenCircuits { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + verifier_pubkey: bitcoin::PublicKey::new(secret.public_key(&secp)), + setup_package: build_setup_package(BABE_N_CC).unwrap(), + }), + ); + let bytes = message.serialize_message().await.unwrap(); + assert!(GOATMessage::is_binary_envelope(&bytes)); + let limits = InboundLimits::with_inbox_bytes(2 << 20, 2 << 30); + assert!(bytes.len() > limits.max_json_bytes, "GenCircuits is {} bytes", bytes.len()); + assert!( + bytes.len() * 5 <= limits.max_binary_bytes * 4, + "GenCircuits is {} bytes, within 20% of the {} byte ceiling", + bytes.len(), + limits.max_binary_bytes + ); + assert_eq!(check_envelope(&bytes, true, &limits), Ok(())); + assert!( + (bytes.len() as i64) * 8 <= limits.registered_peer.bytes, + "a verifier must be able to have several setups queued" + ); +} + +/// Verify graph-bearing JSON fits the configured verifier-slot capacity. +#[test] +fn default_json_ceiling_fits_graphs_with_several_verifier_slots() { + use bitvm_lib::types::BitvmGcCircuitData; + use goat::assert_scripts::WireHash; + + let secp = bitcoin::secp256k1::Secp256k1::new(); + let secret = bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(); + // 0xff is the widest byte in JSON's decimal array encoding. + let slot = BitvmGcCircuitData { + verifier_pubkey: bitcoin::PublicKey::new(secret.public_key(&secp)), + final_msg_hashlocks: vec![[0xff; 20]; bitvm_lib::babe_adapter::BABE_M_CC], + wire_hashes: std::array::from_fn(|_| WireHash { + true_label_hash: [0xff; 20], + false_label_hash: [0xff; 20], + }), + }; + let slot_len = serde_json::to_vec(&slot).unwrap().len(); + assert!( + slot_len * 8 <= crate::env::DEFAULT_P2P_MAX_JSON_MESSAGE_BYTES, + "one GC slot is {slot_len} bytes of JSON" + ); +} + +#[test] +fn quota_is_charged_per_sender_and_globally() { + use SenderClass::{Committee, Registered}; + + assert_eq!(check_inbox_quota(Registered, 100, &[], &SMALL), Ok(())); + + // The sender's own rows, in whichever class they were admitted (a + // committee lookup may complete between two of its messages). + let own = [usage(Registered, 2, 200, (2, 200)), usage(Committee, 1, 100, (1, 100))]; + assert_eq!(check_inbox_quota(Committee, 100, &own, &SMALL), Err(DropReason::SenderQuota)); + let other = [usage(Registered, 3, 300, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &other, &SMALL), Ok(())); + let own_bytes = [usage(Registered, 1, 4950, (1, 4950))]; + assert_eq!( + check_inbox_quota(Registered, 100, &own_bytes, &SMALL), + Err(DropReason::SenderQuota) + ); + + // The global ceiling counts every class. + let full = [usage(Committee, 3, 300, (0, 0)), usage(Registered, 3, 300, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &full, &SMALL), Err(DropReason::GlobalQuota)); + let heavy = [usage(Registered, 1, 5950, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &heavy, &SMALL), Err(DropReason::GlobalQuota)); +} + +#[test] +fn derived_quotas_nest_inside_the_global_ceiling() { + let limits = InboundLimits::with_inbox_bytes(2 << 20, 2 << 30); + assert!(limits.registered_peer.bytes < limits.global.bytes); + assert!(limits.committee_reserve.bytes < limits.global.bytes); + assert!(limits.registered_peer.rows < limits.global.rows); + assert!(limits.committee_reserve.rows < limits.global.rows); + assert!(limits.max_json_bytes as i64 <= limits.registered_peer.bytes); +} + +#[test] +fn registry_classifies_from_cache_and_never_blocks() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (verifier, member, stranger) = (PeerId::random(), PeerId::random(), PeerId::random()); + + // Nothing is known yet: everyone is unregistered, and lookups are due. + assert_eq!(registry.sender_class(&member, now), SenderClass::Unregistered); + assert_eq!( + registry.plan_refresh(Some(&member), now), + RefreshPlan { verifier_set: true, committee_peer: true } + ); + assert_eq!( + registry.plan_refresh(Some(&member), now), + RefreshPlan::default(), + "a lookup already in flight is not started twice" + ); + + registry.record_verifier_set(Some(vec![verifier.to_bytes()]), now); + registry.record_committee_peer(&member, Some(true), now); + assert!(registry.is_verifier(&verifier)); + assert_eq!(registry.sender_class(&verifier, now), SenderClass::Registered); + assert_eq!(registry.sender_class(&member, now), SenderClass::Committee); + assert!(registry.is_committee(&member, now)); + assert!(!registry.is_committee(&verifier, now), "registered, but not the committee"); + assert_eq!(registry.sender_class(&stranger, now), SenderClass::Unregistered); + assert_eq!( + registry.plan_refresh(Some(&verifier), now), + RefreshPlan::default(), + "a verifier needs no committee lookup" + ); + + // Past its TTL the member is re-validated but keeps its class meanwhile. + // The verifier set has aged out by then as well. + let later = now + COMMITTEE_POSITIVE_TTL; + assert_eq!(registry.sender_class(&member, later), SenderClass::Committee); + assert_eq!( + registry.plan_refresh(Some(&member), later), + RefreshPlan { verifier_set: true, committee_peer: true } + ); + registry.record_committee_peer(&member, None, later); + assert_eq!( + registry.sender_class(&member, later), + SenderClass::Committee, + "a failed lookup leaves the cached answer in place" + ); + assert_eq!( + registry.sender_class(&member, now + COMMITTEE_STALE_LIMIT), + SenderClass::Unregistered, + "but it is not trusted forever" + ); + + // A failed verifier refresh keeps the previous set and is retried soon, + // but not on every message. + registry.record_verifier_set(None, later); + assert!(registry.is_verifier(&verifier)); + assert!(!registry.plan_refresh(None, later + Duration::from_secs(1)).verifier_set); + assert!(registry.plan_refresh(None, later + VERIFIER_SET_RETRY_INTERVAL).verifier_set); +} + +#[test] +fn unknown_identities_cannot_starve_revalidation_of_known_members() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + let member = PeerId::random(); + registry.record_committee_peer(&member, Some(true), now); + + let later = now + COMMITTEE_POSITIVE_TTL; + let mut started = 0; + for _ in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let stranger = PeerId::random(); + if registry.plan_refresh(Some(&stranger), later).committee_peer { + started += 1; + registry.record_committee_peer(&stranger, Some(false), later); + } + } + assert_eq!(started, COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE); + assert!( + registry.plan_refresh(Some(&member), later).committee_peer, + "re-validating a registered peer is not charged to the discovery budget" + ); +} + +/// Item 5: even with the unknown-lookup slots saturated and in flight, a +/// known member's re-check must still find a reserved slot. +#[test] +fn known_member_recheck_survives_saturated_unknown_lookups() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + let member = PeerId::random(); + registry.record_committee_peer(&member, Some(true), now); + + // Fill and hold the unknown-lookup slots (do not resolve them). + let mut unknown_in_flight = 0; + for _ in 0..COMMITTEE_MAX_IN_FLIGHT_LOOKUPS * 2 { + if registry.plan_refresh(Some(&PeerId::random()), now).committee_peer { + unknown_in_flight += 1; + } + } + assert_eq!( + unknown_in_flight, COMMITTEE_MAX_UNKNOWN_IN_FLIGHT, + "unknown lookups are capped below the total, leaving slots reserved" + ); + let later = now + COMMITTEE_POSITIVE_TTL; + assert!( + registry.plan_refresh(Some(&member), later).committee_peer, + "a known member re-check uses a reserved slot despite the unknown flood" + ); +} + +/// An operator becomes registered only once its proven binding is confirmed +/// to be a staked operator, and a key maps to one peer id at a time. +#[test] +fn operator_binding_grants_registered_only_after_stake_confirmation() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let peer = PeerId::random(); + let pubkey = &key(1); + + registry.observe_operator_binding(&peer, pubkey, 1); + assert_eq!( + registry.sender_class(&peer, now), + SenderClass::Unregistered, + "a proven binding alone is not enough; stake is unconfirmed" + ); + assert!(registry.plan_operator_refresh(&peer, pubkey, now)); + assert!( + !registry.plan_operator_refresh(&peer, pubkey, now), + "a lookup already in flight is not started twice" + ); + registry.record_operator_stake(&peer, pubkey, Some(true), now); + assert_eq!(registry.sender_class(&peer, now), SenderClass::Registered); + + // Verify one key registers only one peer. + let new_peer = PeerId::random(); + registry.observe_operator_binding(&new_peer, pubkey, 2); + registry.record_operator_stake(&new_peer, pubkey, Some(true), now); + assert_eq!(registry.sender_class(&new_peer, now), SenderClass::Registered); + assert_eq!( + registry.sender_class(&peer, now), + SenderClass::Unregistered, + "the superseded peer id loses the class" + ); + + // An older binding for a key already owned by another peer is ignored. + registry.observe_operator_binding(&peer, pubkey, 1); + assert_eq!(registry.sender_class(&peer, now), SenderClass::Unregistered); +} + +/// The rate budget rejects a flood before decode, charges every tier, and +/// only debits when all tiers pass. +#[test] +fn inbound_rate_budget_bounds_bytes_and_messages_per_tier() { + const MIB: i64 = 1024 * 1024; + let limits = RateLimits { + direct_peer: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + registered_author: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + unregistered_author: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 2 * MIB, + byte_per_sec: 0, + }, + unregistered_total: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 3 * MIB, + byte_per_sec: 0, + }, + unregistered_forward: TierRate { + msg_burst: 100, + msg_per_sec: 0, + byte_burst: 100 * MIB, + byte_per_sec: 0, + }, + global: TierRate { msg_burst: 100, msg_per_sec: 0, byte_burst: 100 * MIB, byte_per_sec: 0 }, + committee_reserve: 0.0, + max_tracked_peers: 16, + }; + let limiter = InboundRateLimiter::new(limits); + let now = Instant::now(); + let relay = PeerId::random(); + use SenderClass::{Registered, Unregistered}; + + // One unregistered author is capped at its own 2 MiB byte burst. + let a = PeerId::random(); + assert!(limiter.charge(&relay, &a, Unregistered, MIB, now).is_ok()); + assert_eq!( + limiter.charge(&relay, &a, Unregistered, 2 * MIB, now), + Err(DropReason::AuthorRate), + "the author's own bucket has only 1 MiB left" + ); + + // Verify distinct unregistered authors share the class byte budget. + let (b, c, d) = (PeerId::random(), PeerId::random(), PeerId::random()); + assert!(limiter.charge(&relay, &b, Unregistered, MIB, now).is_ok()); + assert!(limiter.charge(&relay, &c, Unregistered, MIB, now).is_ok()); + assert_eq!( + limiter.charge(&relay, &d, Unregistered, MIB, now), + Err(DropReason::UnregisteredRate) + ); + + // A registered author is not charged against the unregistered pool, and + // the messages rejected above did not drain the global tier: a fresh + // registered author is still admitted up to the global burst. + let op = PeerId::random(); + assert!(limiter.charge(&relay, &op, Registered, 50 * MIB, now).is_ok()); +} + +#[test] +fn rate_limiter_refills_and_bounds_its_own_memory() { + let limiter = PeerRateLimiter::new(2.0, 1.0, 2); + let now = Instant::now(); + let (a, b, c) = (PeerId::random(), PeerId::random(), PeerId::random()); + assert!(limiter.allow(&a, now)); + assert!(limiter.allow(&a, now)); + assert!(!limiter.allow(&a, now), "the burst is spent"); + assert!(limiter.allow(&a, now + Duration::from_secs(1)), "one token per second"); + + assert!(limiter.allow(&b, now + Duration::from_secs(1))); + assert!( + !limiter.allow(&c, now + Duration::from_secs(1)), + "no room to track a new peer while the tracked ones are active" + ); + assert!( + limiter.allow(&c, now + Duration::from_secs(60)), + "idle peers are forgotten to make room" + ); +} + +#[test] +fn response_gate_coalesces_requests_inside_the_cooldown() { + let gate = ResponseGate::new(Duration::from_secs(10)); + let now = Instant::now(); + assert!(gate.try_respond(now)); + assert!(!gate.take_pending(now + Duration::from_secs(20)), "nothing was deferred"); + + assert!(!gate.try_respond(now + Duration::from_secs(1))); + assert!(!gate.try_respond(now + Duration::from_secs(2))); + assert!(!gate.take_pending(now + Duration::from_secs(5)), "still cooling down"); + assert!(gate.take_pending(now + Duration::from_secs(10)), "one response for both"); + assert!(!gate.take_pending(now + Duration::from_secs(30))); + assert!(gate.try_respond(now + Duration::from_secs(30))); +} + +#[tokio::test] +async fn verdicts_follow_the_queue_state() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + let limits = InboundLimits { registered_peer: Quota { rows: 1, bytes: 4096 }, ..SMALL }; + + let verdict = gates.evaluate(&local_db, &limits, &source, 1, b"not a message", now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::Undecodable))); + + // An unregistered author's durable message is relayed, never stored. + let first = durable_bytes().await; + let verdict = gates.evaluate(&local_db, &limits, &source, 2, &first, now).await; + assert!(matches!(verdict, InboundVerdict::Forward)); + let key = key(1); + gates.registry.observe_operator_binding(&source, &key, 1); + gates.registry.record_operator_stake(&source, &key, Some(true), now); + let verdict = gates.evaluate(&local_db, &limits, &source, 3, &first, now).await; + let InboundVerdict::Enqueue { class, .. } = verdict else { + panic!("an empty inbox admits the first message of a registered author"); + }; + assert_eq!(class, SenderClass::Registered); + // What admission recorded: a queued network event of this author. + let mut storage = local_db.acquire().await.unwrap(); + storage + .insert_engine_inbox_event( + &store::NewEngineInboxEvent { + event_id: "first".to_string(), + aggregate_kind: "graph".to_string(), + aggregate_id: "g".to_string(), + kind: "AggNonceConsensus".to_string(), + source: "network".to_string(), + author: source.to_string(), + admission_class: class.to_string(), + payload: first.clone(), + ack: None, + }, + 1, + ) + .await + .unwrap(); + drop(storage); + + // Another payload from the same sender is over its row quota, in + // whichever class it is admitted. + let second = durable_bytes().await; + let verdict = gates.evaluate(&local_db, &limits, &source, 4, &second, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::SenderQuota))); + gates.registry.record_committee_peer(&source, Some(true), now); + let verdict = gates.evaluate(&local_db, &limits, &source, 5, &second, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::SenderQuota))); +} + +/// Verify replay neither passes admission nor consumes rate budget. +#[tokio::test] +async fn replayed_registered_messages_are_dropped_before_the_rate_charge() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let member = PeerId::random(); + gates.registry.record_committee_peer(&member, Some(true), now); + + let message = durable_bytes().await; + let verdict = gates.evaluate(&local_db, &SMALL, &member, 100, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + for _ in 0..3 { + let verdict = gates.evaluate(&local_db, &SMALL, &member, 100, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ReplayedSequence))); + } + + // Unregistered authors use rate limits rather than replay tracking: + // the same sequence number relayed twice is not a replay to them. + let stranger = PeerId::random(); + let other = durable_bytes().await; + for _ in 0..2 { + let verdict = gates.evaluate(&local_db, &SMALL, &stranger, 7, &other, now).await; + assert!(matches!(verdict, InboundVerdict::Forward)); + } +} + +#[test] +fn replay_window_tolerates_reordering_but_not_reuse() { + let guard = ReplayGuard::default(); + let author = PeerId::random(); + let base = 1_000_000; + assert_eq!(guard.admit(&author, Some(base), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&author, Some(base + 5), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&author, Some(base + 3), WALL_CLOCK), Ok(()), "reordered, but new"); + assert_eq!(guard.admit(&author, Some(base + 3), WALL_CLOCK), Err(DropReason::ReplayedSequence)); + assert_eq!(guard.admit(&author, Some(base + 5), WALL_CLOCK), Err(DropReason::ReplayedSequence)); + assert_eq!(guard.admit(&author, None, WALL_CLOCK), Err(DropReason::ReplayedSequence)); + + // The author restarts: numbering jumps to the new start-up time, and + // everything from the previous session falls out of the window. + let restarted = base + 10 * REPLAY_WINDOW; + assert_eq!(guard.admit(&author, Some(restarted), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&author, Some(base + 4), WALL_CLOCK), Err(DropReason::ReplayedSequence)); + assert_eq!(guard.admit(&author, Some(restarted - REPLAY_WINDOW + 1), WALL_CLOCK), Ok(())); + assert_eq!( + guard.admit(&author, Some(restarted - REPLAY_WINDOW), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + + // Windows are per author. + assert_eq!(guard.admit(&PeerId::random(), Some(base), WALL_CLOCK), Ok(())); +} + +/// Verify re-announcement preserves stake classification. +#[test] +fn reannounced_operator_binding_keeps_its_stake_verdict() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let peer = PeerId::random(); + let pubkey = &key(1); + + registry.observe_operator_binding(&peer, pubkey, 100); + assert!(registry.plan_operator_refresh(&peer, pubkey, now)); + assert_eq!(registry.record_operator_stake(&peer, pubkey, Some(true), now), Some(true)); + assert_eq!(registry.sender_class(&peer, now), SenderClass::Registered); + + registry.observe_operator_binding(&peer, pubkey, 400); + assert_eq!(registry.sender_class(&peer, now), SenderClass::Registered); + assert!( + !registry.plan_operator_refresh(&peer, pubkey, now), + "the verdict is still fresh, so no lookup is due" + ); + + // Keep the newest binding timestamp. + let usurper = PeerId::random(); + registry.observe_operator_binding(&usurper, pubkey, 300); + assert_eq!(registry.sender_class(&peer, now), SenderClass::Registered); + assert!(!registry.plan_operator_refresh(&usurper, pubkey, now)); + + // Every definite verdict is handed back for persisting, changed or not. + let later = now + OPERATOR_STAKE_TTL; + assert!(registry.plan_operator_refresh(&peer, pubkey, later)); + assert_eq!(registry.record_operator_stake(&peer, pubkey, Some(true), later), Some(true)); + assert!(registry.plan_operator_refresh(&peer, pubkey, later + OPERATOR_STAKE_TTL)); + assert_eq!( + registry.record_operator_stake(&peer, pubkey, Some(false), later + OPERATOR_STAKE_TTL), + Some(false) + ); +} + +/// Self-signed operator bindings are free to mint. They may exhaust the +/// operator discovery budget, but not the committee's, and not the +/// re-validation of operators already confirmed. +#[test] +fn operator_binding_flood_spends_only_the_operator_discovery_budget() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + let operator = PeerId::random(); + let real = key(0); + registry.observe_operator_binding(&operator, &real, 1); + assert!(registry.plan_operator_refresh(&operator, &real, now)); + registry.record_operator_stake(&operator, &real, Some(true), now); + + let mut started = 0; + for index in 0..(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let (sybil, sybil_key) = (PeerId::random(), key(index + 1)); + registry.observe_operator_binding(&sybil, &sybil_key, 1); + if registry.plan_operator_refresh(&sybil, &sybil_key, now) { + started += 1; + registry.record_operator_stake(&sybil, &sybil_key, Some(false), now); + } + } + // The real operator's own first lookup took one token of the same budget. + assert_eq!(started + 1, OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE); + + assert!( + registry.plan_refresh(Some(&PeerId::random()), now).committee_peer, + "committee discovery has its own budget" + ); + let due = registry.plan_due_operator_refreshes(now + OPERATOR_STAKE_TTL); + assert!( + due.first().is_some_and(|(peer, _)| *peer == operator), + "the confirmed operator is re-validated first, outside the spent budget" + ); +} + +/// Registrations confirmed in an earlier session are trusted at once and are +/// re-validated as known members, not rediscovered through the budget. +#[test] +fn seeded_registrations_are_trusted_and_revalidated_as_known() { + let registry = PeerRegistry::default(); + // `Instant` cannot be dated before the machine booted; keep clear of it. + let now = Instant::now() + COMMITTEE_STALE_LIMIT; + let (member, operator) = (PeerId::random(), PeerId::random()); + registry.seed_verified_committee_peer(&member, now); + registry.seed_verified_operator(&operator, &key(0), 1, now); + assert_eq!(registry.sender_class(&member, now), SenderClass::Committee); + assert_eq!(registry.sender_class(&operator, now), SenderClass::Registered); + + // Spend both discovery budgets on strangers. + registry.record_verifier_set(Some(vec![]), now); + for index in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let stranger = PeerId::random(); + if registry.plan_refresh(Some(&stranger), now).committee_peer { + registry.record_committee_peer(&stranger, Some(false), now); + } + let sybil_key = key(index + 1); + registry.observe_operator_binding(&stranger, &sybil_key, 1); + if registry.plan_operator_refresh(&stranger, &sybil_key, now) { + registry.record_operator_stake(&stranger, &sybil_key, Some(false), now); + } + } + + assert_eq!(registry.plan_due_committee_refreshes(now), vec![member]); + assert!(registry.plan_due_operator_refreshes(now).iter().any(|(peer, _)| *peer == operator)); + // Revoke restored registrations after a negative chain answer. + assert_eq!(registry.record_committee_peer(&member, Some(false), now), Some(false)); + assert_eq!(registry.sender_class(&member, now), SenderClass::Unregistered); +} + +/// A flood of fresh identities must neither grow the bucket table nor evict +/// the peers it already tracks: past the cap they share one bucket. +#[test] +fn rate_table_is_bounded_and_overflow_identities_share_one_bucket() { + let tier = TierRate { msg_burst: 2, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; + let roomy = TierRate { msg_burst: 1_000, msg_per_sec: 0, byte_burst: 1 << 30, byte_per_sec: 0 }; + let limiter = InboundRateLimiter::new(RateLimits { + direct_peer: roomy, + registered_author: roomy, + unregistered_author: tier, + unregistered_total: roomy, + unregistered_forward: roomy, + global: roomy, + committee_reserve: 0.0, + max_tracked_peers: 4, + }); + let now = Instant::now(); + let relay = PeerId::random(); + use SenderClass::{Registered, Unregistered}; + + for _ in 0..4 { + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); + } + // New authors share overflow capacity when the table is full. + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now).is_ok()); + for _ in 0..64 { + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Unregistered, 1, now), + Err(DropReason::AuthorRate) + ); + } + assert_eq!(limiter.tracked_authors(), 4, "fresh identities do not grow the table"); + + // A registered author is bounded by the chain registry and always tracked. + let member = PeerId::random(); + assert!(limiter.charge(&relay, &member, Registered, 1, now).is_ok()); + assert_eq!(limiter.tracked_authors(), 5); + + // Idle buckets are forgotten, making room for new authors again. + let later = now + RATE_BUCKET_IDLE_FORGET; + assert!(limiter.charge(&relay, &PeerId::random(), Unregistered, 1, later).is_ok()); + assert_eq!(limiter.tracked_authors(), 1); +} + +/// Forgetting an idle bucket must not hand its owner tokens it had spent. +#[test] +fn default_tiers_refill_within_the_idle_forget_window() { + let limits = RateLimits::defaults(); + for tier in [ + limits.direct_peer, + limits.registered_author, + limits.unregistered_author, + limits.unregistered_total, + limits.unregistered_forward, + limits.global, + ] { + let window = RATE_BUCKET_IDLE_FORGET.as_secs() as i64; + assert!(i64::from(tier.msg_burst) <= i64::from(tier.msg_per_sec) * window); + assert!(tier.byte_burst <= tier.byte_per_sec * window); + } +} + +/// Verify admissions immediately update cached class totals. +#[tokio::test] +async fn cached_class_totals_count_admissions_immediately() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let limits = InboundLimits { global: Quota { rows: 2, bytes: 3000 }, ..SMALL }; + let member = |index: u8| { + let peer = PeerId::random(); + gates.registry.record_committee_peer(&peer, Some(true), now); + let _ = index; + peer + }; + + // Nothing is inserted between evaluations: only the cache can know that + // two rows are already on their way into the inbox. + for sequence_number in 0..2 { + let message = durable_bytes().await; + let verdict = + gates.evaluate(&local_db, &limits, &member(0), sequence_number, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + } + let message = durable_bytes().await; + let verdict = gates.evaluate(&local_db, &limits, &member(1), 9, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::GlobalQuota))); + + // Reload totals after cache expiry. + let later = now + INBOX_CLASS_TOTALS_TTL; + let verdict = gates.evaluate(&local_db, &limits, &member(2), 10, &message, later).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); +} + +#[tokio::test] +async fn binary_envelope_is_reserved_for_gen_circuits_from_verifiers() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + + // A small message wrapped in the binary envelope: well-formed, but it + // is not the payload the envelope exists for. + let message = committee_vote(); + let mut wrapped = b"GOATBIN1".to_vec(); + wrapped.extend(bincode::serialize(&message).unwrap()); + + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &wrapped, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::BinaryFromNonVerifier))); + + gates.registry.record_verifier_set(Some(vec![source.to_bytes()]), now); + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &wrapped, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::UnexpectedBinaryKind))); +} + +/// A verifier's endorsement of the graph parameters. +fn verifier_endorsement() -> GOATMessage { + use crate::messages::VerifierGraphParamsEndorsement; + let keypair = test_keypair(9); + let digest = bitcoin::secp256k1::Message::from_digest([5; 32]); + GOATMessage::new( + Actor::Committee, + GOATMessageContent::VerifierGraphParamsEndorsement(VerifierGraphParamsEndorsement { + instance_id: Uuid::new_v4(), + graph_id: Uuid::new_v4(), + verifier_pubkey: bitcoin::PublicKey::new(keypair.public_key()), + verifier_index: 0, + canonical_graph_params_hash: [5; 32], + signature: bitcoin::secp256k1::Secp256k1::new().sign_schnorr(&digest, &keypair), + }), + ) +} + +#[tokio::test] +async fn parameter_endorsements_come_from_the_verifier_set_once_it_is_loaded() { + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let (verifier, stranger) = (PeerId::random(), PeerId::random()); + // The stranger is registered (a committee peer), just not a verifier. + gates.registry.record_committee_peer(&stranger, Some(true), now); + let message = verifier_endorsement().serialize_message().await.unwrap(); + + // Before the set is known nobody is turned away on it. + let verdict = gates.evaluate(&local_db, &SMALL, &stranger, 1, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); + + gates.registry.record_verifier_set(Some(vec![verifier.to_bytes()]), now); + let verdict = gates.evaluate(&local_db, &SMALL, &stranger, 2, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::EndorsementFromNonVerifier))); + let verdict = gates.evaluate(&local_db, &SMALL, &verifier, 1, &message, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); +} + +#[tokio::test] +async fn immediate_messages_are_rate_limited_per_sender() { + use crate::messages::NodeInfo; + + let local_db = store::create_local_db("sqlite::memory:").await; + let gates = TestGates::new(PeerRateLimiter::new(1.0, 0.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + let request = GOATMessage::new( + Actor::All, + GOATMessageContent::RequestNodeInfo(NodeInfo { + peer_id: source.to_string(), + actor: "Operator".to_string(), + goat_addr: String::new(), + btc_pub_key: String::new(), + socket_addr: String::new(), + node_name: String::new(), + service_fee_rate: 0.0, + available_peg_btc: "0".to_string(), + ..Default::default() + }), + ) + .serialize_message() + .await + .unwrap(); + + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Immediate(_))); + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ImmediateRateLimited))); + let other = PeerId::random(); + let verdict = gates.evaluate(&local_db, &SMALL, &other, 1, &request, now).await; + assert!(matches!(verdict, InboundVerdict::Immediate(_)), "limits are per sender"); +} + +/// Verify equivalent key encodings share one operator identity. +#[test] +fn operator_identity_ignores_how_the_key_is_spelled() { + use crate::messages::{NodeInfo, sign_node_info_binding, verify_node_info_binding}; + use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; + + let secp = Secp256k1::new(); + let keypair = Keypair::from_secret_key(&secp, &SecretKey::from_slice(&[9; 32]).unwrap()); + let compressed = bitcoin::PublicKey::new(keypair.public_key()); + let mut other_parity = compressed.to_string(); + other_parity.replace_range(..2, if other_parity.starts_with("02") { "03" } else { "02" }); + let mut uncompressed = compressed; + uncompressed.compressed = false; + let spellings = [ + compressed.to_string(), + compressed.to_string().to_uppercase(), + other_parity, + uncompressed.to_string(), + ]; + + let registry = PeerRegistry::default(); + let now = Instant::now(); + let mut keys = HashSet::new(); + let mut peers = Vec::new(); + for (index, spelling) in spellings.iter().enumerate() { + let peer = PeerId::random(); + let issued_at = 100 + index as i64; + let node_info = NodeInfo { + peer_id: peer.to_string(), + btc_pub_key: spelling.clone(), + binding_sig: sign_node_info_binding(&peer.to_string(), spelling, issued_at, &keypair), + binding_issued_at: issued_at, + ..Default::default() + }; + let verified = verify_node_info_binding(&node_info) + .unwrap_or_else(|| panic!("{spelling} is a valid spelling of the key")); + let operator = operator_key(&verified); + keys.insert(operator); + registry.observe_operator_binding(&peer, &operator, issued_at); + if registry.plan_operator_refresh(&peer, &operator, now) { + registry.record_operator_stake(&peer, &operator, Some(true), now); + } + peers.push(peer); + } + assert_eq!(keys.len(), 1, "every spelling is the same operator"); + let registered = + peers.iter().filter(|peer| registry.sender_class(peer, now).is_registered()).count(); + assert_eq!(registered, 1, "one stake backs one peer id, however the key is written"); + assert!(registry.sender_class(peers.last().unwrap(), now).is_registered()); +} + +/// Verify equal timestamps cannot transfer ownership and key changes revoke prior registration. +#[test] +fn operator_key_changes_hands_only_forward_and_rebinding_revokes() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (first, second) = (PeerId::random(), PeerId::random()); + let (staked, unstaked) = (key(1), key(2)); + + assert!(!registry.observe_operator_binding(&first, &staked, 100)); + assert!(registry.plan_operator_refresh(&first, &staked, now)); + assert_eq!(registry.record_operator_stake(&first, &staked, Some(true), now), Some(true)); + + registry.observe_operator_binding(&second, &staked, 100); + assert!(registry.sender_class(&first, now).is_registered(), "same issue time: no change"); + assert!(!registry.plan_operator_refresh(&second, &staked, now)); + + // Moving to an unstaked key gives up the confirmed one ... + assert!(registry.observe_operator_binding(&first, &unstaked, 200)); + assert!(!registry.sender_class(&first, now).is_registered()); + // ... and the negative verdict for the new key is still handed back, so + // the store is cleaned even though nothing "changed" for this binding. + assert!(registry.plan_operator_refresh(&first, &unstaked, now)); + assert_eq!(registry.record_operator_stake(&first, &unstaked, Some(false), now), Some(false)); + // A verdict for the key the peer no longer holds is ignored. + assert_eq!(registry.record_operator_stake(&first, &staked, Some(true), now), None); +} + +/// A stored confirmation is restored only for the key it was made for. +#[tokio::test] +async fn stored_operator_confirmation_is_bound_to_its_key() { + let local_db = store::create_local_db("sqlite::memory:").await; + let now = Instant::now() + OPERATOR_STAKE_STALE_LIMIT; + let (kept, rebound, member) = (PeerId::random(), PeerId::random(), PeerId::random()); + let (staked, unstaked) = (key(1), key(2)); + { + let mut storage = local_db.acquire().await.unwrap(); + for peer in [&kept, &rebound] { + storage + .upsert_p2p_registered_peer( + &peer.to_string(), + REGISTERED_KIND_OPERATOR, + &hex::encode(if peer == &kept { staked } else { key(3) }), + ) + .await + .unwrap(); + } + storage + .upsert_p2p_registered_peer(&member.to_string(), REGISTERED_KIND_COMMITTEE, "") + .await + .unwrap(); + } + // `rebound` was confirmed for key 3 but now presents a (validly signed) + // binding for an unstaked key. + let bindings = HashMap::from([(kept, (staked, 10)), (rebound, (unstaked, 20))]); + let registry = PeerRegistry::default(); + for (peer, (operator, issued_at)) in &bindings { + registry.observe_operator_binding(peer, operator, *issued_at); + } + let seeded = seed_registry_from_store(&local_db, ®istry, &bindings, now).await.unwrap(); + assert_eq!(seeded, 2, "the kept operator and the committee member"); + assert_eq!(registry.sender_class(&kept, now), SenderClass::Registered); + assert_eq!(registry.sender_class(&member, now), SenderClass::Committee); + assert_eq!(registry.sender_class(&rebound, now), SenderClass::Unregistered); + + // The revocation a re-binding triggers removes the stored row. + revoke_persisted_operator(&local_db, &rebound).await; + let stored = local_db.acquire().await.unwrap().load_p2p_registered_peers().await.unwrap(); + assert!(!stored.iter().any(|(peer_id, _, _)| *peer_id == rebound.to_string())); +} + +#[test] +fn committee_reserve_is_closed_to_other_senders() { + use SenderClass::{Committee, Registered}; + let limits = InboundLimits { committee_reserve: Quota { rows: 2, bytes: 2000 }, ..SMALL }; + + // Four rows from staked operators fill everything outside the reserve. + let crowded = [usage(Registered, 4, 400, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &crowded, &limits), Err(DropReason::GlobalQuota)); + assert_eq!(check_inbox_quota(Committee, 100, &crowded, &limits), Ok(())); + // Committee rows do not count against the others' share. + let committee_heavy = [usage(Committee, 3, 300, (0, 0)), usage(Registered, 2, 200, (0, 0))]; + assert_eq!(check_inbox_quota(Registered, 100, &committee_heavy, &limits), Ok(())); + // The committee is still bounded by the global ceiling. + let full = [usage(Committee, 6, 600, (0, 0))]; + assert_eq!(check_inbox_quota(Committee, 100, &full, &limits), Err(DropReason::GlobalQuota)); +} + +/// Verify a full gate rejects new live keys. +#[test] +fn gates_and_sets_refuse_new_keys_when_full_of_live_ones() { + let now = Instant::now(); + let gate = CooldownGate::new(Duration::from_secs(60), 4); + for index in 0..4 { + assert!(gate.allow(&format!("key-{index}"), now)); + } + for index in 4..64 { + assert!(!gate.allow(&format!("key-{index}"), now + Duration::from_secs(1))); + } + assert_eq!(gate.len(), 4); + assert!(gate.is_cooling("key-0", now + Duration::from_secs(1))); + assert!(!gate.is_cooling("key-9", now + Duration::from_secs(1)), "never recorded"); + // Expired keys make room again, once the sweep interval has passed. + assert!(gate.allow("late", now + Duration::from_secs(61))); + assert_eq!(gate.len(), 1); +} + +#[test] +fn global_rate_spends_its_burst_and_refills_at_its_rate() { + let rate = GlobalRate::new(2.0, 1.0); + let now = Instant::now(); + assert!(rate.allow(now)); + assert!(rate.allow(now)); + assert!(!rate.allow(now), "the burst is spent"); + assert!(rate.allow(now + Duration::from_secs(1)), "one token per second comes back"); + assert!(!rate.allow(now + Duration::from_secs(1))); +} + +/// Verify future sequence numbers do not move the window. +#[test] +fn future_sequence_numbers_cannot_raise_the_mark() { + let guard = ReplayGuard::default(); + let author = PeerId::random(); + let ahead = WALL_CLOCK + REPLAY_MAX_FUTURE.as_nanos() as u64; + assert_eq!(guard.admit(&author, Some(ahead + 1), WALL_CLOCK), Err(DropReason::FutureSequence)); + assert_eq!( + guard.admit(&author, Some(ahead - REPLAY_WINDOW), WALL_CLOCK), + Ok(()), + "nothing was recorded: the window did not move up to the refused number" + ); + assert_eq!(guard.admit(&author, Some(ahead), WALL_CLOCK), Ok(()), "inside the tolerance"); + + // The corrected clock numbers lower than the mark: refused, and surfaced + // for the operator once it keeps happening. + assert!(guard.take_lockout_suspects().is_empty()); + for offset in 0..REPLAY_LOCKOUT_SUSPECT_REJECTIONS as u64 { + assert_eq!( + guard.admit(&author, Some(WALL_CLOCK - REPLAY_WINDOW * 2 + offset), WALL_CLOCK), + Err(DropReason::ReplayedSequence) + ); + } + assert_eq!(guard.take_lockout_suspects(), vec![(author, REPLAY_LOCKOUT_SUSPECT_REJECTIONS)]); + assert!(guard.take_lockout_suspects().is_empty(), "reported once per tick"); +} + +#[test] +fn direct_peer_is_banned_at_the_strike_limit_and_released_later() { + let strikes = DirectPeerStrikes::default(); + let now = Instant::now(); + let (flooder, bystander) = (PeerId::random(), PeerId::random()); + for _ in 0..(STRIKE_LIMIT - 1) { + assert!(!strikes.strike(&flooder, now)); + } + assert!(!strikes.strike(&bystander, now)); + assert!(strikes.strike(&flooder, now), "the limit is reached exactly once"); + assert!(!strikes.strike(&flooder, now), "already banned"); + assert!(strikes.take_expired_bans(now).is_empty()); + assert_eq!(strikes.take_expired_bans(now + STRIKE_BAN), vec![flooder]); + + // Strikes spread thinner than the window never add up. + let slow = PeerId::random(); + for round in 0..(STRIKE_LIMIT * 2) { + assert!(!strikes.strike(&slow, now + STRIKE_WINDOW * round)); + } + + // Only verdicts that do not depend on this node's state are blamed on + // the neighbour; and an unanswered lookup is not "known unregistered". + assert!(DropReason::Undecodable.blames_direct_peer()); + assert!(DropReason::UnexpectedBinaryKind.blames_direct_peer()); + for local in [ + DropReason::SenderQuota, + DropReason::ReplayedSequence, + // An honest relay earns this one just by being the only neighbour. + DropReason::DirectPeerRate, + // The JSON ceiling is configurable, so nodes may disagree on it. + DropReason::OversizedJson, + DropReason::BinaryFromNonVerifier, + ] { + assert!(!local.blames_direct_peer(), "{local:?} depends on this node"); + } + assert!(!strikes.is_banned(&bystander, now)); + let banned = PeerId::random(); + for _ in 0..STRIKE_LIMIT { + strikes.strike(&banned, now); + } + assert!(strikes.is_banned(&banned, now), "a ban holds against reconnecting"); + assert!(!strikes.is_banned(&banned, now + STRIKE_BAN)); + let registry = PeerRegistry::default(); + assert!(!registry.is_known_unregistered(&flooder, now)); + registry.record_committee_peer(&flooder, Some(false), now); + assert!(registry.is_known_unregistered(&flooder, now)); + registry.record_committee_peer(&bystander, Some(true), now); + assert!(!registry.is_known_unregistered(&bystander, now)); +} + +/// A durable message no handler of this node's role acts on is relayed, but +/// takes no inbox row and no quota. +#[tokio::test] +async fn messages_for_another_role_are_forwarded_without_being_stored() { + let local_db = store::create_local_db("sqlite::memory:").await; + let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let now = Instant::now(); + let source = PeerId::random(); + gates.registry.record_committee_peer(&source, Some(true), now); + let vote = durable_bytes().await; + + gates.local_actor = Actor::Verifier; + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &vote, now).await; + assert!(matches!(verdict, InboundVerdict::Forward), "a verifier node has no handler"); + gates.local_actor = Actor::Committee; + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &vote, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { .. })); +} + +/// Every kind `dispatch` or an aggregate handles is kept by `handled_by` +/// for the roles that handle it, and relayed for the others. +#[test] +fn handled_kinds_follow_the_role_table() { + use crate::messages::MessageKind; + + let roles = [Actor::Committee, Actor::Operator, Actor::Verifier, Actor::Watchtower]; + // Immediate kinds reach every role's dispatch. + for kind in + [MessageKind::GraphSetupAck, MessageKind::RequestNodeInfo, MessageKind::ResponseNodeInfo] + { + for role in &roles { + assert!(kind.handled_by(role), "{kind} for {role}"); + } + } + // The PeginConfirm round is the committee's. + for kind in [ + MessageKind::PeginConfirmNonce, + MessageKind::PeginConfirmNonceConsensus, + MessageKind::PeginConfirmPartialSig, + ] { + assert!(kind.handled_by(&Actor::Committee), "{kind}"); + for role in [Actor::Operator, Actor::Verifier, Actor::Watchtower] { + assert!(!kind.handled_by(&role), "{kind} for {role}"); + } + } + // Graph-phase kinds follow the Graph aggregate's routing table. + assert!(MessageKind::InitGraph.handled_by(&Actor::Verifier)); + assert!(!MessageKind::InitGraph.handled_by(&Actor::Committee)); + assert!(MessageKind::GraphFinalize.handled_by(&Actor::Watchtower)); +} + +/// Verify committee-only reserves in shared rate tiers. +#[test] +fn shared_rate_tiers_keep_a_reserve_for_the_committee() { + let tier = TierRate { msg_burst: 8, msg_per_sec: 0, byte_burst: 8_000, byte_per_sec: 0 }; + let roomy = TierRate { msg_burst: 1_000, msg_per_sec: 0, byte_burst: 1 << 30, byte_per_sec: 0 }; + let limiter = InboundRateLimiter::new(RateLimits { + direct_peer: roomy, + registered_author: roomy, + unregistered_author: roomy, + unregistered_total: roomy, + unregistered_forward: roomy, + global: tier, + committee_reserve: 0.25, + max_tracked_peers: 16, + }); + let now = Instant::now(); + let relay = PeerId::random(); + use SenderClass::{Committee, Registered, Unregistered}; + + // Six of eight messages are open to everyone ... + for _ in 0..6 { + assert!(limiter.charge(&relay, &PeerId::random(), Registered, 100, now).is_ok()); + } + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Registered, 100, now), + Err(DropReason::GlobalRate) + ); + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Unregistered, 100, now), + Err(DropReason::GlobalRate) + ); + // ... and the last two only to the committee. + for _ in 0..2 { + assert!(limiter.charge(&relay, &PeerId::random(), Committee, 100, now).is_ok()); + } + assert_eq!( + limiter.charge(&relay, &PeerId::random(), Committee, 100, now), + Err(DropReason::GlobalRate) + ); +} + +/// Verify unregistered forward-only traffic is rate-limited. +#[tokio::test] +async fn forwarding_for_unregistered_authors_is_budgeted() { + let local_db = store::create_local_db("sqlite::memory:").await; + // A committee node relays the operator's recruitment to verifiers. + let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let tier = TierRate { msg_burst: 2, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; + gates.rate_limiter = InboundRateLimiter::new(RateLimits { + unregistered_forward: tier, + ..RateLimits::defaults() + }); + let now = Instant::now(); + + for sequence_number in 0..2 { + let data = init_graph().serialize_message().await.unwrap(); + let verdict = + gates.evaluate(&local_db, &SMALL, &PeerId::random(), sequence_number, &data, now).await; + assert!(matches!(verdict, InboundVerdict::Forward)); + } + let data = init_graph().serialize_message().await.unwrap(); + let verdict = gates.evaluate(&local_db, &SMALL, &PeerId::random(), 9, &data, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ForwardRate))); + + let member = PeerId::random(); + gates.registry.record_committee_peer(&member, Some(true), now); + let verdict = gates.evaluate(&local_db, &SMALL, &member, 1, &data, now).await; + assert!(matches!(verdict, InboundVerdict::Forward), "registered authors are not charged"); +} + +/// A durable message this role handles is relayed, not stored, while its +/// author is unregistered; once the author is known it is stored. +#[tokio::test] +async fn durable_messages_from_unregistered_authors_are_relayed_not_stored() { + let local_db = store::create_local_db("sqlite::memory:").await; + let mut gates = TestGates::new(PeerRateLimiter::new(10.0, 1.0, 16)); + let tier = TierRate { msg_burst: 1, msg_per_sec: 0, byte_burst: 1 << 20, byte_per_sec: 0 }; + gates.rate_limiter = InboundRateLimiter::new(RateLimits { + unregistered_forward: tier, + ..RateLimits::defaults() + }); + let now = Instant::now(); + let source = PeerId::random(); + let vote = durable_bytes().await; + + let verdict = gates.evaluate(&local_db, &SMALL, &source, 1, &vote, now).await; + assert!(matches!(verdict, InboundVerdict::Forward)); + // The relay budget is the unregistered one; past it the message is dropped. + let verdict = gates.evaluate(&local_db, &SMALL, &source, 2, &vote, now).await; + assert!(matches!(verdict, InboundVerdict::Drop(DropReason::ForwardRate))); + // Nothing was queued for the quota. + assert!( + local_db.acquire().await.unwrap().engine_inbox_class_totals().await.unwrap().is_empty() + ); + + gates.registry.record_committee_peer(&source, Some(true), now); + let verdict = gates.evaluate(&local_db, &SMALL, &source, 3, &vote, now).await; + assert!(matches!(verdict, InboundVerdict::Enqueue { class: SenderClass::Committee, .. })); +} + +/// Verify neighbour discovery is independent of author discovery. +#[test] +fn neighbour_lookups_have_their_own_budget() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + registry.record_verifier_set(Some(vec![]), now); + for _ in 0..(COMMITTEE_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let author = PeerId::random(); + if registry.plan_refresh(Some(&author), now).committee_peer { + registry.record_committee_peer(&author, Some(false), now); + } + } + let neighbour = PeerId::random(); + assert!(!registry.plan_refresh(Some(&neighbour), now).committee_peer, "budget spent"); + assert!(registry.plan_neighbour_lookup(&neighbour, now)); + + // Verify refused peers cannot consume neighbour discovery budget. + let mut refused = 0; + for _ in 0..(REFUSED_PEER_LOOKUPS_PER_MINUTE * 2) { + let peer = PeerId::random(); + if registry.plan_refused_peer_lookup(&peer, now) { + refused += 1; + registry.record_committee_peer(&peer, Some(false), now); + } + } + assert_eq!(refused, REFUSED_PEER_LOOKUPS_PER_MINUTE); + assert!(registry.plan_neighbour_lookup(&PeerId::random(), now)); +} + +/// Verify committee-only lookup does not claim verifier-set refresh. +#[test] +fn neighbour_lookup_does_not_claim_the_verifier_set() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + // The verifier set has never been fetched, so it is due. + assert!(registry.plan_neighbour_lookup(&PeerId::random(), now)); + assert!(registry.plan_refused_peer_lookup(&PeerId::random(), now)); + assert!( + registry.plan_refresh(None, now).verifier_set, + "the verifier set is still there to be claimed by a caller that fetches it" + ); +} + +/// Verify configured bindings still require confirmed stake. +#[test] +fn trusted_operator_binding_needs_the_chain_but_not_the_budget() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + // Spend the operator discovery budget on self-signed bindings. + for index in 0..(OPERATOR_DISCOVERY_LOOKUPS_PER_MINUTE * 2) { + let (sybil, sybil_key) = (PeerId::random(), key(index + 10)); + registry.observe_operator_binding(&sybil, &sybil_key, 1); + if registry.plan_operator_refresh(&sybil, &sybil_key, now) { + registry.record_operator_stake(&sybil, &sybil_key, Some(false), now); + } + } + let (newcomer, newcomer_key) = (PeerId::random(), key(1)); + registry.observe_operator_binding(&newcomer, &newcomer_key, 1); + assert!(!registry.plan_operator_refresh(&newcomer, &newcomer_key, now), "budget spent"); + + let (trusted, trusted_key) = (PeerId::random(), key(2)); + assert_eq!(registry.trust_operator_binding(&trusted, &trusted_key), Ok(())); + assert!(!registry.sender_class(&trusted, now).is_registered(), "configured is not staked"); + let due = registry.plan_due_operator_refreshes(now); + assert!(due.contains(&(trusted, trusted_key)), "looked up despite the spent budget"); + assert!( + due.iter().all(|(_, operator)| *operator == trusted_key), + "keys the chain said are not staked are not swept again, and the newcomer still \ + has no budget: {} due", + due.len() + ); + registry.record_operator_stake(&trusted, &trusted_key, Some(true), now); + assert_eq!(registry.sender_class(&trusted, now), SenderClass::Registered); + + // The operator's own signed announcement later keeps the verdict ... + registry.observe_operator_binding(&trusted, &trusted_key, 500); + assert!(registry.sender_class(&trusted, now).is_registered()); + // ... and if it moves to another peer id, its signature outranks the + // static configuration. + let moved = PeerId::random(); + registry.observe_operator_binding(&moved, &trusted_key, 600); + assert!(!registry.sender_class(&trusted, now).is_registered()); +} + +/// Verify signed bindings take precedence over configuration. +#[test] +fn configured_binding_does_not_override_a_signed_one() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let (operator, other_peer) = (PeerId::random(), PeerId::random()); + let (signed_key, configured_key) = (key(1), key(2)); + + // Restored from the store: the operator's own signed binding, confirmed. + registry.seed_verified_operator(&operator, &signed_key, 100, now); + let now = now + Duration::from_secs(1); + assert!(registry.sender_class(&operator, now).is_registered()); + + // The config still names an older key for that peer ... + assert_eq!( + registry.trust_operator_binding(&operator, &configured_key), + Err(TrustedBindingConflict::PeerBoundToAnotherKey(signed_key)) + ); + // ... or names the operator's key under a peer id it has moved away from. + assert_eq!( + registry.trust_operator_binding(&other_peer, &signed_key), + Err(TrustedBindingConflict::KeyBoundToAnotherPeer(operator)) + ); + assert!(registry.sender_class(&operator, now).is_registered(), "the signed binding stands"); + assert!(!registry.sender_class(&other_peer, now).is_registered()); + + // Agreeing with the signed binding keeps its verdict and adds the trust. + assert_eq!(registry.trust_operator_binding(&operator, &signed_key), Ok(())); + assert!(registry.sender_class(&operator, now).is_registered()); + + // The same peer configured twice with different keys: first one stands. + let fresh = PeerId::random(); + assert_eq!(registry.trust_operator_binding(&fresh, &key(3)), Ok(())); + assert_eq!( + registry.trust_operator_binding(&fresh, &key(4)), + Err(TrustedBindingConflict::PeerBoundToAnotherKey(key(3))) + ); +} + +/// Verify clock leads are recorded within and beyond tolerance. +#[test] +fn clock_leads_are_reported_per_author_and_then_forgotten() { + let guard = ReplayGuard::default(); + let (slightly_ahead, far_ahead, on_time) = + (PeerId::random(), PeerId::random(), PeerId::random()); + let minute = Duration::from_secs(60).as_nanos() as u64; + + assert_eq!(guard.admit(&on_time, Some(WALL_CLOCK - minute), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&slightly_ahead, Some(WALL_CLOCK + 2 * minute), WALL_CLOCK), Ok(())); + assert_eq!(guard.admit(&slightly_ahead, Some(WALL_CLOCK + 3 * minute), WALL_CLOCK), Ok(())); + assert_eq!( + guard.admit(&far_ahead, Some(WALL_CLOCK + 30 * minute), WALL_CLOCK), + Err(DropReason::FutureSequence) + ); + + let mut leads = guard.take_clock_leads(); + leads.sort_by_key(|(_, lead)| *lead); + assert_eq!( + leads, + vec![ + (slightly_ahead, Duration::from_secs(3 * 60)), + (far_ahead, Duration::from_secs(30 * 60)) + ], + "the largest lead per author; an author on time is not listed" + ); + assert!(guard.take_clock_leads().is_empty()); +} + +/// Verify only successful recovery closes the gate. +/// A "no" that has gone stale is not grounds for acting against a neighbour. +#[test] +fn a_stale_negative_is_not_known_unregistered() { + let registry = PeerRegistry::default(); + let now = Instant::now(); + let peer = PeerId::random(); + registry.record_committee_peer(&peer, Some(false), now); + assert!(registry.is_known_unregistered(&peer, now)); + assert!(!registry.is_known_unregistered(&peer, now + COMMITTEE_NEGATIVE_TTL)); +} + +#[test] +fn trusted_operator_bindings_are_parsed_strictly() { + use crate::env::parse_trusted_operator_bindings; + use bitcoin::secp256k1::{Keypair, Secp256k1, SecretKey}; + + let secp = Secp256k1::new(); + let keypair = Keypair::from_secret_key(&secp, &SecretKey::from_slice(&[9; 32]).unwrap()); + let pubkey = bitcoin::PublicKey::new(keypair.public_key()); + let xonly = bitcoin::XOnlyPublicKey::from(pubkey).serialize(); + let (first, second) = (PeerId::random(), PeerId::random()); + let value = format!( + " {first}={pubkey} , not-a-binding, {second} = {} ,{first}=zz,", + pubkey.to_string().to_uppercase() + ); + assert_eq!( + parse_trusted_operator_bindings(&value), + vec![(first, xonly), (second, xonly)], + "malformed entries are skipped, and every spelling is the same key" + ); + assert!(parse_trusted_operator_bindings("").is_empty()); +} diff --git a/node/src/p2p_msg_handler.rs b/node/src/p2p_msg_handler.rs index 5ac2c6ad6..0d4722d77 100644 --- a/node/src/p2p_msg_handler.rs +++ b/node/src/p2p_msg_handler.rs @@ -1,30 +1,34 @@ -use crate::action::{ - GOATMessage, GOATMessageContent, handle_inbound_p2p_message, handle_self_p2p_msg, send_to_peer, -}; +//! The swarm's message handler: admission of inbound gossip, the control +//! worker for `NodeInfo` and ACKs, the admission maintenance task and the +//! direct `GetGraph` service. Protocol work runs on the engine. + +use crate::aggregates::common::inbound_event; +use crate::control::{ControlContext, dispatch_control}; use crate::env::get_local_node_info; +use crate::messages::{GOATMessage, GOATMessageContent}; use crate::metrics_service::MetricsState; -use crate::middleware::publisher::NetworkPublisher; +use crate::middleware::AllBehaviours; +use crate::middleware::get_graph::{GetGraphRequest, GetGraphResponse}; +use crate::middleware::publisher::{MessagePublisher, NetworkPublisher, publish_to_role}; use crate::middleware::swarm::{BitvmSwarmWrapper, P2pMessageHandler, TickMessageType}; -use crate::utils::detect_heart_beat; +use crate::utils::{detect_heart_beat, update_node_timestamp}; +use anyhow::{Context, Result}; use bitvm_lib::actors::Actor; -use bitvm_lib::babe_adapter::BabeBundleBuilder; -use client::http_client::async_client::HttpAsyncClient; -use client::{btc_chain::BTCClient, goat_chain::GOATClient}; +use client::goat_chain::GOATClient; use futures::FutureExt; -use libp2p::PeerId; use libp2p::gossipsub::MessageId; +use libp2p::request_response::ResponseChannel; +use libp2p::{PeerId, Swarm, gossipsub}; use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; use store::localdb::LocalDB; -use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; use tokio_util::sync::CancellationToken; #[derive(Clone)] pub struct BitvmNodeProcessor { pub local_db: LocalDB, - pub btc_client: Arc, pub goat_client: Arc, - pub http_client: Arc, - pub soldering_builder: Option>, pub metrics_state: MetricsState, pub shutdown_token: CancellationToken, pub worker: Arc>, @@ -32,7 +36,6 @@ pub struct BitvmNodeProcessor { pub struct ImmediateMessage { pub source: PeerId, - pub id: MessageId, pub message: GOATMessage, _bytes: OwnedSemaphorePermit, } @@ -40,7 +43,6 @@ pub struct ImmediateMessage { pub struct WorkerControl { immediate: mpsc::Sender, bytes: Arc, - tick: Arc, joins: tokio::sync::Mutex>>, } @@ -84,119 +86,61 @@ where } impl WorkerControl { - pub fn enqueue( - &self, - source: PeerId, - id: MessageId, - message: GOATMessage, - bytes: usize, - ) -> bool { + pub fn enqueue(&self, source: PeerId, message: GOATMessage, bytes: usize) -> bool { let Ok(bytes) = u32::try_from(bytes.max(1)) else { return false }; let Ok(permit) = self.bytes.clone().try_acquire_many_owned(bytes) else { return false }; - self.immediate.try_send(ImmediateMessage { source, id, message, _bytes: permit }).is_ok() + self.immediate.try_send(ImmediateMessage { source, message, _bytes: permit }).is_ok() } } impl BitvmNodeProcessor { - fn worker(&self, swarm: &BitvmSwarmWrapper, actor: Actor) -> &WorkerControl { - self.worker.get_or_init(|| { - // Run sequential business dispatch on a separate thread. + /// Start the control worker and the admission maintenance task. Called at + /// boot so maintenance (verifier-set retries, registry refreshes, drop + /// summaries) runs before the first gossip message arrives. + pub fn start(&self, publisher: NetworkPublisher) { + self.worker.get_or_init(|| self.spawn_worker(publisher)); + } + + fn worker(&self, swarm: &BitvmSwarmWrapper) -> &WorkerControl { + self.worker.get_or_init(|| self.spawn_worker(swarm.publisher())) + } + + fn spawn_worker(&self, publisher: NetworkPublisher) -> WorkerControl { + { + // Control dispatch (NodeInfo, ACKs) runs on its own thread, off + // the network loop; the engine runs everything else. let (tx, rx) = mpsc::channel(16); - let tick = Arc::new(Notify::new()); let mut processor = self.clone(); processor.worker = Default::default(); - let publisher = swarm.publisher(); - let worker_tick = tick.clone(); let control_processor = processor.clone(); let control_publisher = publisher.clone(); - let control_actor = actor.clone(); let control_join = spawn_isolated_worker( "bitvm-control", self.shutdown_token.clone(), move || async move { - control_processor - .run_immediate_worker(control_publisher, control_actor, rx) - .await; - }, - ); - let maintenance_join = - tokio::spawn(processor.clone().run_admission_maintenance(publisher.clone())); - let join = spawn_isolated_worker( - "bitvm-business", - self.shutdown_token.clone(), - move || async move { - processor.run_worker(publisher, actor, worker_tick).await; + control_processor.run_immediate_worker(control_publisher, rx).await; }, ); + let maintenance_join = tokio::spawn(processor.run_admission_maintenance(publisher)); WorkerControl { immediate: tx, bytes: Arc::new(Semaphore::new(16 * 1024 * 1024)), - tick, - joins: tokio::sync::Mutex::new(vec![join, control_join, maintenance_join]), + joins: tokio::sync::Mutex::new(vec![control_join, maintenance_join]), } - }) - } - - async fn run_worker(&self, mut publisher: NetworkPublisher, actor: Actor, tick: Arc) { - let mut backlog = false; - loop { - // Coalesce tick notifications and continue immediately while backlogged. - // Each pass serves local messages and outbox before inbox. - if backlog { - tokio::task::yield_now().await; - } else { - tick.notified().await; - } - let data = - GOATMessage::new(actor.clone(), GOATMessageContent::Tick).serialize_message().await; - let result = match data { - Ok(data) => { - handle_self_p2p_msg( - &mut publisher, - &self.local_db, - &self.btc_client, - &self.goat_client, - &self.http_client, - &self.soldering_builder, - actor.clone(), - crate::env::get_peer_id().parse().expect("configured peer id"), - GOATMessage::default_message_id(), - &data, - &self.metrics_state, - &self.shutdown_token, - ) - .await - } - Err(error) => Err(error), - }; - backlog = match result { - Ok(backlog) => backlog, - Err(error) => { - tracing::error!(event = "p2p_worker", error = %error, "business dispatch failed"); - false - } - }; } } async fn run_immediate_worker( &self, mut publisher: NetworkPublisher, - actor: Actor, mut immediate: mpsc::Receiver, ) { - // Control queue: NodeInfo and ACK only; graph sync uses the durable inbox. while let Some(message) = immediate.recv().await { - if let Err(error) = crate::action::dispatch_immediate_message( + if let Err(error) = dispatch_immediate_message( &mut publisher, &self.local_db, - &self.btc_client, &self.goat_client, - &self.http_client, - &self.soldering_builder, - actor.clone(), message.source, - message.id, message.message, &self.metrics_state, ) @@ -207,8 +151,9 @@ impl BitvmNodeProcessor { } } - /// Independently refresh registrations, persist replay marks and report clock skew. - /// This task is the sole caller of admission maintenance. + /// Refresh registrations, report drops and clock skew, purge stale node + /// rows and flush deferred NodeInfo answers. This task is the sole caller + /// of admission maintenance. async fn run_admission_maintenance(self, mut publisher: NetworkPublisher) { let mut clock = tokio::time::interval(std::time::Duration::from_secs( crate::env::REGULAR_TASK_INTERVAL_SECOND, @@ -221,14 +166,14 @@ impl BitvmNodeProcessor { _ = clock.tick() => {} } // A pass is cut short by shutdown too: `graceful_shutdown` joins - // this task before it releases the queue claims. + // this task. tokio::select! { biased; _ = self.shutdown_token.cancelled() => return, _ = async { - crate::action::run_p2p_admission_maintenance(&self.local_db, &self.goat_client) + crate::p2p_admission::run_p2p_admission_maintenance(&self.local_db, &self.goat_client) .await; - crate::handle::flush_deferred_node_info_response(&mut publisher).await; + crate::control::flush_deferred_node_info_response(&mut publisher).await; } => {} } } @@ -245,14 +190,11 @@ impl P2pMessageHandler for BitvmNodeProcessor { id: MessageId, message: &[u8], ) -> anyhow::Result<()> { - let worker = self.worker(swarm, actor.clone()); + let worker = self.worker(swarm); handle_inbound_p2p_message( swarm, &self.local_db, - &self.btc_client, &self.goat_client, - &self.http_client, - &self.soldering_builder, actor, from_peer_id, propagation_source, @@ -284,7 +226,9 @@ impl P2pMessageHandler for BitvmNodeProcessor { Ok(()) } TickMessageType::RegularlyAction => { - self.worker(swarm, actor).tick.notify_one(); + // Nothing runs on the tick any more: the engine schedules its + // own work and the admission maintenance has its own timer. + let _ = (swarm, actor); Ok(()) } } @@ -298,7 +242,7 @@ impl P2pMessageHandler for BitvmNodeProcessor { ) -> anyhow::Result<()> { if topic == Actor::All.to_string() { let message_content = GOATMessageContent::RequestNodeInfo(get_local_node_info()); - match send_to_peer(swarm, GOATMessage::new(Actor::All, message_content)).await { + match publish_to_role(swarm, GOATMessage::new(Actor::All, message_content)).await { Ok(_) => {} Err(e) => { println!("finish_subscribe_topic: send request NodeInfo {e}"); @@ -308,6 +252,63 @@ impl P2pMessageHandler for BitvmNodeProcessor { Ok(()) } + /// Serve a stored graph definition to a direct `GetGraph` request. Any + /// role answers from its own copy; the requester authenticates what it + /// gets. Admission is a global budget by requester class first (it has no + /// per-key table an unregistered flood could fill), then the per-peer + /// bucket, the per-request cooldown and a cap on definitions in flight; + /// the read and the serialization run off the network loop. + fn serve_graph_request( + &self, + peer: PeerId, + request: GetGraphRequest, + channel: ResponseChannel, + publisher: NetworkPublisher, + ) -> Option> { + let now = std::time::Instant::now(); + let registered = + crate::p2p_admission::peer_registry().sender_class(&peer, now).is_registered(); + if !crate::p2p_admission::get_graph_response_budget(registered).allow(now) { + tracing::debug!(event = "get_graph", outcome = "budget_spent", %peer, registered, "GetGraph refused"); + return Some(channel); + } + if !crate::p2p_admission::get_graph_peer_budget().allow(&peer, now) { + tracing::debug!(event = "get_graph", outcome = "peer_budget_spent", %peer, "GetGraph refused"); + return Some(channel); + } + let gate_key = format!( + "{peer}:{:?}:{:?}:{:?}", + request.graph_id, request.operator_pubkey, request.graph_nonce + ); + if !crate::p2p_admission::get_graph_response_gate().allow(&gate_key, now) { + return Some(channel); + } + let Ok(slot) = crate::p2p_admission::get_graph_serve_slots().clone().try_acquire_owned() + else { + tracing::debug!(event = "get_graph", outcome = "busy", %peer, "GetGraph refused"); + return Some(channel); + }; + let local_db = self.local_db.clone(); + tokio::spawn(async move { + let _slot = slot; + let response = match crate::aggregates::graph_fetch::serve_graph_request( + &local_db, &request, + ) + .await + { + Ok(response) => response, + Err(error) => { + tracing::warn!(event = "get_graph", outcome = "serve_failed", %peer, error = %error, "could not serve GetGraph request"); + GetGraphResponse::not_found() + } + }; + if let Err(error) = publisher.respond_graph(channel, response).await { + tracing::debug!(event = "get_graph", outcome = "response_dropped", %peer, error = %error, "GetGraph response not delivered to the swarm"); + } + }); + None + } + async fn graceful_shutdown(&self) -> anyhow::Result<()> { self.shutdown_token.cancel(); if let Some(worker) = self.worker.get() { @@ -315,21 +316,273 @@ impl P2pMessageHandler for BitvmNodeProcessor { join.await?; } } - let mut storage = self.local_db.start_immediate_transaction().await?; - let local_released = storage.release_processing_local_messages().await?; - let inbox_released = storage.release_processing_p2p_inbox_messages().await?; - storage.commit().await?; - tracing::info!( - event = "message_queue_shutdown", - outcome = "claims_released", - local_released, - inbox_released, - "released active queue claims without charging abandon counters" - ); Ok(()) } } +/// Budget of one control message (NodeInfo, ACK): they touch the registry and +/// the engine outbox only. +const IMMEDIATE_DISPATCH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Report exactly one forwarding verdict for each received gossipsub message. +fn report_gossip_validation( + swarm: &mut Swarm, + id: &MessageId, + propagation_source: &PeerId, + acceptance: gossipsub::MessageAcceptance, +) { + let cached = swarm.behaviour_mut().gossipsub.report_message_validation_result( + id, + propagation_source, + acceptance, + ); + if !cached { + tracing::debug!( + event = "p2p_message", + outcome = "validation_report_missed", + message_id = %hex::encode(&id.0), + "gossipsub had already evicted the message awaiting validation" + ); + } +} + +/// Apply admission before routing; a dropped message is not an error. +#[allow(clippy::too_many_arguments)] +pub async fn handle_inbound_p2p_message( + swarm: &mut Swarm, + local_db: &LocalDB, + goat_client: &Arc, + actor: Actor, + from_peer_id: PeerId, + propagation_source: PeerId, + sequence_number: Option, + id: MessageId, + message: &[u8], + metrics_state: &MetricsState, + worker: &WorkerControl, +) -> Result<()> { + use crate::p2p_admission::{self, InboundVerdict}; + use gossipsub::MessageAcceptance; + + let now = Instant::now(); + let verdict = p2p_admission::evaluate_inbound_message( + local_db, + &p2p_admission::AdmissionGates::global(&actor), + &p2p_admission::InboundGossip { + direct_peer: &propagation_source, + source: &from_peer_id, + sequence_number, + data: message, + }, + now, + ) + .await; + let verdict = match verdict { + Ok(verdict) => verdict, + Err(error) => { + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Ignore); + return Err(error).context("evaluate inbound P2P message admission"); + } + }; + + match verdict { + InboundVerdict::Drop(reason) => { + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Ignore); + if reason.blames_direct_peer() { + strike_direct_peer(swarm, local_db, goat_client, propagation_source, now); + } + if reason == p2p_admission::DropReason::Undecodable { + metrics_state.record_p2p_receive(false); + } else if reason.after_decode() { + metrics_state.record_p2p_receive(true); + } + p2p_admission::record_drop(reason); + Ok(()) + } + // Forward messages for other roles without persistence or ACK. + InboundVerdict::Forward => { + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + Ok(()) + } + // Record durable messages before allowing forwarding: graph-phase + // messages for the Graph aggregate, PeginConfirm round messages for + // the Instance aggregate. The engine deduplicates by content id and + // ACKs heavy messages once they are durably applied. + InboundVerdict::Enqueue { message: decoded, class } => { + let author = from_peer_id.to_string(); + let event = + inbound_event(&actor, &author, &crate::env::get_peer_id(), &decoded, message); + let recorded = match event { + Ok(Some(event)) => { + record_engine_message( + local_db, + event.with_admission_class(class.to_string()), + &decoded, + from_peer_id, + message.len(), + ) + .await + } + // Kept by `handled_by` but routed by no aggregate: a table + // mismatch, not a bad message. + Ok(None) => Err(anyhow::anyhow!( + "{} is handled by {actor} but no aggregate routes it", + decoded.content.event_type() + )), + Err(error) => Err(error), + }; + if let Err(error) = recorded { + report_gossip_validation( + swarm, + &id, + &propagation_source, + MessageAcceptance::Ignore, + ); + return Err(error); + } + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + Ok(()) + } + InboundVerdict::Immediate(decoded) => { + if !worker.enqueue(from_peer_id, decoded, message.len()) { + report_gossip_validation( + swarm, + &id, + &propagation_source, + MessageAcceptance::Ignore, + ); + p2p_admission::record_drop(p2p_admission::DropReason::ControlQueueFull); + return Ok(()); + } + report_gossip_validation(swarm, &id, &propagation_source, MessageAcceptance::Accept); + metrics_state.record_p2p_receive(true); + refresh_peer_timestamp_throttled(local_db, from_peer_id, now).await; + Ok(()) + } + } +} + +/// Dispatch a control message (NodeInfo, ACK) on the control worker. +pub(crate) async fn dispatch_immediate_message( + publisher: &mut (dyn MessagePublisher + Sync), + local_db: &LocalDB, + goat_client: &Arc, + from_peer_id: PeerId, + message: GOATMessage, + metrics_state: &MetricsState, +) -> Result<()> { + let message_type = message.content.event_type(); + let started_at = Instant::now(); + let mut handler_ctx = ControlContext { publisher, local_db, goat_client, from_peer_id }; + let result = match tokio::time::timeout( + IMMEDIATE_DISPATCH_TIMEOUT, + dispatch_control(&mut handler_ctx, message.content()), + ) + .await + { + Ok(result) => result, + Err(_) => Err(anyhow::anyhow!("control message exceeded its dispatch budget")), + }; + metrics_state + .record_message_dispatch(message_type, if result.is_ok() { "success" } else { "failed" }); + match &result { + Ok(()) => tracing::debug!( + event = "message_dispatch_result", + outcome = "handled", + message_type, + from_peer_id = %from_peer_id, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "control message dispatched" + ), + Err(err) => tracing::warn!( + event = "message_dispatch_result", + outcome = "failed", + message_type, + from_peer_id = %from_peer_id, + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %err, + "control message dispatch failed" + ), + } + result +} + +/// Count relay strikes and temporarily ban repeat offenders. +/// Registered peers and peers with unresolved registration are exempt. +fn strike_direct_peer( + swarm: &mut Swarm, + local_db: &LocalDB, + goat_client: &Arc, + direct_peer: PeerId, + now: Instant, +) { + let registry = crate::p2p_admission::peer_registry(); + if !registry.is_known_unregistered(&direct_peer, now) { + crate::p2p_admission::refresh_neighbour_in_background(local_db, goat_client, direct_peer); + return; + } + if !crate::p2p_admission::direct_peer_strikes().strike(&direct_peer, now) { + return; + } + // The strike table now lists the peer as banned, and the connection gate + // consults it for every connection: closing this one is all that is left. + let _ = swarm.disconnect_peer_id(direct_peer); + tracing::warn!( + event = "p2p_admission", + outcome = "direct_peer_banned", + peer_id = %direct_peer, + "temporarily ignoring a neighbour that keeps relaying invalid messages" + ); +} + +/// Refresh peer liveness at most once per cooldown. +async fn refresh_peer_timestamp_throttled(local_db: &LocalDB, from_peer_id: PeerId, now: Instant) { + let peer = from_peer_id.to_string(); + if !crate::p2p_admission::peer_timestamp_gate().allow(&peer, now) { + return; + } + if let Err(error) = update_node_timestamp(local_db, &peer).await { + tracing::debug!( + event = "p2p_message", + outcome = "peer_timestamp_update_failed", + from_peer_id = %peer, + error = %error, + "failed to refresh peer liveness timestamp" + ); + } +} + +/// Record an admitted durable message as an inbox event of its aggregate. +/// The engine deduplicates by content id and ACKs a heavy message again when +/// its material is already applied or parked. +async fn record_engine_message( + local_db: &LocalDB, + event: engine::NewEvent, + decoded: &GOATMessage, + from_peer_id: PeerId, + content_size: usize, +) -> Result<()> { + let outcome = crate::engine_bridge::submit_event(local_db, event) + .await + .context("record inbound message")?; + if matches!(outcome, engine::SubmitOutcome::Duplicate { .. }) { + crate::p2p_admission::record_drop(crate::p2p_admission::DropReason::DuplicatePayload); + } + tracing::info!( + event = "engine_inbox", + outcome = ?outcome, + message_type = decoded.content.event_type(), + from_peer_id = %from_peer_id, + content_size, + "received durable message" + ); + Ok(()) +} + #[cfg(test)] mod tests { #[tokio::test] @@ -366,24 +619,26 @@ mod tests { let control = super::WorkerControl { immediate: tx, bytes: std::sync::Arc::new(tokio::sync::Semaphore::new(4)), - tick: Default::default(), joins: Default::default(), }; let source = libp2p::PeerId::random(); let message = || { - super::GOATMessage::new(bitvm_lib::actors::Actor::All, super::GOATMessageContent::Tick) + super::GOATMessage::new( + bitvm_lib::actors::Actor::All, + super::GOATMessageContent::RequestNodeInfo(crate::messages::NodeInfo::default()), + ) }; - let id = || super::GOATMessage::default_message_id(); - assert!(!control.enqueue(source, id(), message(), 5)); - assert!(control.enqueue(source, id(), message(), 4)); - assert!(!control.enqueue(source, id(), message(), 1)); + assert!(!control.enqueue(source, message(), 5)); + assert!(control.enqueue(source, message(), 4)); + assert!(!control.enqueue(source, message(), 1)); drop(rx.recv().await.unwrap()); - assert!(control.enqueue(source, id(), message(), 1)); - assert!(!control.enqueue(source, id(), message(), 1)); + assert!(control.enqueue(source, message(), 1)); + assert!(!control.enqueue(source, message(), 1)); assert_eq!(control.bytes.available_permits(), 3, "failed enqueue releases its permits"); } - use crate::action::{GOATMessage, GOATMessageContent, NodeInfo, send_to_peer}; use crate::env::get_rpc_support_actors; + use crate::messages::{GOATMessage, GOATMessageContent, NodeInfo}; + use crate::middleware::publisher::publish_to_role; use crate::middleware::swarm::{ BitvmNetworkManager, BitvmSwarmConfig, BitvmSwarmWrapper, P2pMessageHandler, TickMessageType, @@ -469,7 +724,7 @@ mod tests { // send to actor let actors = get_rpc_support_actors(); for actor in actors { - match send_to_peer(swarm, GOATMessage::new(actor, message_content.clone())).await { + match publish_to_role(swarm, GOATMessage::new(actor, message_content.clone())).await { Ok(_) => {} Err(err) => warn!("{err}"), } @@ -493,10 +748,6 @@ mod tests { &propagation_source, libp2p::gossipsub::MessageAcceptance::Accept, ); - if id == GOATMessage::default_message_id() { - tracing::info!("recv_and_dispatch receive local message"); - return Ok(()); - } let message = GOATMessage::deserialize_message(message).await?; let content: &GOATMessageContent = message.content(); if let (GOATMessageContent::RequestNodeInfo(node_info), _) = (content, actor) { @@ -758,7 +1009,6 @@ mod tests { } async fn relay_scenario() { - use crate::action::KickoffSent; use crate::middleware::behaviour::MAX_GOSSIPSUB_TRANSMIT_SIZE; init(); @@ -778,12 +1028,7 @@ mod tests { let relay = super::BitvmNodeProcessor { local_db: relay_db.clone(), - btc_client: std::sync::Arc::new(client::btc_chain::BTCClient::new_mock_client().0), goat_client: std::sync::Arc::new(client::goat_chain::GOATClient::new_mock_client().0), - http_client: std::sync::Arc::new( - client::http_client::async_client::HttpAsyncClient::new(None), - ), - soldering_builder: None, metrics_state: crate::metrics_service::MetricsState::new(std::sync::Arc::new( std::sync::Mutex::new(Registry::default()), )), @@ -821,12 +1066,23 @@ mod tests { }); let scenario = async { + // A committee round value: the Committee relay keeps it as an + // engine inbox event of its graph. let protocol_message = |graph_id: uuid::Uuid| { + let secp = bitcoin::secp256k1::Secp256k1::new(); + let keypair = bitcoin::secp256k1::Keypair::from_secret_key( + &secp, + &bitcoin::secp256k1::SecretKey::from_slice(&[7; 32]).unwrap(), + ); + let digest = bitcoin::secp256k1::Message::from_digest([3; 32]); GOATMessage::new( Actor::Committee, - GOATMessageContent::KickoffSent(KickoffSent { + GOATMessageContent::AggNonceConsensus(crate::messages::AggNonceConsensus { instance_id: uuid::Uuid::new_v4(), graph_id, + committee_pubkey: bitcoin::PublicKey::new(keypair.public_key()), + consensus_hash: [3; 32], + signature: secp.sign_schnorr(&digest, &keypair), }), ) }; @@ -865,13 +1121,17 @@ mod tests { } let mut storage = relay_db.acquire().await.unwrap(); - let queued = storage.p2p_inbox_class_totals().await.unwrap(); + let queued = storage.engine_inbox_class_totals().await.unwrap(); let (rows, bytes): (i64, i64) = queued .iter() .fold((0, 0), |(rows, bytes), usage| (rows + usage.rows, bytes + usage.bytes)); assert!(rows <= 2, "only the two protocol messages may be queued, found {rows}"); + // An inbox event wraps the raw bytes in the admitted envelope + // (author peer id, instance id); the junk payloads are kilobytes + // to megabytes larger than that. + let envelope_overhead = 2 * 128; assert!( - bytes <= (first.len() + sentinel.len()) as i64, + bytes <= (first.len() + sentinel.len() + envelope_overhead) as i64, "the relay persisted {bytes} bytes; junk must never reach the inbox" ); }; diff --git a/node/src/rpc_service/auth.rs b/node/src/rpc_service/auth.rs index 8499134dd..e54a50bea 100644 --- a/node/src/rpc_service/auth.rs +++ b/node/src/rpc_service/auth.rs @@ -1,6 +1,7 @@ +use crate::clock::current_time_secs; use crate::env::get_bitvm_key; +use crate::rpc_service::AppState; use crate::rpc_service::response::ErrorResponse; -use crate::rpc_service::{AppState, current_time_secs}; use axum::Json; use axum::body::{Body, to_bytes}; use axum::extract::{Request, State}; diff --git a/node/src/rpc_service/bitvm.rs b/node/src/rpc_service/bitvm.rs index 9e57ea82a..755f17e0a 100644 --- a/node/src/rpc_service/bitvm.rs +++ b/node/src/rpc_service/bitvm.rs @@ -1,8 +1,7 @@ use super::utils::{deserialize_u256, serialize_u256}; -use crate::rpc_service::current_time_secs; -use crate::scheduled_tasks::graph_maintenance_tasks::{ - ChallengeSubStatus, VerifierChallengeStatus, -}; +use crate::aggregates::graph_scan::{ChallengeSubStatus, VerifierChallengeStatus}; +use crate::aggregates::instance::PeginDeadlines; +use crate::clock::current_time_secs; use crate::utils::{check_bridge_in_uxto_available_or_self_spent, reflect_goat_address}; use alloy::hex::ToHexExt; use alloy::primitives::U256; @@ -24,11 +23,8 @@ use uuid::Uuid; const BRIDGE_IN_FAIL_AS_UTXO_BEEN_SPENT: &str = "Your UTXO has already been spent."; // const BRIDGE_IN_FAIL_AS_NO_ENOUGH_COMMITTEES: &str = "Unfortunately, no enough committee answered."; -const BRIDGE_IN_FAIL_AS_PRESIGNED_FAILED: &str = "Unfortunately, the presigned failed."; const BRIDGE_IN_FAIL_AS_VERIFICATION_FAILED: &str = "Unfortunately, the verification failed."; -const BRIDGE_IN_FAIL_AS_L2_MINTED_FAILED: &str = "Unfortunately, PegBTC minted failed."; - const BRIDGE_IN_FAIL_AS_TIMEOUT: &str = "The operation timed out. Please try again."; const _BRIDGE_IN_FAIL_AS_L1_LOCK_TIMEOUT: &str = "The operator timed out and failed to lock BTC. Please cancel the transaction."; @@ -200,6 +196,12 @@ pub struct InstanceExtended { pub confirmations: u32, pub target_confirmations: u32, pub status_extra: StatusExtra, + /// `Prepare height + T0`: from this Bitcoin height the user may broadcast Peg-In-Cancel. + /// `None` before Prepare confirmed. + pub cancel_from_height: Option, + /// `Prepare height + T0 + buffer`: from this height the peg-in is abandoned + /// unless Peg-In-Confirm is confirmed or visible in the mempool. + pub abandon_height: Option, } impl InstanceExtended { @@ -231,6 +233,9 @@ impl InstanceExtended { response_window_blocks, ); + let deadlines = PeginDeadlines::from_env(); + let cancel_from_height = deadlines.cancel_from_height(instance.btc_height); + let abandon_height = deadlines.abandon_height(instance.btc_height); // instance.status = instance.convert_to_display_status(); Ok(Self { waiting_time_in_secs, @@ -238,6 +243,8 @@ impl InstanceExtended { confirmations: 0, target_confirmations: 0, status_extra, + cancel_from_height, + abandon_height, utxo: utxos, instance: instance.into(), }) @@ -287,16 +294,6 @@ async fn get_instance_status_extra( status_extra.error = Some(BRIDGE_IN_FAIL_AS_VERIFICATION_FAILED.to_string()); status_extra.user_action = StatusUserAction::Cancel; } - InstanceBridgeInStatus::PresignedFailed => { - status_extra.is_failed = true; - status_extra.error = Some(BRIDGE_IN_FAIL_AS_PRESIGNED_FAILED.to_string()); - status_extra.user_action = StatusUserAction::None; - } - InstanceBridgeInStatus::RelayerL2MintedFailed => { - status_extra.is_failed = true; - status_extra.error = Some(BRIDGE_IN_FAIL_AS_L2_MINTED_FAILED.to_string()); - status_extra.user_action = StatusUserAction::None; - } InstanceBridgeInStatus::Timeout => { status_extra.is_failed = true; status_extra.error = Some(BRIDGE_IN_FAIL_AS_TIMEOUT.to_string()); @@ -330,7 +327,7 @@ fn get_bridge_in_status_time_window_secs(status: &str, response_window_blocks: i + INSTANCE_RELAYER_L1_BROADCAST_STATUS_DURATION_SECS; match InstanceBridgeInStatus::from_str(status) { - Ok(InstanceBridgeInStatus::UserIniting) | Ok(InstanceBridgeInStatus::UserInited) => { + Ok(InstanceBridgeInStatus::UserInited) => { (response_window_blocks_with_margin * GOAT_BLOCK_INTERVAL_SECS, total_time) } Ok(InstanceBridgeInStatus::CommitteesAnswered) => { @@ -720,9 +717,7 @@ impl DisplayStatusConvert for Instance { Ok(InstanceBridgeInStatus::RelayerL2Minted) => { InstanceDisplayStatus::Success.to_string() } - Ok(InstanceBridgeInStatus::PresignedFailed) - | Ok(InstanceBridgeInStatus::RelayerL2MintedFailed) - | Ok(InstanceBridgeInStatus::NoEnoughCommitteesAnswered) + Ok(InstanceBridgeInStatus::NoEnoughCommitteesAnswered) | Ok(InstanceBridgeInStatus::UserDiscarded) => { InstanceDisplayStatus::Failed.to_string() } @@ -755,8 +750,6 @@ impl DisplayStatusConvert for Instance { vec![InstanceBridgeInStatus::UserCanceled.to_string()] } Ok(InstanceDisplayStatus::Failed) => vec![ - InstanceBridgeInStatus::PresignedFailed.to_string(), - InstanceBridgeInStatus::RelayerL2MintedFailed.to_string(), InstanceBridgeInStatus::NoEnoughCommitteesAnswered.to_string(), InstanceBridgeInStatus::UserDiscarded.to_string(), ], diff --git a/node/src/rpc_service/handler/bitvm_handler.rs b/node/src/rpc_service/handler/bitvm_handler.rs index b216e2edf..599745d30 100644 --- a/node/src/rpc_service/handler/bitvm_handler.rs +++ b/node/src/rpc_service/handler/bitvm_handler.rs @@ -1,8 +1,8 @@ +#[cfg(feature = "rpc-debug-endpoints")] +use crate::aggregates::graph_pegout_jobs::broadcast_verifier_challenge_assert_tx; use crate::env::{ GraphBtcTxName, get_goat_gateway_contract_from_env, get_node_goat_address, get_node_pubkey, }; -#[cfg(feature = "rpc-debug-endpoints")] -use crate::handle::broadcast_verifier_challenge_assert_tx; use crate::rpc_service::AppState; use crate::rpc_service::bitvm::*; use crate::rpc_service::node::ALIVE_TIME_JUDGE_THRESHOLD; @@ -308,7 +308,7 @@ pub async fn get_instance( ok_response(InstanceGetResponse { instance_wrap }) } else { - tracing::info!("instance_id {} has no record in database", instance_id); + tracing::debug!("instance_id {} has no record in database", instance_id); ok_response(InstanceGetResponse { instance_wrap: None }) } } @@ -481,7 +481,7 @@ pub async fn get_graph( ok_response(graph_extended) } else { - tracing::warn!("graph:{} is not record in db", graph_id); + tracing::debug!("graph:{} is not record in db", graph_id); ok_response(GraphExtended::default()) } } @@ -875,7 +875,7 @@ pub async fn get_graph_tx( btc_tx_data: BtcTxData { raw_data, progresses, fail_reason }, }) } else { - tracing::warn!("graph:{} is not record in db", graph_id); + tracing::debug!("graph:{} is not record in db", graph_id); Err(( StatusCode::INTERNAL_SERVER_ERROR, Json(ErrorResponse { @@ -1176,7 +1176,6 @@ pub async fn get_unsigned_pegin_txn( InstanceBridgeInStatus::CommitteesAnswered.to_string(), InstanceBridgeInStatus::UserBroadcastPeginPrepare.to_string(), InstanceBridgeInStatus::Presigned.to_string(), - InstanceBridgeInStatus::PresignedFailed.to_string(), InstanceBridgeInStatus::Timeout.to_string(), ] .contains(&instance.status) diff --git a/node/src/rpc_service/handler/debug_handler.rs b/node/src/rpc_service/handler/debug_handler.rs index 8a246b9a4..0111de3ec 100644 --- a/node/src/rpc_service/handler/debug_handler.rs +++ b/node/src/rpc_service/handler/debug_handler.rs @@ -1,12 +1,11 @@ +use crate::clock::current_time_secs; +use crate::rpc_service::AppState; use crate::rpc_service::response::{ApiErrorExt, ApiResult, ErrorResponse, ok_response}; -use crate::rpc_service::validation::InputValidator; -use crate::rpc_service::{AppState, current_time_secs}; use axum::Json; -use axum::extract::{Path, Query, State}; +use axum::extract::State; use http::StatusCode; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use store::{Graph, GraphStatus, MessageDebugOverview}; #[derive(Serialize)] pub struct DebugStatusResponse { @@ -16,164 +15,8 @@ pub struct DebugStatusResponse { pub bitcoin_height: Option, pub goat_height: Option, pub goat_spv_height: Option, - pub message_queue: DebugMessageQueue, -} - -#[derive(Serialize)] -pub struct DebugMessageQueue { - pub pending_ready: i64, - pub pending_locked: i64, - pub failed: i64, - pub oldest_pending_at: Option, -} - -#[derive(Serialize)] -pub struct GraphDebugMessagesResponse { - pub graph_id: String, - pub status: Option, - pub flow: String, - pub messages: Vec, -} - -#[derive(Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum GraphMessageFlow { - Pegin, - Pegout, -} - -#[derive(Default, Deserialize)] -pub struct GraphDebugMessagesQuery { - /// Optional message-flow filter: `pegin` or `pegout`. - pub flow: Option, -} - -impl GraphMessageFlow { - fn matches_message_type(&self, message_type: &str) -> bool { - match self { - Self::Pegin => matches!( - message_type, - "CreateGraph" - | "InitGraph" - | "GenCircuits" - | "CutCircuits" - | "SolderingProofReady" - | "VerifierGraphParamsEndorsement" - | "NonceGeneration" - | "CommitteePresign" - | "EndorseGraph" - | "GraphFinalize" - ), - Self::Pegout => matches!( - message_type, - "KickoffReady" - | "KickoffSent" - | "PreKickoffSent" - | "ChallengeSent" - | "WatchtowerChallengeInitSent" - | "WatchtowerChallengeSent" - | "WatchtowerChallengeTimeout" - | "NackReady" - | "OperatorCommitPubinReady" - | "OperatorCommitPubinTimeout" - | "AssertReady" - | "AssertSent" - | "ChallengeAssertSent" - | "WronglyChallengeTimeout" - | "DisproveSent" - | "Take1Ready" - | "Take1Sent" - | "Take2Ready" - | "Take2Sent" - ), - } - } -} - -#[derive(Serialize)] -pub struct InstanceDebugMessagesResponse { - pub instance_id: String, - pub messages: Vec, -} - -#[derive(Serialize)] -pub struct GraphDebugMessageOverview { - pub message_id: String, - pub actor: String, - pub message_type: String, - pub state: String, - pub lock_time_until: i64, - pub created_at: i64, - pub updated_at: i64, - pub reason_count: i64, - pub last_reason: Option, -} - -#[derive(Serialize)] -pub struct DebugMessageDetailsResponse { - pub message: GraphDebugMessageOverview, - pub reasons: Vec, -} - -#[derive(Serialize)] -pub struct GraphDebugReasonSummary { - pub code: String, - pub detail: String, - pub last_seen_at: i64, -} - -#[derive(Serialize)] -pub struct GraphDebugReason { - pub code: String, - pub detail: String, - pub first_seen_at: i64, - pub last_seen_at: i64, - pub occurrences: i64, -} - -fn graph_debug_message_overview(message: MessageDebugOverview) -> GraphDebugMessageOverview { - GraphDebugMessageOverview { - message_id: message.message_id, - actor: message.actor, - message_type: message.msg_type, - state: message.state, - lock_time_until: message.lock_time_until, - created_at: message.created_at, - updated_at: message.updated_at, - reason_count: message.reason_count, - last_reason: match ( - message.last_reason_code, - message.last_reason_detail, - message.last_reason_seen_at, - ) { - (Some(code), Some(detail), Some(last_seen_at)) => { - Some(GraphDebugReasonSummary { code, detail, last_seen_at }) - } - _ => None, - }, - } -} - -fn graph_debug_flow(graph: Option<&Graph>) -> &'static str { - let Some(graph) = graph else { - return "unknown"; - }; - if graph.init_withdraw_tx_hash.is_some() - || graph.bridge_out_start_at > 0 - || matches!( - graph.status.parse::(), - Ok(GraphStatus::PreKickoff - | GraphStatus::OperatorKickOff - | GraphStatus::Challenge - | GraphStatus::Disprove - | GraphStatus::OperatorTake1 - | GraphStatus::OperatorTake2) - ) - { - "pegout" - } else { - "pegin" - } + /// Execution engine rows by table and state; `None` without an engine. + pub engine: Option>, } // TODO(auth): Require operator authentication before exposing local debug state. @@ -187,12 +30,10 @@ pub async fn get_debug_status( app_state.goat_client.get_latest_block_number(), app_state.goat_client.btc_spv_latest_height(), ); - let mut storage_processor = - app_state.local_db.acquire().await.api_error("GET_DEBUG_STATUS_ERROR")?; - let queue = storage_processor - .get_message_queue_stats(&app_state.actor.to_string(), checked_at) - .await - .api_error("GET_DEBUG_STATUS_ERROR")?; + let engine = match app_state.engine.as_ref() { + Some(engine) => Some(engine.state_counts().await.api_error("GET_DEBUG_STATUS_ERROR")?), + None => None, + }; ok_response(DebugStatusResponse { checked_at, @@ -201,107 +42,63 @@ pub async fn get_debug_status( bitcoin_height: bitcoin_height.ok(), goat_height: goat_height.ok(), goat_spv_height: goat_spv_height.ok(), - message_queue: DebugMessageQueue { - pending_ready: queue.pending_ready, - pending_locked: queue.pending_locked, - failed: queue.failed, - oldest_pending_at: queue.oldest_pending_at, - }, + engine, }) } // TODO(auth): Require operator authentication before exposing local debug state. +/// Execution engine snapshot: row counts by state, aggregates parked on a +/// `waiting_for`, and poisoned aggregates. #[axum::debug_handler] -pub async fn get_graph_debug_messages( - Path(graph_id): Path, - Query(query): Query, +pub async fn get_debug_engine( State(app_state): State>, -) -> ApiResult { - let graph_id = InputValidator::validate_uuid(&graph_id, "graph_id")?; - let mut storage_processor = - app_state.local_db.acquire().await.api_error("GET_GRAPH_DEBUG_MESSAGES_ERROR")?; - let graph = storage_processor - .find_graph(&graph_id) - .await - .api_error("GET_GRAPH_DEBUG_MESSAGES_ERROR")?; - let flow = graph_debug_flow(graph.as_ref()).to_string(); - let status = graph.map(|graph| graph.status); - let messages = storage_processor - .find_message_debug_overviews(&graph_id) - .await - .api_error("GET_GRAPH_DEBUG_MESSAGES_ERROR")? - .into_iter() - .filter(|message| { - query.flow.as_ref().is_none_or(|flow| flow.matches_message_type(&message.msg_type)) - }) - .map(graph_debug_message_overview) - .collect(); - - ok_response(GraphDebugMessagesResponse { - graph_id: graph_id.to_string(), - status, - flow, - messages, - }) +) -> ApiResult { + let Some(engine) = app_state.engine.as_ref() else { + return Err(( + StatusCode::NOT_FOUND, + Json(ErrorResponse { + error: "ENGINE_UNAVAILABLE".to_string(), + message: "the execution engine handle is not available on this node".to_string(), + }), + )); + }; + let snapshot = engine.snapshot().await.api_error("GET_DEBUG_ENGINE_ERROR")?; + ok_response(snapshot) } -// TODO(auth): Require operator authentication before exposing local debug state. -#[axum::debug_handler] -pub async fn get_instance_debug_messages( - Path(instance_id): Path, - State(app_state): State>, -) -> ApiResult { - let instance_id = InputValidator::validate_uuid(&instance_id, "instance_id")?; - let mut storage_processor = - app_state.local_db.acquire().await.api_error("GET_INSTANCE_DEBUG_MESSAGES_ERROR")?; - let messages = storage_processor - .find_message_debug_overviews(&instance_id) - .await - .api_error("GET_INSTANCE_DEBUG_MESSAGES_ERROR")? - .into_iter() - .map(graph_debug_message_overview) - .collect(); +#[derive(Deserialize)] +pub struct ClearPoisonRequest { + pub aggregate_kind: String, + pub aggregate_id: String, + /// One poisoned action; every action of the aggregate when omitted. + pub action: Option, +} - ok_response(InstanceDebugMessagesResponse { instance_id: instance_id.to_string(), messages }) +#[derive(Serialize)] +pub struct ClearPoisonResponse { + pub cleared: u64, } -// TODO(auth): Require operator authentication before exposing local debug state. +/// Manual release of poisoned actions: the held-back jobs and events run again +/// at once and the backoff count starts over. #[axum::debug_handler] -pub async fn get_debug_message_details( - Path(message_id): Path, +pub async fn clear_engine_poison( State(app_state): State>, -) -> ApiResult { - let mut storage_processor = - app_state.local_db.acquire().await.api_error("GET_DEBUG_MESSAGE_DETAILS_ERROR")?; - let message = storage_processor - .find_message_debug_overview(&message_id) - .await - .api_error("GET_DEBUG_MESSAGE_DETAILS_ERROR")? - .ok_or_else(|| { - ( - StatusCode::NOT_FOUND, - Json(ErrorResponse { - error: "DEBUG_MESSAGE_NOT_FOUND".to_string(), - message: format!("message {message_id} not found"), - }), - ) - })?; - let reasons = storage_processor - .find_message_debug_reasons(&message_id) + Json(request): Json, +) -> ApiResult { + let Some(engine) = app_state.engine.as_ref() else { + return Err(( + StatusCode::NOT_FOUND, + Json(ErrorResponse { + error: "ENGINE_UNAVAILABLE".to_string(), + message: "the execution engine handle is not available on this node".to_string(), + }), + )); + }; + let aggregate = engine::AggregateRef::new(&request.aggregate_kind, &request.aggregate_id); + let cleared = engine + .clear_poison(&aggregate, request.action.as_deref()) .await - .api_error("GET_DEBUG_MESSAGE_DETAILS_ERROR")? - .into_iter() - .map(|reason| GraphDebugReason { - code: reason.reason_code, - detail: reason.reason_detail, - first_seen_at: reason.first_seen_at, - last_seen_at: reason.last_seen_at, - occurrences: reason.occurrences, - }) - .collect(); - - ok_response(DebugMessageDetailsResponse { - message: graph_debug_message_overview(message), - reasons, - }) + .api_error("CLEAR_ENGINE_POISON_ERROR")?; + ok_response(ClearPoisonResponse { cleared }) } diff --git a/node/src/rpc_service/handler/health_handler.rs b/node/src/rpc_service/handler/health_handler.rs new file mode 100644 index 000000000..b5b56ed69 --- /dev/null +++ b/node/src/rpc_service/handler/health_handler.rs @@ -0,0 +1,54 @@ +use crate::clock::current_time_secs; +use crate::metrics_service::{BackendHealthSnapshot, EventWatchSnapshot, ReadinessSnapshot}; +use crate::rpc_service::AppState; +use axum::Json; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use serde::Serialize; +use std::sync::Arc; +use std::time::Duration; + +const DB_PING_TIMEOUT: Duration = Duration::from_secs(2); + +/// `GET /v1/health`: the node's own view of its health, `200` when ready and +/// `503` otherwise. Everything is read from memory except a local database +/// ping, so the database bit does not depend on a metrics scrape. +#[derive(Serialize)] +pub struct HealthResponse { + pub ready: bool, + pub components: ReadinessSnapshot, + pub backends: Vec, + pub event_watch: EventWatchSnapshot, + pub spv_lag_blocks: i64, + pub uptime_secs: u64, + pub version: &'static str, + pub role: String, + pub peer_id: String, + pub checked_at: i64, +} + +#[axum::debug_handler] +pub async fn get_health(State(app_state): State>) -> impl IntoResponse { + let metrics = &app_state.metrics_state; + let database_ok = tokio::time::timeout(DB_PING_TIMEOUT, app_state.local_db.conn.acquire()) + .await + .map(|conn| conn.is_ok()) + .unwrap_or(false); + metrics.mark_database_ready(database_ok); + let components = metrics.readiness_snapshot(); + let body = HealthResponse { + ready: components.ready, + components: components.clone(), + backends: metrics.backend_health_snapshot(), + event_watch: metrics.event_watch_snapshot(), + spv_lag_blocks: metrics.spv_lag_blocks(), + uptime_secs: metrics.uptime_secs(), + version: env!("CARGO_PKG_VERSION"), + role: app_state.actor.to_string(), + peer_id: app_state.peer_id.clone(), + checked_at: current_time_secs(), + }; + let status = if components.ready { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; + (status, Json(body)) +} diff --git a/node/src/rpc_service/handler/mod.rs b/node/src/rpc_service/handler/mod.rs index 81da5ae97..524942864 100644 --- a/node/src/rpc_service/handler/mod.rs +++ b/node/src/rpc_service/handler/mod.rs @@ -1,6 +1,7 @@ pub mod bitvm_handler; #[cfg(feature = "rpc-debug-endpoints")] pub mod debug_handler; +pub mod health_handler; pub mod node_handler; pub mod proof_handler; pub mod swap_handler; @@ -9,6 +10,7 @@ pub mod swap_handler; pub use bitvm_handler::*; #[cfg(feature = "rpc-debug-endpoints")] pub use debug_handler::*; +pub use health_handler::*; pub use node_handler::*; pub use proof_handler::*; pub use swap_handler::*; diff --git a/node/src/rpc_service/handler/node_handler.rs b/node/src/rpc_service/handler/node_handler.rs index 40d7fb30f..5f51125e5 100644 --- a/node/src/rpc_service/handler/node_handler.rs +++ b/node/src/rpc_service/handler/node_handler.rs @@ -1,10 +1,11 @@ +use crate::clock::current_time_secs; +use crate::rpc_service::AppState; use crate::rpc_service::node::{ ALIVE_TIME_JUDGE_THRESHOLD, NODE_STATUS_OFFLINE, NodeDesc, NodeListResponse, NodeOverViewResponse, NodeQueryParams, ToNodeDesc, }; use crate::rpc_service::response::{ApiErrorExt, ApiResult, ok_response}; use crate::rpc_service::validation::InputValidator; -use crate::rpc_service::{AppState, current_time_secs}; use crate::utils::reflect_goat_address; use axum::extract::{Path, Query, State}; use bitvm_lib::actors::Actor; diff --git a/node/src/rpc_service/mod.rs b/node/src/rpc_service/mod.rs index 39b947690..39faf9ed6 100644 --- a/node/src/rpc_service/mod.rs +++ b/node/src/rpc_service/mod.rs @@ -6,6 +6,8 @@ mod node; mod response; pub mod routes; mod swap; +#[cfg(test)] +mod tests; pub(super) mod utils; pub mod validation; @@ -13,16 +15,16 @@ use crate::env::{get_btc_url_from_env, get_goat_network, get_network, goat_confi use crate::metrics_service::{MetricsState, metrics_handler, metrics_middleware}; use crate::rpc_service::auth::require_request_auth; use crate::rpc_service::cors_config::CorsConfig; +#[cfg(feature = "rpc-debug-endpoints")] use crate::rpc_service::handler::{ - get_chain_proof_desc, get_graph, get_graph_neighbor_ids, get_graph_tx, get_graph_txn, - get_graphs, get_instance, get_instances, get_instances_overview, get_node, get_nodes, - get_nodes_overview, get_operator_proof_desc, get_ready_to_kickoff_graph, get_swap, get_swaps, - get_unsigned_pegin_txn, instance_settings, pegout, + clear_engine_poison, get_debug_engine, get_debug_status, send_challenge, + send_verifier_challenge, }; -#[cfg(feature = "rpc-debug-endpoints")] use crate::rpc_service::handler::{ - get_debug_message_details, get_debug_status, get_graph_debug_messages, - get_instance_debug_messages, send_challenge, send_verifier_challenge, + get_chain_proof_desc, get_graph, get_graph_neighbor_ids, get_graph_tx, get_graph_txn, + get_graphs, get_health, get_instance, get_instances, get_instances_overview, get_node, + get_nodes, get_nodes_overview, get_operator_proof_desc, get_ready_to_kickoff_graph, get_swap, + get_swaps, get_unsigned_pegin_txn, instance_settings, pegout, }; use anyhow::Context; use axum::body::Body; @@ -38,7 +40,7 @@ use client::goat_chain::GOATClient; use client::http_client::async_client::HttpAsyncClient; use std::collections::HashMap; use std::sync::{Arc, Mutex}; -use std::time::{Duration, UNIX_EPOCH}; +use std::time::Duration; use store::localdb::LocalDB; use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; @@ -46,11 +48,6 @@ use tower_http::classify::ServerErrorsFailureClass; use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; -#[inline(always)] -pub fn current_time_secs() -> i64 { - std::time::SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64 -} - /// Create secure CORS layer pub fn create_secure_cors_layer() -> CorsLayer { let cors_config = CorsConfig::from_env(); @@ -72,6 +69,8 @@ pub struct AppState { pub metrics_state: MetricsState, pub actor: Actor, pub peer_id: String, + /// Present when the execution engine is enabled (debug and status endpoints). + pub engine: Option, rpc_auth_nonces: Mutex>, } @@ -81,6 +80,7 @@ impl AppState { actor: Actor, peer_id: String, metrics_state: MetricsState, + engine: Option, ) -> anyhow::Result> { let btc_client = BTCClient::new(get_network(), get_btc_url_from_env().as_deref()); let goat_client = GOATClient::new(goat_config_from_env().await, get_goat_network()); @@ -92,6 +92,7 @@ impl AppState { metrics_state, actor, peer_id, + engine, http_client, rpc_auth_nonces: Mutex::new(HashMap::new()), })) @@ -117,6 +118,7 @@ impl AppState { metrics_state, actor, peer_id, + engine: None, http_client: HttpAsyncClient::new(None), rpc_auth_nonces: Mutex::new(HashMap::new()), })) @@ -146,6 +148,7 @@ async fn root() -> &'static str { pub(crate) fn build_business_router(app_state: Arc) -> Router { let public_routes = Router::new() .route(routes::ROOT, get(root)) + .route(routes::v1::HEALTH, get(get_health)) .route(routes::v1::NODES_BASE, get(get_nodes)) .route(routes::v1::NODES_BY_ID, get(get_node)) .route(routes::v1::NODES_OVERVIEW, get(get_nodes_overview)) @@ -168,16 +171,15 @@ pub(crate) fn build_business_router(app_state: Arc) -> Router { #[cfg(feature = "rpc-debug-endpoints")] let public_routes = public_routes .route(routes::v1::DEBUG_STATUS, get(get_debug_status)) - .route(routes::v1::DEBUG_GRAPH_MESSAGES, get(get_graph_debug_messages)) - .route(routes::v1::DEBUG_INSTANCE_MESSAGES, get(get_instance_debug_messages)) - .route(routes::v1::DEBUG_MESSAGE_DETAILS, get(get_debug_message_details)); + .route(routes::v1::DEBUG_ENGINE, get(get_debug_engine)); let signed_routes = Router::new().route(routes::v1::PEGOUT, post(pegout)); #[cfg(feature = "rpc-debug-endpoints")] let signed_routes = signed_routes .route(routes::v1::GRAPHS_SEND_CHALLENGE, post(send_challenge)) - .route(routes::v1::GRAPHS_SEND_VERIFIER_CHALLENGE, post(send_verifier_challenge)); + .route(routes::v1::GRAPHS_SEND_VERIFIER_CHALLENGE, post(send_verifier_challenge)) + .route(routes::v1::DEBUG_ENGINE_CLEAR_POISON, post(clear_engine_poison)); let signed_routes = signed_routes .route_layer(middleware::from_fn_with_state(app_state.clone(), require_request_auth)); @@ -271,18 +273,6 @@ async fn serve_with_listener( } } -pub async fn serve( - addr: String, - local_db: LocalDB, - actor: Actor, - peer_id: String, - metrics_state: MetricsState, - cancellation_token: CancellationToken, -) -> anyhow::Result { - let app_state = AppState::create_arc_app_state(local_db, actor, peer_id, metrics_state).await?; - serve_with_app_state(addr, app_state, cancellation_token).await -} - /// Validates the path the dedicated metrics listener will expose. /// /// Rejects anything `axum` cannot register as a literal route, so a bad @@ -352,853 +342,3 @@ pub async fn serve_metrics( } } } - -#[cfg(test)] -mod tests { - use crate::env::{ - ENV_BITVM_SECRET, ENV_GOAT_CHAIN_URL, ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, - ENV_PROOF_SEVER_URL, get_network, - }; - use crate::metrics_service::MetricsState; - use crate::rpc_service::auth::{ - AUTH_NONCE_HEADER, AUTH_SIGNATURE_HEADER, AUTH_TIMESTAMP_HEADER, sign_request_auth, - }; - use crate::rpc_service::bitvm::{ - BRIDGE_IN_AMOUNTS, GraphGetResponse, GraphListResponse, InstanceGetResponse, - InstanceListResponse, InstanceOverviewResponse, InstanceSettingResponse, - }; - use crate::rpc_service::node::{NodeListResponse, NodeOverViewResponse}; - use crate::rpc_service::{self, Actor, AppState, current_time_secs, routes}; - use crate::utils::{ - generate_local_key, generate_random_bytes, get_rand_btc_address_p2wpkh, - get_rand_goat_address, temp_sqlite_db_path, - }; - use alloy::primitives::U256; - use client::Utxo; - use http::Method; - use prometheus_client::registry::Registry; - use reqwest::Client; - use secp256k1::{Keypair, SECP256K1, Secp256k1}; - use serde_json::Value; - use std::str::FromStr; - use std::sync::{Arc, Mutex}; - use std::time::Duration; - use store::localdb::{GraphRuntimeUpdate, LocalDB, StorageProcessor}; - use store::{ - Graph, GraphStatus, GraphStatusSource, Instance, InstanceBridgeInStatus, Node, - create_local_db, - }; - use tokio::{net::TcpListener, time::sleep}; - use tokio_util::sync::CancellationToken; - use tracing::{error, info}; - use tracing_subscriber::EnvFilter; - use uuid::Uuid; - - struct ApiTestItem { - pub tag: String, - pub url: String, - pub json_payload: Option, - pub method: Method, - pub expe_res: bool, - pub resp_validation: Option bool>>, - } - - impl ApiTestItem { - async fn do_test(&self, client: &Client) -> anyhow::Result<()> { - eprintln!("Start test api: {}", self.tag); - info!("Start test api: {}", self.tag); - let mut request_builder = match self.method { - Method::POST => client.post(self.url.clone()), - Method::GET => client.get(self.url.clone()), - Method::PUT => client.put(self.url.clone()), - Method::DELETE => client.delete(self.url.clone()), - _ => { - anyhow::bail!("wrong method"); - } - }; - if let Some(json_payload) = self.json_payload.clone() { - request_builder = request_builder.json(&json_payload); - } - - let resp = request_builder.send().await?; - let actual_status = resp.status().is_success(); - if actual_status != self.expe_res { - let data = resp.text().await?; - eprintln!( - "Test api '{}' failed: expected status {}, got {}, resp: {data}", - self.tag, self.expe_res, actual_status - ); - error!( - "Test api '{}' failed: expected status {}, got {}, resp: {data}", - self.tag, self.expe_res, actual_status - ); - anyhow::bail!( - "Test failed: expected status {}, got {}", - self.expe_res, - actual_status - ); - } - - if let Some(validate_fn) = &self.resp_validation { - let text = resp.text().await?; - if !validate_fn(text) { - eprintln!("Test api '{}' fail validate_fn", self.tag); - error!("Test api '{}' fail validate_fn", self.tag); - anyhow::bail!("Test api '{}' fail validate_fn", self.tag); - } - } - - Ok(()) - } - } - - async fn do_batch_tests( - batch_tag: &str, - client: &Client, - items: &[ApiTestItem], - ) -> anyhow::Result<()> { - info!("Start batch test:{batch_tag}"); - for item in items { - item.do_test(client).await?; - } - info!("Finish batch test:{batch_tag}"); - Ok(()) - } - fn init(remote_proof_server: Option) { - unsafe { - std::env::set_var("RUST_LOG", "info"); - std::env::set_var(ENV_GOAT_CHAIN_URL, "https://rpc.testnet3.goat.network"); - std::env::set_var( - ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, - "0x21f619040AC2eAcacEF8Fe17Ae8bDF53ec69C66f", - ); - if let Some(remote_proof_server) = remote_proof_server { - std::env::set_var(ENV_PROOF_SEVER_URL, remote_proof_server); - } - } - let _ = tracing_subscriber::fmt().with_env_filter(EnvFilter::from_default_env()).try_init(); - } - - async fn spawn_metrics_listener( - app_state: Arc, - metrics_path: &str, - cancellation_token: CancellationToken, - ) -> anyhow::Result<(String, tokio::task::JoinHandle>)> { - let listener = rpc_service::bind_metrics_listener("127.0.0.1:0").await?; - let addr = listener.local_addr()?.to_string(); - let metrics_path = metrics_path.to_string(); - let handle = tokio::spawn(rpc_service::serve_metrics( - listener, - metrics_path, - app_state, - cancellation_token, - )); - Ok((addr, handle)) - } - - async fn mock_app_state() -> anyhow::Result> { - let local_db = create_local_db(&temp_sqlite_db_path()).await; - let metrics_state = MetricsState::new(Arc::new(Mutex::new(Registry::default()))); - rpc_service::AppState::create_arc_mock_app_state( - local_db, - Actor::Verifier, - generate_local_key().public().to_peer_id().to_string(), - metrics_state, - ) - .await - } - - fn set_test_auth_key() -> Keypair { - let keypair = Keypair::from_seckey_slice(SECP256K1, &[0x42; 32]).unwrap(); - unsafe { - std::env::set_var(ENV_BITVM_SECRET, hex::encode(keypair.secret_key().secret_bytes())) - }; - keypair - } - - async fn spawn_business_listener( - cancellation_token: CancellationToken, - ) -> anyhow::Result<(String, tokio::task::JoinHandle<()>)> { - let app_state = mock_app_state().await?; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?.to_string(); - let server = rpc_service::build_business_router(app_state); - let handle = tokio::spawn(async move { - let _ = tokio::select! { - result = axum::serve(listener, server) => result, - _ = cancellation_token.cancelled() => Ok(()), - }; - }); - Ok((addr, handle)) - } - - #[tokio::test(flavor = "multi_thread")] - async fn signed_request_nonce_is_rejected_after_first_use() - -> Result<(), Box> { - let keypair = set_test_auth_key(); - let cancellation_token = CancellationToken::new(); - let (addr, server) = spawn_business_listener(cancellation_token.clone()).await?; - let request_target = routes::v1::PEGOUT.to_owned(); - let url = format!("http://{addr}{request_target}"); - let (timestamp, nonce, signature) = - sign_request_auth(&keypair, &Method::POST, &request_target, &[]); - let client = Client::new(); - - let first = client - .post(&url) - .header(AUTH_TIMESTAMP_HEADER, ×tamp) - .header(AUTH_NONCE_HEADER, &nonce) - .header(AUTH_SIGNATURE_HEADER, &signature) - .send() - .await?; - assert_ne!(first.status().as_u16(), 409); - - let replay = client - .post(&url) - .header(AUTH_TIMESTAMP_HEADER, ×tamp) - .header(AUTH_NONCE_HEADER, &nonce) - .header(AUTH_SIGNATURE_HEADER, &signature) - .send() - .await?; - assert_eq!(replay.status().as_u16(), 409); - let replay_body: Value = replay.json().await?; - assert_eq!(replay_body["error"], "AUTH_REPLAY"); - - cancellation_token.cancel(); - server.await?; - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn debug_routes_follow_rpc_debug_endpoints_feature() - -> Result<(), Box> { - let cancellation_token = CancellationToken::new(); - let (addr, server) = spawn_business_listener(cancellation_token.clone()).await?; - let client = Client::new(); - let id = Uuid::new_v4(); - - let status = client.get(format!("http://{addr}/v1/debug/status")).send().await?; - let graph_messages = - client.get(format!("http://{addr}/v1/debug/graphs/{id}/messages")).send().await?; - let instance_messages = - client.get(format!("http://{addr}/v1/debug/instances/{id}/messages")).send().await?; - let message_details = - client.get(format!("http://{addr}/v1/debug/messages/missing")).send().await?; - let verifier_challenge = client - .post(format!("http://{addr}/v1/graphs/{id}/send-verifier-challenge")) - .send() - .await?; - - if cfg!(feature = "rpc-debug-endpoints") { - assert_eq!(status.status().as_u16(), 200); - assert_eq!(graph_messages.status().as_u16(), 200); - assert_eq!(instance_messages.status().as_u16(), 200); - assert_eq!(message_details.status().as_u16(), 404); - let message_body: Value = message_details.json().await?; - assert_eq!(message_body["error"], "DEBUG_MESSAGE_NOT_FOUND"); - assert_eq!(verifier_challenge.status().as_u16(), 401); - } else { - assert_eq!(status.status().as_u16(), 404); - assert_eq!(graph_messages.status().as_u16(), 404); - assert_eq!(instance_messages.status().as_u16(), 404); - assert_eq!(message_details.status().as_u16(), 404); - assert_eq!(verifier_challenge.status().as_u16(), 404); - } - - cancellation_token.cancel(); - server.await?; - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn metrics_are_served_only_by_the_dedicated_listener() - -> Result<(), Box> { - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?.to_string(); - let app_state = mock_app_state().await?; - let cancellation_token = CancellationToken::new(); - let server = tokio::spawn(rpc_service::serve_with_listener( - listener, - app_state.clone(), - cancellation_token.clone(), - )); - let (metrics_addr, metrics_server) = - spawn_metrics_listener(app_state, "/metrics", cancellation_token.clone()).await?; - sleep(Duration::from_millis(100)).await; - - let client = Client::new(); - client.get(format!("http://{addr}/")).send().await?.error_for_status()?; - let unmatched = client.get(format!("http://{addr}/missing")).send().await?; - assert_eq!(unmatched.status().as_u16(), 404); - - // The business listener no longer exposes metrics at all. - let business_metrics = client.get(format!("http://{addr}/metrics")).send().await?; - assert_eq!(business_metrics.status().as_u16(), 404); - - // The metrics listener exposes nothing but the metrics path. - for path in ["/", "/v1/nodes", "/v1/debug/status"] { - let response = client.get(format!("http://{metrics_addr}{path}")).send().await?; - assert_eq!( - response.status().as_u16(), - 404, - "unexpected route on metrics listener: {path}" - ); - } - - let first = client.get(format!("http://{metrics_addr}/metrics")).send().await?; - assert_eq!(first.status().as_u16(), 200); - assert_eq!( - first.headers().get(http::header::CONTENT_TYPE).unwrap(), - "application/openmetrics-text;charset=utf-8;version=1.0.0" - ); - let first_scrape = first.text().await?; - let second_scrape = - client.get(format!("http://{metrics_addr}/metrics")).send().await?.text().await?; - - assert!( - first_scrape - .contains("http_requests_total{method=\"GET\",route=\"/\",status=\"200\"} 1") - ); - // `/missing` and the rejected `/metrics` request on the business listener. - assert!( - first_scrape.contains( - "http_requests_total{method=\"GET\",route=\"unmatched\",status=\"404\"} 2" - ) - ); - assert!(first_scrape.contains("http_requests_in_flight 0")); - // Requests served by the metrics listener are not counted at all. - assert!(!first_scrape.contains("route=\"/metrics\"")); - assert_eq!(first_scrape, second_scrape); - - cancellation_token.cancel(); - server.await??; - assert_eq!(metrics_server.await??, "metrics_shutdown"); - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn metrics_listener_honors_custom_path() -> Result<(), Box> { - let app_state = mock_app_state().await?; - let cancellation_token = CancellationToken::new(); - let (metrics_addr, metrics_server) = - spawn_metrics_listener(app_state, "/internal/metrics", cancellation_token.clone()) - .await?; - sleep(Duration::from_millis(100)).await; - - let client = Client::new(); - let response = client.get(format!("http://{metrics_addr}/internal/metrics")).send().await?; - assert_eq!(response.status().as_u16(), 200); - assert!(response.text().await?.contains("http_requests_in_flight")); - let default_path = client.get(format!("http://{metrics_addr}/metrics")).send().await?; - assert_eq!(default_path.status().as_u16(), 404); - - cancellation_token.cancel(); - metrics_server.await??; - Ok(()) - } - - #[tokio::test] - async fn metrics_listener_reports_bind_conflicts() -> Result<(), Box> { - let occupied = std::net::TcpListener::bind("127.0.0.1:0")?; - let addr = occupied.local_addr()?.to_string(); - let error = rpc_service::bind_metrics_listener(&addr).await.unwrap_err(); - assert!( - error.to_string().contains(&format!("failed to bind metrics listener to {addr}")), - "unexpected error: {error}" - ); - Ok(()) - } - - #[test] - fn metrics_path_validation_rejects_unusable_paths() { - for path in ["/metrics", "/", "/internal/metrics"] { - assert!( - rpc_service::validate_metrics_path(path).is_ok(), - "rejected valid path: {path}" - ); - } - for path in ["metrics", "", "/metrics?scrape=1", "/metrics#frag", "/{id}", "/met rics"] { - assert!( - rpc_service::validate_metrics_path(path).is_err(), - "accepted invalid path: {path}" - ); - } - } - - async fn init_nodes_data(local_db: &LocalDB, nodes: &[Node]) -> anyhow::Result<()> { - let mut tx = local_db.start_transaction().await?; - for node in nodes { - tx.upsert_node(node).await?; - } - tx.commit().await?; - Ok(()) - } - - async fn seed_graph_runtime( - tx: &mut StorageProcessor<'_>, - graph: &Graph, - ) -> anyhow::Result<()> { - let target_status = GraphStatus::from_str(&graph.status)?; - let sub_status = graph.sub_status.clone(); - let challenge_txid = graph.challenge_txid.clone(); - let init_withdraw_tx_hash = graph.init_withdraw_tx_hash.clone(); - let bridge_out_start_at = graph.bridge_out_start_at; - let proceed_withdraw_height = graph.proceed_withdraw_height; - - let mut definition = graph.clone(); - definition.status = GraphStatus::OperatorPresigned.to_string(); - definition.sub_status.clear(); - definition.challenge_txid = None; - definition.init_withdraw_tx_hash = None; - definition.bridge_out_start_at = 0; - definition.proceed_withdraw_height = 0; - tx.upsert_graph_definition(&definition).await?; - - if target_status != GraphStatus::OperatorPresigned { - tx.transition_graph_status( - graph.instance_id, - graph.graph_id, - target_status, - GraphStatusSource::ChainReconcile, - (!sub_status.is_empty()).then_some(sub_status), - ) - .await?; - } - - let mut runtime = GraphRuntimeUpdate::new(graph.instance_id, graph.graph_id); - if let Some(challenge_txid) = challenge_txid { - runtime = runtime.with_challenge_txid(challenge_txid); - } - if let Some(init_withdraw_tx_hash) = init_withdraw_tx_hash { - runtime = runtime.with_init_withdraw_tx_hash(init_withdraw_tx_hash); - } - if bridge_out_start_at != 0 { - runtime = runtime.with_bridge_out_start_at(bridge_out_start_at); - } - if proceed_withdraw_height != 0 { - runtime = runtime.with_proceed_withdraw_height(proceed_withdraw_height); - } - tx.update_graph_runtime(&runtime).await?; - Ok(()) - } - - async fn init_instance_graph_data( - local_db: &LocalDB, - instances: &[Instance], - graphs: &[Graph], - ) -> anyhow::Result<()> { - let mut tx = local_db.start_transaction().await?; - for instance in instances { - tx.insert_instance_if_absent(instance).await?; - } - for graph in graphs { - seed_graph_runtime(&mut tx, graph).await?; - } - tx.commit().await?; - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_nodes_api() -> Result<(), Box> { - init(None); - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?.to_string(); - let mut nodes = Vec::::new(); - let (_, public_key) = Secp256k1::new().generate_keypair(&mut rand::thread_rng()); - let pub_key = public_key.to_string(); - let actor = Actor::Verifier; - nodes.push(Node { - peer_id: generate_local_key().public().to_peer_id().to_string(), - actor: actor.to_string(), - node_name: "ZKM".to_string(), - goat_addr: get_rand_goat_address(), - btc_pub_key: pub_key.clone(), - socket_addr: "".to_string(), - reward: "0".to_string(), - service_fee_rate: 0.0, - available_peg_btc: U256::from_str("4700000000000000000000000") - .unwrap_or_default() - .to_string(), - updated_at: current_time_secs(), - created_at: current_time_secs(), - binding_sig: String::new(), - binding_issued_at: 0, - }); - let goat_addr = get_rand_goat_address(); - nodes.push(Node { - peer_id: generate_local_key().public().to_peer_id().to_string(), - actor: Actor::Committee.to_string(), - node_name: "ZKM".to_string(), - goat_addr: goat_addr.clone(), - btc_pub_key: pub_key.clone(), - socket_addr: "".to_string(), - reward: "0".to_string(), - service_fee_rate: 0.0, - available_peg_btc: U256::from_str("4700000000000000000000000") - .unwrap_or_default() - .to_string(), - updated_at: current_time_secs(), - created_at: current_time_secs(), - binding_sig: String::new(), - binding_issued_at: 0, - }); - - let local_db = create_local_db(&temp_sqlite_db_path()).await; - init_nodes_data(&local_db, &nodes).await?; - tokio::spawn(rpc_service::serve_with_listener( - listener, - AppState::create_arc_app_state( - local_db.clone(), - Actor::Verifier, - generate_local_key().public().to_peer_id().to_string(), - MetricsState::new(Arc::new(Mutex::new(Registry::default()))), - ) - .await?, - CancellationToken::new(), - )); - sleep(Duration::from_secs(3)).await; - let api_test_items = [ - ApiTestItem { - tag: format!("{} get node", routes::v1::NODES_BASE), - url: format!("http://{addr}{}?actor={}", routes::v1::NODES_BASE, actor), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(|text| -> bool { - matches!( - serde_json::from_str::(&text), - Ok(node_resp) if node_resp.nodes.len() == 1 - ) - })), - }, - ApiTestItem { - tag: format!("{} get node overview", routes::v1::NODES_OVERVIEW), - url: format!("http://{addr}{}", routes::v1::NODES_OVERVIEW), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(|text| -> bool { - matches!( - serde_json::from_str::(&text), - Ok(node_overview) if node_overview.nodes_overview.online_verifiers == 1 && - node_overview.nodes_overview.online_committees == 1 - ) - })), - }, - ]; - do_batch_tests("node apis", &Client::new(), &api_test_items).await?; - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_bitvm_api() -> Result<(), Box> { - init(None); - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?.to_string(); - let actor = Actor::Verifier; - let local_key = generate_local_key(); - let peer_id = local_key.public().to_peer_id().to_string(); - let local_db = create_local_db(&temp_sqlite_db_path()).await; - let bridge_in_instance_id = Uuid::new_v4(); - let bridge_in_from = get_rand_btc_address_p2wpkh(get_network()); - let bridge_in_to = get_rand_goat_address(); - let bridge_in_amount = 20000000; - let utxo: Vec = vec![Utxo { - txid: generate_random_bytes(32).try_into().unwrap(), - vout: 1, - amount_sats: bridge_in_amount as u64, - }]; - let mut instances = Vec::::new(); - let mut graphs = Vec::::new(); - instances.push(Instance { - instance_id: bridge_in_instance_id, - network: get_network().to_string(), - from_addr: bridge_in_from.clone(), - to_addr: bridge_in_to.clone(), - amount: bridge_in_amount, - fees: Default::default(), - input_utxos: serde_json::to_string(&utxo).unwrap(), - status: InstanceBridgeInStatus::RelayerL2Minted.to_string(), - goat_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), - goat_tx_height: 1000, - user_xonly_pubkey: Default::default(), - user_change_addr: get_rand_btc_address_p2wpkh(get_network()), - user_refund_addr: get_rand_btc_address_p2wpkh(get_network()), - btc_txid: None, - btc_height: 0, - pegin_confirm_txid: None, - pegin_cancel_txid: None, - committees_answers: Default::default(), - pegin_data_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), - parameters: None, - post_pegin_txhash: None, - status_updated_at: current_time_secs(), - created_at: current_time_secs(), - updated_at: current_time_secs(), - }); - - instances.push(Instance { - instance_id: Uuid::new_v4(), - network: get_network().to_string(), - from_addr: bridge_in_from.clone(), - to_addr: bridge_in_to.clone(), - amount: bridge_in_amount, - fees: Default::default(), - input_utxos: serde_json::to_string(&utxo).unwrap(), - status: InstanceBridgeInStatus::RelayerL1Broadcasted.to_string(), - goat_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), - goat_tx_height: 1000, - user_xonly_pubkey: Default::default(), - user_change_addr: get_rand_btc_address_p2wpkh(get_network()), - user_refund_addr: get_rand_btc_address_p2wpkh(get_network()), - btc_txid: None, - btc_height: 0, - pegin_confirm_txid: None, - pegin_cancel_txid: None, - committees_answers: Default::default(), - pegin_data_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), - parameters: None, - post_pegin_txhash: None, - status_updated_at: current_time_secs(), - created_at: current_time_secs(), - updated_at: current_time_secs(), - }); - let graph_id = Uuid::new_v4(); - let graph_status = GraphStatus::OperatorTake1.to_string(); - let graph_to = get_rand_btc_address_p2wpkh(get_network()); - let graph_from = get_rand_goat_address(); - - graphs.push(Graph { - graph_id, - instance_id: bridge_in_instance_id, - kickoff_index: 0, - from_addr: graph_from.clone(), - to_addr: graph_to.clone(), - amount: bridge_in_amount, - challenge_amount: bridge_in_amount, - status: graph_status.clone(), - sub_status: "".to_string(), - operator_pubkey: "".to_string(), - definition_hash: format!("fixture-{graph_id}"), - next_prekickoff: None, - cur_prekickoff_txid: None, - force_skip_kickoff_txid: None, - quick_challenge_txid: None, - challenge_incomplete_kickoff_txid: None, - pegin_txid: None, - kickoff_txid: None, - take1_txid: None, - challenge_txid: None, - take2_txid: None, - operator_assert_txid: None, - verifier_assert_txids: vec![], - disprove_txids: vec![], - watchtower_challenge_timeout_txids: vec![], - operator_challenge_nack_txids: vec![], - operator_commit_timeout_txid: None, - watchtower_challenge_init_txid: None, - init_withdraw_tx_hash: Some(format!("0x{}", hex::encode(generate_random_bytes(32)))), - bridge_out_start_at: current_time_secs() + 100, - status_updated_at: current_time_secs(), - proceed_withdraw_height: 0, - created_at: current_time_secs(), - updated_at: current_time_secs(), - }); - let finalized_graph_id = Uuid::new_v4(); - graphs.push(Graph { - graph_id: finalized_graph_id, - instance_id: bridge_in_instance_id, - kickoff_index: 0, - from_addr: graph_from.clone(), - to_addr: graph_to.clone(), - amount: bridge_in_amount, - challenge_amount: bridge_in_amount, - status: GraphStatus::CommitteePresigned.to_string(), - sub_status: "".to_string(), - operator_pubkey: "".to_string(), - definition_hash: format!("fixture-{finalized_graph_id}"), - next_prekickoff: None, - cur_prekickoff_txid: None, - force_skip_kickoff_txid: None, - quick_challenge_txid: None, - challenge_incomplete_kickoff_txid: None, - pegin_txid: None, - kickoff_txid: None, - take1_txid: None, - challenge_txid: None, - take2_txid: None, - operator_assert_txid: None, - verifier_assert_txids: vec![], - disprove_txids: vec![], - watchtower_challenge_timeout_txids: vec![], - operator_challenge_nack_txids: vec![], - operator_commit_timeout_txid: None, - watchtower_challenge_init_txid: None, - init_withdraw_tx_hash: None, - bridge_out_start_at: 0, - status_updated_at: current_time_secs(), - proceed_withdraw_height: 0, - created_at: current_time_secs(), - updated_at: current_time_secs(), - }); - - init_instance_graph_data(&local_db, &instances, &graphs).await?; - - tokio::spawn(rpc_service::serve_with_listener( - listener, - AppState::create_arc_app_state( - local_db.clone(), - actor.clone(), - peer_id.clone(), - MetricsState::new(Arc::new(Mutex::new(Registry::default()))), - ) - .await?, - CancellationToken::new(), - )); - sleep(Duration::from_secs(3)).await; - - let target_instance_id = bridge_in_instance_id; - let api_test_items = vec![ - ApiTestItem { - tag: routes::v1::INSTANCES_SETTINGS.to_string(), - url: format!("http://{addr}{}", routes::v1::INSTANCES_SETTINGS), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(|text| -> bool { - matches!( - serde_json::from_str::(&text), - Ok(instances_setting) if instances_setting.bridge_in_amount == BRIDGE_IN_AMOUNTS.to_vec() - ) - })), - }, - ApiTestItem { - tag: routes::v1::INSTANCES_BY_ID.to_string(), - url: format!( - "http://{addr}{}/{}", - routes::v1::INSTANCES_BASE, - bridge_in_instance_id - ), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(move |text| -> bool { - if let Ok(instance_res) = serde_json::from_str::(&text) - && let Some(instance_wrap) = instance_res.instance_wrap - && instance_wrap.instance.instance_id.eq(&target_instance_id) - { - true - } else { - false - } - })), - }, - ApiTestItem { - tag: format!("{} get instances", routes::v1::INSTANCES_BASE), - url: format!( - "http://{addr}{}?from_addr={}", - routes::v1::INSTANCES_BASE, - bridge_in_from, - ), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(|text| -> bool { - matches!( - serde_json::from_str::(&text), - Ok(instance_list) if instance_list.total == 2 - ) - })), - }, - ApiTestItem { - tag: format!("{} info", routes::v1::INSTANCES_OVERVIEW), - url: format!("http://{addr}{}", routes::v1::INSTANCES_OVERVIEW,), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(move |text| -> bool { - matches!( - serde_json::from_str::(&text), - Ok(instance_overview) if instance_overview.instances_overview.total_bridge_in_amount == bridge_in_amount - ) - })), - }, - ApiTestItem { - tag: format!("{} info", routes::v1::GRAPHS_BY_ID), - url: format!("http://{addr}{}/{}", routes::v1::GRAPHS_BASE, graph_id), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(move |text| -> bool { - if let Ok(graph_res) = serde_json::from_str::(&text) - && let Some(graph) = graph_res.graph - && graph.graph_id == graph_id - { - true - } else { - false - } - })), - }, - ApiTestItem { - tag: format!("{} get graphs info", routes::v1::GRAPHS_BASE), - url: format!( - "http://{addr}{}?status={}&from_addr={}&graph_id={}", - routes::v1::GRAPHS_BASE, - graph_status, - graph_from, - graph_id - ), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: Some(Box::new(move |text| -> bool { - matches!( - serde_json::from_str::(&text), - Ok(graph_list) if graph_list.total == 1 - ) - })), - }, - ]; - do_batch_tests("bitvm apis", &Client::new(), &api_test_items).await?; - Ok(()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_proof_api() -> Result<(), Box> { - init(None); - let listener = TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?.to_string(); - info!("Start api server"); - let committee = Actor::Committee; - let committee_peer_id = generate_local_key().public().to_peer_id().to_string(); - let local_db = create_local_db(&temp_sqlite_db_path()).await; - tokio::spawn(rpc_service::serve_with_listener( - listener, - AppState::create_arc_app_state( - local_db, - committee, - committee_peer_id, - MetricsState::new(Arc::new(Mutex::new(Registry::default()))), - ) - .await?, - CancellationToken::new(), - )); - sleep(Duration::from_secs(3)).await; - let client = reqwest::Client::new(); - - let api_test_items = [ApiTestItem { - tag: format!("{} get proofs desc", routes::v1::PROOFS_CHAIN_PROOFS_DESC), - url: format!( - "http://{addr}{}?proof_type=header_chain", - routes::v1::PROOFS_CHAIN_PROOFS_DESC - ), - json_payload: None, - method: Method::GET, - expe_res: true, - resp_validation: None, - }]; - do_batch_tests("node apis", &client, &api_test_items).await?; - Ok(()) - } -} diff --git a/node/src/rpc_service/routes.rs b/node/src/rpc_service/routes.rs index 93f150cbb..d2f27e84d 100644 --- a/node/src/rpc_service/routes.rs +++ b/node/src/rpc_service/routes.rs @@ -4,6 +4,8 @@ pub(crate) mod v1 { pub const NODES_BASE: &str = "/v1/nodes"; pub const NODES_BY_ID: &str = "/v1/nodes/{:id}"; pub const NODES_OVERVIEW: &str = "/v1/nodes/overview"; + /// Liveness / readiness of this node, served from memory. + pub const HEALTH: &str = "/v1/health"; pub const INSTANCES_BASE: &str = "/v1/instances"; pub const INSTANCES_SETTINGS: &str = "/v1/instances/settings"; @@ -30,20 +32,14 @@ pub(crate) mod v1 { pub const DEBUG_STATUS: &str = "/v1/debug/status"; // TODO(auth): Restrict debug endpoints before exposing the RPC outside trusted operators. #[cfg(feature = "rpc-debug-endpoints")] - pub const DEBUG_GRAPH_MESSAGES: &str = "/v1/debug/graphs/{:id}/messages"; + pub const DEBUG_ENGINE: &str = "/v1/debug/engine"; // TODO(auth): Restrict debug endpoints before exposing the RPC outside trusted operators. #[cfg(feature = "rpc-debug-endpoints")] - pub const DEBUG_INSTANCE_MESSAGES: &str = "/v1/debug/instances/{:id}/messages"; - // TODO(auth): Restrict debug endpoints before exposing the RPC outside trusted operators. - #[cfg(feature = "rpc-debug-endpoints")] - pub const DEBUG_MESSAGE_DETAILS: &str = "/v1/debug/messages/{:id}"; + pub const DEBUG_ENGINE_CLEAR_POISON: &str = "/v1/debug/engine/clear-poison"; // pub const PROOFS_BASE: &str = "/v1/proofs"; pub const PROOFS_CHAIN_PROOFS_DESC: &str = "/v1/proofs/chain_proofs_desc"; pub const NODES_WATCHTOWER_BASE: &str = "/v1/proofs/watchtower_proofs"; - #[allow(dead_code)] pub const NODES_OPERATOR_BASE: &str = "/v1/proofs/operator_proofs"; pub const PROOFS_OPERATOR_PROOF_DESC: &str = "/v1/proofs/operator_proofs_desc"; pub const PROOFS_WATCHTOWER_PROOF_TIMEOUT: &str = "/v1/proofs/watchtower_proofs_timeout"; - #[allow(dead_code)] - pub const PROOFS_OPERATOR_PROOF_TIMEOUT: &str = "/v1/proofs/operator_proofs_timeout"; } diff --git a/node/src/rpc_service/swap.rs b/node/src/rpc_service/swap.rs index c00b9789c..01642dd72 100644 --- a/node/src/rpc_service/swap.rs +++ b/node/src/rpc_service/swap.rs @@ -1,6 +1,6 @@ use super::utils::{deserialize_u256, serialize_u256}; +use crate::clock::current_time_secs; use crate::rpc_service::bitvm::{StatusExtra, StatusUserAction}; -use crate::rpc_service::current_time_secs; use alloy::primitives::U256; use client::btc_chain::BTCClient; use serde::{Deserialize, Serialize}; diff --git a/node/src/rpc_service/tests.rs b/node/src/rpc_service/tests.rs new file mode 100644 index 000000000..d80db4694 --- /dev/null +++ b/node/src/rpc_service/tests.rs @@ -0,0 +1,818 @@ +use crate::clock::current_time_secs; +use crate::env::{ + ENV_BITVM_SECRET, ENV_GOAT_CHAIN_URL, ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, ENV_PROOF_SEVER_URL, + get_network, +}; +use crate::metrics_service::MetricsState; +use crate::rpc_service::auth::{ + AUTH_NONCE_HEADER, AUTH_SIGNATURE_HEADER, AUTH_TIMESTAMP_HEADER, sign_request_auth, +}; +use crate::rpc_service::bitvm::{ + BRIDGE_IN_AMOUNTS, GraphGetResponse, GraphListResponse, InstanceGetResponse, + InstanceListResponse, InstanceOverviewResponse, InstanceSettingResponse, +}; +use crate::rpc_service::node::{NodeListResponse, NodeOverViewResponse}; +use crate::rpc_service::{self, Actor, AppState, routes}; +use crate::utils::{ + generate_local_key, generate_random_bytes, get_rand_btc_address_p2wpkh, get_rand_goat_address, + temp_sqlite_db_path, +}; +use alloy::primitives::U256; +use client::Utxo; +use http::Method; +use prometheus_client::registry::Registry; +use reqwest::Client; +use secp256k1::{Keypair, SECP256K1, Secp256k1}; +use serde_json::Value; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use store::localdb::{GraphRuntimeUpdate, LocalDB, StorageProcessor}; +use store::{ + Graph, GraphStatus, GraphStatusSource, Instance, InstanceBridgeInStatus, Node, create_local_db, +}; +use tokio::{net::TcpListener, time::sleep}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; +use tracing_subscriber::EnvFilter; +use uuid::Uuid; + +struct ApiTestItem { + pub tag: String, + pub url: String, + pub json_payload: Option, + pub method: Method, + pub expe_res: bool, + pub resp_validation: Option bool>>, +} + +impl ApiTestItem { + async fn do_test(&self, client: &Client) -> anyhow::Result<()> { + eprintln!("Start test api: {}", self.tag); + info!("Start test api: {}", self.tag); + let mut request_builder = match self.method { + Method::POST => client.post(self.url.clone()), + Method::GET => client.get(self.url.clone()), + Method::PUT => client.put(self.url.clone()), + Method::DELETE => client.delete(self.url.clone()), + _ => { + anyhow::bail!("wrong method"); + } + }; + if let Some(json_payload) = self.json_payload.clone() { + request_builder = request_builder.json(&json_payload); + } + + let resp = request_builder.send().await?; + let actual_status = resp.status().is_success(); + if actual_status != self.expe_res { + let data = resp.text().await?; + eprintln!( + "Test api '{}' failed: expected status {}, got {}, resp: {data}", + self.tag, self.expe_res, actual_status + ); + error!( + "Test api '{}' failed: expected status {}, got {}, resp: {data}", + self.tag, self.expe_res, actual_status + ); + anyhow::bail!("Test failed: expected status {}, got {}", self.expe_res, actual_status); + } + + if let Some(validate_fn) = &self.resp_validation { + let text = resp.text().await?; + if !validate_fn(text) { + eprintln!("Test api '{}' fail validate_fn", self.tag); + error!("Test api '{}' fail validate_fn", self.tag); + anyhow::bail!("Test api '{}' fail validate_fn", self.tag); + } + } + + Ok(()) + } +} + +async fn do_batch_tests( + batch_tag: &str, + client: &Client, + items: &[ApiTestItem], +) -> anyhow::Result<()> { + info!("Start batch test:{batch_tag}"); + for item in items { + item.do_test(client).await?; + } + info!("Finish batch test:{batch_tag}"); + Ok(()) +} +fn init(remote_proof_server: Option) { + unsafe { + std::env::set_var("RUST_LOG", "info"); + std::env::set_var(ENV_GOAT_CHAIN_URL, "https://rpc.testnet3.goat.network"); + std::env::set_var( + ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, + "0x21f619040AC2eAcacEF8Fe17Ae8bDF53ec69C66f", + ); + if let Some(remote_proof_server) = remote_proof_server { + std::env::set_var(ENV_PROOF_SEVER_URL, remote_proof_server); + } + } + let _ = tracing_subscriber::fmt().with_env_filter(EnvFilter::from_default_env()).try_init(); +} + +async fn spawn_metrics_listener( + app_state: Arc, + metrics_path: &str, + cancellation_token: CancellationToken, +) -> anyhow::Result<(String, tokio::task::JoinHandle>)> { + let listener = rpc_service::bind_metrics_listener("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + let metrics_path = metrics_path.to_string(); + let handle = tokio::spawn(rpc_service::serve_metrics( + listener, + metrics_path, + app_state, + cancellation_token, + )); + Ok((addr, handle)) +} + +async fn mock_app_state() -> anyhow::Result> { + let local_db = create_local_db(&temp_sqlite_db_path()).await; + let metrics_state = MetricsState::new(Arc::new(Mutex::new(Registry::default()))); + rpc_service::AppState::create_arc_mock_app_state( + local_db, + Actor::Verifier, + generate_local_key().public().to_peer_id().to_string(), + metrics_state, + ) + .await +} + +fn set_test_auth_key() -> Keypair { + let keypair = Keypair::from_seckey_slice(SECP256K1, &[0x42; 32]).unwrap(); + unsafe { + std::env::set_var(ENV_BITVM_SECRET, hex::encode(keypair.secret_key().secret_bytes())) + }; + keypair +} + +async fn spawn_business_listener( + cancellation_token: CancellationToken, +) -> anyhow::Result<(String, tokio::task::JoinHandle<()>)> { + let app_state = mock_app_state().await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + let server = rpc_service::build_business_router(app_state); + let handle = tokio::spawn(async move { + let _ = tokio::select! { + result = axum::serve(listener, server) => result, + _ = cancellation_token.cancelled() => Ok(()), + }; + }); + Ok((addr, handle)) +} + +#[tokio::test(flavor = "multi_thread")] +async fn signed_request_nonce_is_rejected_after_first_use() -> Result<(), Box> +{ + let keypair = set_test_auth_key(); + let cancellation_token = CancellationToken::new(); + let (addr, server) = spawn_business_listener(cancellation_token.clone()).await?; + let request_target = routes::v1::PEGOUT.to_owned(); + let url = format!("http://{addr}{request_target}"); + let (timestamp, nonce, signature) = + sign_request_auth(&keypair, &Method::POST, &request_target, &[]); + let client = Client::new(); + + let first = client + .post(&url) + .header(AUTH_TIMESTAMP_HEADER, ×tamp) + .header(AUTH_NONCE_HEADER, &nonce) + .header(AUTH_SIGNATURE_HEADER, &signature) + .send() + .await?; + assert_ne!(first.status().as_u16(), 409); + + let replay = client + .post(&url) + .header(AUTH_TIMESTAMP_HEADER, ×tamp) + .header(AUTH_NONCE_HEADER, &nonce) + .header(AUTH_SIGNATURE_HEADER, &signature) + .send() + .await?; + assert_eq!(replay.status().as_u16(), 409); + let replay_body: Value = replay.json().await?; + assert_eq!(replay_body["error"], "AUTH_REPLAY"); + + cancellation_token.cancel(); + server.await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn debug_routes_follow_rpc_debug_endpoints_feature() -> Result<(), Box> +{ + let cancellation_token = CancellationToken::new(); + let (addr, server) = spawn_business_listener(cancellation_token.clone()).await?; + let client = Client::new(); + let id = Uuid::new_v4(); + + let status = client.get(format!("http://{addr}/v1/debug/status")).send().await?; + let engine = client.get(format!("http://{addr}/v1/debug/engine")).send().await?; + let verifier_challenge = + client.post(format!("http://{addr}/v1/graphs/{id}/send-verifier-challenge")).send().await?; + + if cfg!(feature = "rpc-debug-endpoints") { + assert_eq!(status.status().as_u16(), 200); + // The engine is not running on the mock node. + assert_eq!(engine.status().as_u16(), 404); + let engine_body: Value = engine.json().await?; + assert_eq!(engine_body["error"], "ENGINE_UNAVAILABLE"); + assert_eq!(verifier_challenge.status().as_u16(), 401); + } else { + assert_eq!(status.status().as_u16(), 404); + assert_eq!(engine.status().as_u16(), 404); + assert_eq!(verifier_challenge.status().as_u16(), 404); + } + + cancellation_token.cancel(); + server.await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn metrics_are_served_only_by_the_dedicated_listener() +-> Result<(), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + let app_state = mock_app_state().await?; + let cancellation_token = CancellationToken::new(); + let server = tokio::spawn(rpc_service::serve_with_listener( + listener, + app_state.clone(), + cancellation_token.clone(), + )); + let (metrics_addr, metrics_server) = + spawn_metrics_listener(app_state, "/metrics", cancellation_token.clone()).await?; + sleep(Duration::from_millis(100)).await; + + let client = Client::new(); + client.get(format!("http://{addr}/")).send().await?.error_for_status()?; + let unmatched = client.get(format!("http://{addr}/missing")).send().await?; + assert_eq!(unmatched.status().as_u16(), 404); + + // The business listener no longer exposes metrics at all. + let business_metrics = client.get(format!("http://{addr}/metrics")).send().await?; + assert_eq!(business_metrics.status().as_u16(), 404); + + // The metrics listener exposes nothing but the metrics path. + for path in ["/", "/v1/nodes", "/v1/debug/status"] { + let response = client.get(format!("http://{metrics_addr}{path}")).send().await?; + assert_eq!(response.status().as_u16(), 404, "unexpected route on metrics listener: {path}"); + } + + let first = client.get(format!("http://{metrics_addr}/metrics")).send().await?; + assert_eq!(first.status().as_u16(), 200); + assert_eq!( + first.headers().get(http::header::CONTENT_TYPE).unwrap(), + "application/openmetrics-text;charset=utf-8;version=1.0.0" + ); + let first_scrape = first.text().await?; + let second_scrape = + client.get(format!("http://{metrics_addr}/metrics")).send().await?.text().await?; + + assert!( + first_scrape.contains("http_requests_total{method=\"GET\",route=\"/\",status=\"200\"} 1") + ); + // `/missing` and the rejected `/metrics` request on the business listener. + assert!( + first_scrape + .contains("http_requests_total{method=\"GET\",route=\"unmatched\",status=\"404\"} 2") + ); + assert!(first_scrape.contains("http_requests_in_flight 0")); + // Requests served by the metrics listener are not counted at all. + assert!(!first_scrape.contains("route=\"/metrics\"")); + assert_eq!(first_scrape, second_scrape); + + cancellation_token.cancel(); + server.await??; + assert_eq!(metrics_server.await??, "metrics_shutdown"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn metrics_listener_honors_custom_path() -> Result<(), Box> { + let app_state = mock_app_state().await?; + let cancellation_token = CancellationToken::new(); + let (metrics_addr, metrics_server) = + spawn_metrics_listener(app_state, "/internal/metrics", cancellation_token.clone()).await?; + sleep(Duration::from_millis(100)).await; + + let client = Client::new(); + let response = client.get(format!("http://{metrics_addr}/internal/metrics")).send().await?; + assert_eq!(response.status().as_u16(), 200); + assert!(response.text().await?.contains("http_requests_in_flight")); + let default_path = client.get(format!("http://{metrics_addr}/metrics")).send().await?; + assert_eq!(default_path.status().as_u16(), 404); + + cancellation_token.cancel(); + metrics_server.await??; + Ok(()) +} + +#[tokio::test] +async fn metrics_listener_reports_bind_conflicts() -> Result<(), Box> { + let occupied = std::net::TcpListener::bind("127.0.0.1:0")?; + let addr = occupied.local_addr()?.to_string(); + let error = rpc_service::bind_metrics_listener(&addr).await.unwrap_err(); + assert!( + error.to_string().contains(&format!("failed to bind metrics listener to {addr}")), + "unexpected error: {error}" + ); + Ok(()) +} + +#[test] +fn metrics_path_validation_rejects_unusable_paths() { + for path in ["/metrics", "/", "/internal/metrics"] { + assert!(rpc_service::validate_metrics_path(path).is_ok(), "rejected valid path: {path}"); + } + for path in ["metrics", "", "/metrics?scrape=1", "/metrics#frag", "/{id}", "/met rics"] { + assert!(rpc_service::validate_metrics_path(path).is_err(), "accepted invalid path: {path}"); + } +} + +async fn init_nodes_data(local_db: &LocalDB, nodes: &[Node]) -> anyhow::Result<()> { + let mut tx = local_db.start_transaction().await?; + for node in nodes { + tx.upsert_node(node).await?; + } + tx.commit().await?; + Ok(()) +} + +async fn seed_graph_runtime(tx: &mut StorageProcessor<'_>, graph: &Graph) -> anyhow::Result<()> { + let target_status = GraphStatus::from_str(&graph.status)?; + let sub_status = graph.sub_status.clone(); + let challenge_txid = graph.challenge_txid.clone(); + let init_withdraw_tx_hash = graph.init_withdraw_tx_hash.clone(); + let bridge_out_start_at = graph.bridge_out_start_at; + let proceed_withdraw_height = graph.proceed_withdraw_height; + + let mut definition = graph.clone(); + definition.status = GraphStatus::OperatorPresigned.to_string(); + definition.sub_status.clear(); + definition.challenge_txid = None; + definition.init_withdraw_tx_hash = None; + definition.bridge_out_start_at = 0; + definition.proceed_withdraw_height = 0; + tx.upsert_graph_definition(&definition).await?; + + if target_status != GraphStatus::OperatorPresigned { + tx.transition_graph_status( + graph.instance_id, + graph.graph_id, + target_status, + GraphStatusSource::ChainReconcile, + (!sub_status.is_empty()).then_some(sub_status), + ) + .await?; + } + + let mut runtime = GraphRuntimeUpdate::new(graph.instance_id, graph.graph_id); + if let Some(challenge_txid) = challenge_txid { + runtime = runtime.with_challenge_txid(challenge_txid); + } + if let Some(init_withdraw_tx_hash) = init_withdraw_tx_hash { + runtime = runtime.with_init_withdraw_tx_hash(init_withdraw_tx_hash); + } + if bridge_out_start_at != 0 { + runtime = runtime.with_bridge_out_start_at(bridge_out_start_at); + } + if proceed_withdraw_height != 0 { + runtime = runtime.with_proceed_withdraw_height(proceed_withdraw_height); + } + tx.update_graph_runtime(&runtime).await?; + Ok(()) +} + +async fn init_instance_graph_data( + local_db: &LocalDB, + instances: &[Instance], + graphs: &[Graph], +) -> anyhow::Result<()> { + let mut tx = local_db.start_transaction().await?; + for instance in instances { + tx.insert_instance_if_absent(instance).await?; + } + for graph in graphs { + seed_graph_runtime(&mut tx, graph).await?; + } + tx.commit().await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_nodes_api() -> Result<(), Box> { + init(None); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + let mut nodes = Vec::::new(); + let (_, public_key) = Secp256k1::new().generate_keypair(&mut rand::thread_rng()); + let pub_key = public_key.to_string(); + let actor = Actor::Verifier; + nodes.push(Node { + peer_id: generate_local_key().public().to_peer_id().to_string(), + actor: actor.to_string(), + node_name: "ZKM".to_string(), + goat_addr: get_rand_goat_address(), + btc_pub_key: pub_key.clone(), + socket_addr: "".to_string(), + reward: "0".to_string(), + service_fee_rate: 0.0, + available_peg_btc: U256::from_str("4700000000000000000000000") + .unwrap_or_default() + .to_string(), + updated_at: current_time_secs(), + created_at: current_time_secs(), + binding_sig: String::new(), + binding_issued_at: 0, + }); + let goat_addr = get_rand_goat_address(); + nodes.push(Node { + peer_id: generate_local_key().public().to_peer_id().to_string(), + actor: Actor::Committee.to_string(), + node_name: "ZKM".to_string(), + goat_addr: goat_addr.clone(), + btc_pub_key: pub_key.clone(), + socket_addr: "".to_string(), + reward: "0".to_string(), + service_fee_rate: 0.0, + available_peg_btc: U256::from_str("4700000000000000000000000") + .unwrap_or_default() + .to_string(), + updated_at: current_time_secs(), + created_at: current_time_secs(), + binding_sig: String::new(), + binding_issued_at: 0, + }); + + let local_db = create_local_db(&temp_sqlite_db_path()).await; + init_nodes_data(&local_db, &nodes).await?; + tokio::spawn(rpc_service::serve_with_listener( + listener, + AppState::create_arc_app_state( + local_db.clone(), + Actor::Verifier, + generate_local_key().public().to_peer_id().to_string(), + MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + None, + ) + .await?, + CancellationToken::new(), + )); + sleep(Duration::from_secs(3)).await; + let api_test_items = [ + ApiTestItem { + tag: format!("{} get node", routes::v1::NODES_BASE), + url: format!("http://{addr}{}?actor={}", routes::v1::NODES_BASE, actor), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(|text| -> bool { + matches!( + serde_json::from_str::(&text), + Ok(node_resp) if node_resp.nodes.len() == 1 + ) + })), + }, + ApiTestItem { + tag: format!("{} get node overview", routes::v1::NODES_OVERVIEW), + url: format!("http://{addr}{}", routes::v1::NODES_OVERVIEW), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(|text| -> bool { + matches!( + serde_json::from_str::(&text), + Ok(node_overview) if node_overview.nodes_overview.online_verifiers == 1 && + node_overview.nodes_overview.online_committees == 1 + ) + })), + }, + ]; + do_batch_tests("node apis", &Client::new(), &api_test_items).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_bitvm_api() -> Result<(), Box> { + init(None); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + let actor = Actor::Verifier; + let local_key = generate_local_key(); + let peer_id = local_key.public().to_peer_id().to_string(); + let local_db = create_local_db(&temp_sqlite_db_path()).await; + let bridge_in_instance_id = Uuid::new_v4(); + let bridge_in_from = get_rand_btc_address_p2wpkh(get_network()); + let bridge_in_to = get_rand_goat_address(); + let bridge_in_amount = 20000000; + let utxo: Vec = vec![Utxo { + txid: generate_random_bytes(32).try_into().unwrap(), + vout: 1, + amount_sats: bridge_in_amount as u64, + }]; + let mut instances = Vec::::new(); + let mut graphs = Vec::::new(); + instances.push(Instance { + instance_id: bridge_in_instance_id, + network: get_network().to_string(), + from_addr: bridge_in_from.clone(), + to_addr: bridge_in_to.clone(), + amount: bridge_in_amount, + fees: Default::default(), + input_utxos: serde_json::to_string(&utxo).unwrap(), + status: InstanceBridgeInStatus::RelayerL2Minted.to_string(), + goat_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), + goat_tx_height: 1000, + user_xonly_pubkey: Default::default(), + user_change_addr: get_rand_btc_address_p2wpkh(get_network()), + user_refund_addr: get_rand_btc_address_p2wpkh(get_network()), + btc_txid: None, + btc_height: 0, + pegin_confirm_txid: None, + pegin_cancel_txid: None, + committees_answers: Default::default(), + pegin_data_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), + parameters: None, + post_pegin_txhash: None, + key_deleted: false, + status_updated_at: current_time_secs(), + created_at: current_time_secs(), + updated_at: current_time_secs(), + }); + + instances.push(Instance { + instance_id: Uuid::new_v4(), + network: get_network().to_string(), + from_addr: bridge_in_from.clone(), + to_addr: bridge_in_to.clone(), + amount: bridge_in_amount, + fees: Default::default(), + input_utxos: serde_json::to_string(&utxo).unwrap(), + status: InstanceBridgeInStatus::RelayerL1Broadcasted.to_string(), + goat_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), + goat_tx_height: 1000, + user_xonly_pubkey: Default::default(), + user_change_addr: get_rand_btc_address_p2wpkh(get_network()), + user_refund_addr: get_rand_btc_address_p2wpkh(get_network()), + btc_txid: None, + btc_height: 0, + pegin_confirm_txid: None, + pegin_cancel_txid: None, + committees_answers: Default::default(), + pegin_data_tx_hash: format!("0x{}", hex::encode(generate_random_bytes(32))), + parameters: None, + post_pegin_txhash: None, + key_deleted: false, + status_updated_at: current_time_secs(), + created_at: current_time_secs(), + updated_at: current_time_secs(), + }); + let graph_id = Uuid::new_v4(); + let graph_status = GraphStatus::OperatorTake1.to_string(); + let graph_to = get_rand_btc_address_p2wpkh(get_network()); + let graph_from = get_rand_goat_address(); + + graphs.push(Graph { + graph_id, + instance_id: bridge_in_instance_id, + kickoff_index: 0, + from_addr: graph_from.clone(), + to_addr: graph_to.clone(), + amount: bridge_in_amount, + challenge_amount: bridge_in_amount, + status: graph_status.clone(), + sub_status: "".to_string(), + operator_pubkey: "".to_string(), + definition_hash: format!("fixture-{graph_id}"), + next_prekickoff: None, + cur_prekickoff_txid: None, + force_skip_kickoff_txid: None, + quick_challenge_txid: None, + challenge_incomplete_kickoff_txid: None, + pegin_txid: None, + kickoff_txid: None, + take1_txid: None, + challenge_txid: None, + take2_txid: None, + operator_assert_txid: None, + verifier_assert_txids: vec![], + disprove_txids: vec![], + watchtower_challenge_timeout_txids: vec![], + operator_challenge_nack_txids: vec![], + operator_commit_timeout_txid: None, + watchtower_challenge_init_txid: None, + init_withdraw_tx_hash: Some(format!("0x{}", hex::encode(generate_random_bytes(32)))), + bridge_out_start_at: current_time_secs() + 100, + status_updated_at: current_time_secs(), + proceed_withdraw_height: 0, + created_at: current_time_secs(), + updated_at: current_time_secs(), + }); + let finalized_graph_id = Uuid::new_v4(); + graphs.push(Graph { + graph_id: finalized_graph_id, + instance_id: bridge_in_instance_id, + // One definition per (operator, nonce): the second fixture takes the next slot. + kickoff_index: 1, + from_addr: graph_from.clone(), + to_addr: graph_to.clone(), + amount: bridge_in_amount, + challenge_amount: bridge_in_amount, + status: GraphStatus::CommitteePresigned.to_string(), + sub_status: "".to_string(), + operator_pubkey: "".to_string(), + definition_hash: format!("fixture-{finalized_graph_id}"), + next_prekickoff: None, + cur_prekickoff_txid: None, + force_skip_kickoff_txid: None, + quick_challenge_txid: None, + challenge_incomplete_kickoff_txid: None, + pegin_txid: None, + kickoff_txid: None, + take1_txid: None, + challenge_txid: None, + take2_txid: None, + operator_assert_txid: None, + verifier_assert_txids: vec![], + disprove_txids: vec![], + watchtower_challenge_timeout_txids: vec![], + operator_challenge_nack_txids: vec![], + operator_commit_timeout_txid: None, + watchtower_challenge_init_txid: None, + init_withdraw_tx_hash: None, + bridge_out_start_at: 0, + status_updated_at: current_time_secs(), + proceed_withdraw_height: 0, + created_at: current_time_secs(), + updated_at: current_time_secs(), + }); + + init_instance_graph_data(&local_db, &instances, &graphs).await?; + + tokio::spawn(rpc_service::serve_with_listener( + listener, + AppState::create_arc_app_state( + local_db.clone(), + actor.clone(), + peer_id.clone(), + MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + None, + ) + .await?, + CancellationToken::new(), + )); + sleep(Duration::from_secs(3)).await; + + let target_instance_id = bridge_in_instance_id; + let api_test_items = vec![ + ApiTestItem { + tag: routes::v1::INSTANCES_SETTINGS.to_string(), + url: format!("http://{addr}{}", routes::v1::INSTANCES_SETTINGS), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(|text| -> bool { + matches!( + serde_json::from_str::(&text), + Ok(instances_setting) if instances_setting.bridge_in_amount == BRIDGE_IN_AMOUNTS.to_vec() + ) + })), + }, + ApiTestItem { + tag: routes::v1::INSTANCES_BY_ID.to_string(), + url: format!("http://{addr}{}/{}", routes::v1::INSTANCES_BASE, bridge_in_instance_id), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(move |text| -> bool { + if let Ok(instance_res) = serde_json::from_str::(&text) + && let Some(instance_wrap) = instance_res.instance_wrap + && instance_wrap.instance.instance_id.eq(&target_instance_id) + { + true + } else { + false + } + })), + }, + ApiTestItem { + tag: format!("{} get instances", routes::v1::INSTANCES_BASE), + url: format!( + "http://{addr}{}?from_addr={}", + routes::v1::INSTANCES_BASE, + bridge_in_from, + ), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(|text| -> bool { + matches!( + serde_json::from_str::(&text), + Ok(instance_list) if instance_list.total == 2 + ) + })), + }, + ApiTestItem { + tag: format!("{} info", routes::v1::INSTANCES_OVERVIEW), + url: format!("http://{addr}{}", routes::v1::INSTANCES_OVERVIEW,), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(move |text| -> bool { + matches!( + serde_json::from_str::(&text), + Ok(instance_overview) if instance_overview.instances_overview.total_bridge_in_amount == bridge_in_amount + ) + })), + }, + ApiTestItem { + tag: format!("{} info", routes::v1::GRAPHS_BY_ID), + url: format!("http://{addr}{}/{}", routes::v1::GRAPHS_BASE, graph_id), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(move |text| -> bool { + if let Ok(graph_res) = serde_json::from_str::(&text) + && let Some(graph) = graph_res.graph + && graph.graph_id == graph_id + { + true + } else { + false + } + })), + }, + ApiTestItem { + tag: format!("{} get graphs info", routes::v1::GRAPHS_BASE), + url: format!( + "http://{addr}{}?status={}&from_addr={}&graph_id={}", + routes::v1::GRAPHS_BASE, + graph_status, + graph_from, + graph_id + ), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: Some(Box::new(move |text| -> bool { + matches!( + serde_json::from_str::(&text), + Ok(graph_list) if graph_list.total == 1 + ) + })), + }, + ]; + do_batch_tests("bitvm apis", &Client::new(), &api_test_items).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_proof_api() -> Result<(), Box> { + init(None); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?.to_string(); + info!("Start api server"); + let committee = Actor::Committee; + let committee_peer_id = generate_local_key().public().to_peer_id().to_string(); + let local_db = create_local_db(&temp_sqlite_db_path()).await; + tokio::spawn(rpc_service::serve_with_listener( + listener, + AppState::create_arc_app_state( + local_db, + committee, + committee_peer_id, + MetricsState::new(Arc::new(Mutex::new(Registry::default()))), + None, + ) + .await?, + CancellationToken::new(), + )); + sleep(Duration::from_secs(3)).await; + let client = reqwest::Client::new(); + + let api_test_items = [ApiTestItem { + tag: format!("{} get proofs desc", routes::v1::PROOFS_CHAIN_PROOFS_DESC), + url: format!( + "http://{addr}{}?proof_type=header_chain", + routes::v1::PROOFS_CHAIN_PROOFS_DESC + ), + json_payload: None, + method: Method::GET, + expe_res: true, + resp_validation: None, + }]; + do_batch_tests("node apis", &client, &api_test_items).await?; + Ok(()) +} diff --git a/node/src/scheduled_tasks/event_watch/bridge_out.rs b/node/src/scheduled_tasks/event_watch/bridge_out.rs new file mode 100644 index 000000000..da350b0a3 --- /dev/null +++ b/node/src/scheduled_tasks/event_watch/bridge_out.rs @@ -0,0 +1,302 @@ +//! Bridge-out (swap) events: escrow initialisations, claims and refunds. + +use crate::env::get_network; +use crate::scheduled_tasks::get_timestamp_from_contract_data; +use crate::utils::evm_swap_utils::{extract_claim_data_from_tx, extract_escrow_data_from_tx}; +use crate::utils::{get_bridge_out_global_stats, strip_hex_prefix_owned}; +use alloy::primitives::{Address as EvmAddress, U256}; +use alloy::sol_types::SolValue; +use bitcoin::Address; +use client::goat_chain::GOATClient; +use client::graphs::GraphQueryClient; +use client::graphs::graph_query::{ + BlockRange, SwapClaimEvent, SwapEventEntity, SwapInitializeEvent, SwapRefundEvent, + get_bridge_out_events_query, +}; +use std::ops::AddAssign; +use std::str::FromStr; +use std::sync::Arc; +use store::localdb::{StorageProcessor, SwapEscrowUpdate}; +use store::{SwapEscrow, SwapEscrowStatus, normalize_escrow_hash}; +use tracing::{info, warn}; + +#[allow(clippy::too_many_arguments)] +pub async fn fetch_and_handle_bridge_out_events<'a>( + goat_client: Arc, + client: &GraphQueryClient, + storage_processor: &mut StorageProcessor<'a>, + contract_addr: &EvmAddress, + graph_url: &str, + event_entities: &[SwapEventEntity], + from_height: i64, + to_height: i64, + gateway_peg_btc_address: EvmAddress, +) -> anyhow::Result<()> { + let query_res = client + .execute_query( + graph_url, + &get_bridge_out_events_query( + event_entities, + Some(BlockRange::new(from_height, to_height)), + ), + ) + .await?; + for event_entity in event_entities { + let entity = event_entity.clone(); + if let Some(value_vec) = query_res[entity.to_string()].as_array() { + match entity { + SwapEventEntity::Initializes => { + handle_swap_init_events( + storage_processor, + goat_client.clone(), + contract_addr, + &gateway_peg_btc_address, + serde_json::from_value::>( + serde_json::Value::Array(value_vec.clone()), + )?, + ) + .await?; + } + SwapEventEntity::Claims => { + handle_swap_claim_events( + storage_processor, + goat_client.clone(), + contract_addr, + &gateway_peg_btc_address, + serde_json::from_value::>(serde_json::Value::Array( + value_vec.clone(), + ))?, + ) + .await?; + } + SwapEventEntity::Refunds => { + handle_swap_refund_events( + storage_processor, + &gateway_peg_btc_address, + serde_json::from_value::>(serde_json::Value::Array( + value_vec.clone(), + ))?, + ) + .await?; + } + }; + } + } + Ok(()) +} + +async fn handle_swap_init_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + goat_client: Arc, + swap_contract_address: &EvmAddress, + gateway_peg_btc_address: &EvmAddress, + init_events: Vec, +) -> anyhow::Result<()> { + for event in init_events { + let Some(escrow_data) = extract_escrow_data_from_tx( + &goat_client, + &event.transaction_hash, + swap_contract_address, + &hex::decode(strip_hex_prefix_owned(&event.escrow_hash))?.try_into().map_err( + |v: Vec| anyhow::anyhow!("escrow_hash length is {}, expected 32", v.len()), + )?, + ) + .await? + else { + warn!("failed to parse escrow_data for event:{event:?}"); + continue; + }; + let escrow_hash = normalize_escrow_hash(&event.escrow_hash); + let create_time = event.block_timestamp.parse::()?; + let event_height = event.block_number.parse::()?; + let escrow = SwapEscrow { + escrow_hash: escrow_hash.clone(), + network: get_network().to_string(), + status: SwapEscrowStatus::Initialize.to_string(), + offerer_addr: escrow_data.offerer.to_string(), + claimer_addr: escrow_data.claimer.to_string(), + token: escrow_data.token.to_string(), + amount: escrow_data.amount.to_string(), + refund_deadline: get_timestamp_from_contract_data(&escrow_data.refundData.0), + escrow_data: Some(hex::encode(escrow_data.abi_encode())), + init_tx_hash: event.transaction_hash.clone(), + init_tx_height: event_height, + status_updated_at: create_time, + created_at: create_time, + ..Default::default() + }; + // Swap records are derived exclusively from the on-chain Initialize + // event; duplicate event scans must not overwrite the first record. + let initialized = storage_processor.insert_swap_escrow_if_absent(&escrow).await?; + if initialized && escrow_data.token == *gateway_peg_btc_address { + let mut bridge_out_global_stats = + get_bridge_out_global_stats(storage_processor).await?; + let mut initial_amount = + U256::from_str(&bridge_out_global_stats.initial_amount).unwrap_or_default(); + initial_amount.add_assign(&escrow_data.amount); + bridge_out_global_stats.initial_amount = initial_amount.to_string(); + bridge_out_global_stats.initial_txn += 1; + storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; + info!( + "swap initialize stats included: tx_hash={}, escrow_hash={}, token={}, amount={}", + event.transaction_hash, escrow_hash, escrow_data.token, escrow_data.amount, + ); + } else if initialized { + info!( + "swap initialize stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}, token={}, expect_token={}", + event.transaction_hash, escrow_hash, escrow_data.token, gateway_peg_btc_address, + ); + } else { + info!( + "swap initialize ignored for resolved or previously initialized escrow {escrow_hash}" + ); + } + } + Ok(()) +} + +async fn handle_swap_claim_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + goat_client: Arc, + swap_contract_address: &EvmAddress, + gateway_peg_btc_address: &EvmAddress, + claim_events: Vec, +) -> anyhow::Result<()> { + for event in claim_events { + let Some(claim_data) = extract_claim_data_from_tx( + &goat_client, + &event.transaction_hash, + swap_contract_address, + &hex::decode(strip_hex_prefix_owned(&event.escrow_hash))?.try_into().map_err( + |v: Vec| anyhow::anyhow!("escrow_hash length is {}, expected 32", v.len()), + )?, + ) + .await? + else { + warn!("failed to parse claim_data for event:{event:?}"); + continue; + }; + let escrow_hash = normalize_escrow_hash(&event.escrow_hash); + let Some(escrow) = storage_processor.find_swap_escrow(&escrow_hash).await? else { + warn!( + "swap claim ignored for unknown escrow {escrow_hash}, tx_hash={}", + event.transaction_hash + ); + continue; + }; + let btc_addr = Address::from_script( + bitcoin::Script::from_bytes(&claim_data.output_script), + get_network(), + ) + .map(|address| address.to_string()) + .unwrap_or_else(|error| { + warn!( + escrow_hash = %escrow_hash, + tx_hash = %event.transaction_hash, + output_script = %hex::encode(&claim_data.output_script), + %error, + "swap claim output script is not a Bitcoin address; storing an empty display address" + ); + String::new() + }); + // The claim event is authoritative: it also overrides a locally + // derived Timeout. + let transitioned = storage_processor + .update_swap_escrow( + &SwapEscrowUpdate::new(escrow_hash.clone()) + .with_status(SwapEscrowStatus::Claim.to_string()) + .with_claim_tx_hash(event.transaction_hash.clone()) + .with_claim_btc_txid(claim_data.txid.into()) + .with_btc_addr(btc_addr) + .with_only_if_status_in(vec![ + SwapEscrowStatus::Initialize.to_string(), + SwapEscrowStatus::Timeout.to_string(), + ]), + ) + .await?; + if !transitioned { + info!( + "swap claim ignored for resolved escrow {escrow_hash} with status {}", + escrow.status + ); + continue; + } + if escrow.token == gateway_peg_btc_address.to_string() { + let mut bridge_out_global_stats = + get_bridge_out_global_stats(storage_processor).await?; + let mut claim_amount = + U256::from_str(&bridge_out_global_stats.claim_amount).unwrap_or_default(); + claim_amount.add_assign(&U256::from_str(&escrow.amount).unwrap_or_default()); + bridge_out_global_stats.claim_amount = claim_amount.to_string(); + bridge_out_global_stats.claim_txn += 1; + storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; + info!( + "swap claim stats included: tx_hash={}, escrow_hash={}, amount={}", + event.transaction_hash, escrow_hash, escrow.amount, + ); + } else { + info!( + "swap claim stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}", + event.transaction_hash, escrow_hash, + ); + } + } + Ok(()) +} + +async fn handle_swap_refund_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + gateway_peg_btc_address: &EvmAddress, + refund_events: Vec, +) -> anyhow::Result<()> { + for event in refund_events { + let escrow_hash = normalize_escrow_hash(&event.escrow_hash); + let Some(escrow) = storage_processor.find_swap_escrow(&escrow_hash).await? else { + warn!( + "swap refund ignored for unknown escrow {escrow_hash}, tx_hash={}", + event.transaction_hash + ); + continue; + }; + // The refund event is authoritative: it also overrides a locally + // derived Timeout. + let transitioned = storage_processor + .update_swap_escrow( + &SwapEscrowUpdate::new(escrow_hash.clone()) + .with_status(SwapEscrowStatus::Refund.to_string()) + .with_refund_tx_hash(event.transaction_hash.clone()) + .with_only_if_status_in(vec![ + SwapEscrowStatus::Initialize.to_string(), + SwapEscrowStatus::Timeout.to_string(), + ]), + ) + .await?; + if !transitioned { + info!( + "swap refund ignored for resolved escrow {escrow_hash} with status {}", + escrow.status + ); + continue; + } + if escrow.token == gateway_peg_btc_address.to_string() { + let mut bridge_out_global_stats = + get_bridge_out_global_stats(storage_processor).await?; + let mut refund_amount = + U256::from_str(&bridge_out_global_stats.refund_amount).unwrap_or_default(); + refund_amount.add_assign(&U256::from_str(&escrow.amount).unwrap_or_default()); + bridge_out_global_stats.refund_amount = refund_amount.to_string(); + bridge_out_global_stats.refund_txn += 1; + storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; + info!( + "swap refund stats included: tx_hash={}, escrow_hash={}, amount={}", + event.transaction_hash, escrow_hash, escrow.amount, + ); + } else { + info!( + "swap refund stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}", + event.transaction_hash, escrow_hash, + ); + } + } + Ok(()) +} diff --git a/node/src/scheduled_tasks/event_watch/gateway.rs b/node/src/scheduled_tasks/event_watch/gateway.rs new file mode 100644 index 000000000..c55e13932 --- /dev/null +++ b/node/src/scheduled_tasks/event_watch/gateway.rs @@ -0,0 +1,774 @@ +//! Gateway events: withdraws, bridge-ins, committee answers and posted graphs. + +use crate::aggregates::graph::{graph_posted_event, graph_pull_event}; +use crate::aggregates::graph_fetch::GraphLookup; +use crate::aggregates::graph_pegout::{ + goat_status_event, withdraw_cancel_event, withdraw_init_event, +}; +use crate::aggregates::instance::l2_minted_event; +use crate::clock::current_time_secs; +use crate::engine_bridge::record_event_in; +use crate::utils::{ + generate_instance_from_bridge_in_request_event, obsolete_instance_graphs_except, + reflect_goat_address, strip_hex_prefix_owned, +}; +use alloy::primitives::U256; +use bitvm_lib::actors::Actor; +use client::btc_chain::BTCClient; +use client::goat_chain::GOATClient; +use client::graphs::GraphQueryClient; +use client::graphs::graph_query::{ + BlockRange, BridgeInEvent, BridgeInRequestEvent, CancelWithdrawEvent, CommitteeResponseEvent, + GatewayEventEntity, InitWithdrawEvent, PostGraphDataEvent, ProceedWithdrawEvent, + UserGraphWithdrawEvent, WithdrawDisprovedEvent, WithdrawHappyEvent, WithdrawPathsEvent, + WithdrawUnhappyEvent, get_gateway_events_query, +}; +use std::ops::AddAssign; +use std::str::FromStr; +use std::sync::Arc; +use store::localdb::{GraphRuntimeUpdate, NodeQuery, StorageProcessor}; +use store::{ + GoatTxProcessingStatus, GoatTxRecord, GoatTxType, GraphStatus, GraphStatusSource, + GraphStatusTransitionOutcome, InstanceBridgeInStatus, +}; +use tracing::{info, warn}; +use uuid::Uuid; + +#[allow(clippy::too_many_arguments)] +pub async fn fetch_and_handle_gateway_events<'a>( + actor: Actor, + btc_client: Arc, + goat_client: Arc, + client: &GraphQueryClient, + storage_processor: &mut StorageProcessor<'a>, + graph_url: &str, + event_entities: &[GatewayEventEntity], + from_height: i64, + to_height: i64, +) -> anyhow::Result<()> { + let query_res = client + .execute_query( + graph_url, + &get_gateway_events_query( + event_entities, + Some(BlockRange::new(from_height, to_height)), + ), + ) + .await?; + + let mut init_withdraw_events: Vec = vec![]; + let mut cancel_withdraw_events = vec![]; + let mut proceed_withdraw_events: Vec = vec![]; + let mut withdraw_paths_events: Vec = vec![]; + let mut withdraw_disproved_events: Vec = vec![]; + let mut bridge_in_request_events: Vec = vec![]; + let mut committee_response_events: Vec = vec![]; + let mut bridge_in_events: Vec = vec![]; + let mut post_graph_data_events: Vec = vec![]; + for event_entity in event_entities { + let entity = event_entity.clone(); + if let Some(value_vec) = query_res[entity.to_string()].as_array() { + match entity { + GatewayEventEntity::InitWithdraws => { + init_withdraw_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::CancelWithdraws => { + cancel_withdraw_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::ProceedWithdraws => { + proceed_withdraw_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::WithdrawHappyPaths => { + let events: Vec = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + let mut events: Vec = + events.into_iter().map(WithdrawPathsEvent::WithdrawHappyEvent).collect(); + withdraw_paths_events.append(&mut events); + } + GatewayEventEntity::WithdrawUnhappyPaths => { + let events: Vec = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + let mut events: Vec = + events.into_iter().map(WithdrawPathsEvent::WithdrawUnhappyEvent).collect(); + withdraw_paths_events.append(&mut events); + } + GatewayEventEntity::WithdrawDisproveds => { + withdraw_disproved_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::BridgeInRequests => { + bridge_in_request_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::CommitteeResponses => { + committee_response_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::BridgeIns => { + bridge_in_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + GatewayEventEntity::PostGraphDatas => { + post_graph_data_events = + serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; + } + }; + } + } + let event_counts = [ + init_withdraw_events.len(), + cancel_withdraw_events.len(), + proceed_withdraw_events.len(), + withdraw_paths_events.len(), + withdraw_disproved_events.len(), + bridge_in_request_events.len(), + committee_response_events.len(), + bridge_in_events.len(), + post_graph_data_events.len(), + ]; + let summary = format!( + "get user init withdraw events: {}, cancel withdraw events: {}, proceed_withdraw_events: {}, \ + withdraw_paths_events: {}, withdraw_disproved_events: {}, bridge_in_request_events: {} \ + committee_response_events: {}, bridge_in_events: {} post_graph_data_events: {} block range {from_height}:{to_height}", + event_counts[0], + event_counts[1], + event_counts[2], + event_counts[3], + event_counts[4], + event_counts[5], + event_counts[6], + event_counts[7], + event_counts[8], + ); + // An empty range every few seconds is the normal idle state: keep it out of the INFO log. + if event_counts.iter().any(|count| *count > 0) { + info!("{summary}"); + } else { + tracing::debug!("{summary}"); + } + handle_user_withdraw_events(storage_processor, init_withdraw_events, cancel_withdraw_events) + .await?; + handle_proceed_withdraw_events(storage_processor, proceed_withdraw_events).await?; + handle_withdraw_paths_events(storage_processor, withdraw_paths_events).await?; + handle_withdraw_disproved_events(storage_processor, withdraw_disproved_events).await?; + handle_bridge_in_request_events(storage_processor, bridge_in_request_events).await?; + handle_committee_response_events(storage_processor, committee_response_events).await?; + handle_bridge_in_events( + storage_processor, + btc_client.clone(), + goat_client.clone(), + bridge_in_events, + ) + .await?; + handle_post_graph_data_events(&actor, storage_processor, post_graph_data_events).await?; + Ok(()) +} + +async fn handle_user_withdraw_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + init_withdraw_events: Vec, + cancel_withdraw_events: Vec, +) -> anyhow::Result<()> { + // Within a block, cancels go first: a cancel and a new init of the same + // graph in one block must leave the new withdraw active. The opposite + // order (an init cancelled in its own block) leaves a stale withdraw + // active, which the kickoff ignores after checking it on Goat. + let mut user_withdraw_events: Vec = + cancel_withdraw_events.into_iter().map(UserGraphWithdrawEvent::CancelWithdraw).collect(); + user_withdraw_events + .extend(init_withdraw_events.into_iter().map(UserGraphWithdrawEvent::InitWithdraw)); + user_withdraw_events.sort_by_key(|v| v.get_block_number()); + for event in user_withdraw_events { + match event { + UserGraphWithdrawEvent::InitWithdraw(init_event) => { + let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&init_event.instance_id))?; + let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&init_event.graph_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + if !storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(instance_id, graph_id) + .with_bridge_out_start_at(current_time_secs()) + .with_init_withdraw_tx_hash(init_event.transaction_hash.clone()), + ) + .await? + { + warn!("ignore InitWithdraw for missing graph {instance_id}:{graph_id}"); + continue; + } + // The Graph aggregate takes the withdraw from here. + record_event_in( + storage_processor, + withdraw_init_event(graph_id, &init_event.transaction_hash)?, + ) + .await?; + storage_processor + .upsert_goat_tx_record(&GoatTxRecord { + instance_id, + graph_id, + tx_type: GoatTxType::InitWithdraw.to_string(), + tx_hash: init_event.transaction_hash, + height: init_event.block_number.parse::()?, + is_local: false, + processing_status: GoatTxProcessingStatus::Processed.to_string(), + extra: None, + created_at: current_time_secs(), + }) + .await? + } + UserGraphWithdrawEvent::CancelWithdraw(cancel_event) => { + let instance_id = + Uuid::from_str(&strip_hex_prefix_owned(&cancel_event.instance_id))?; + let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&cancel_event.graph_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + if !storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(instance_id, graph_id) + .with_bridge_out_start_at(0) + .with_init_withdraw_tx_hash("".to_string()), + ) + .await? + { + warn!("ignore CancelWithdraw for missing graph {instance_id}:{graph_id}"); + continue; + } + record_event_in( + storage_processor, + withdraw_cancel_event(graph_id, &cancel_event.transaction_hash)?, + ) + .await?; + storage_processor + .upsert_goat_tx_record(&GoatTxRecord { + instance_id, + graph_id, + tx_type: GoatTxType::CancelWithdraw.to_string(), + tx_hash: cancel_event.transaction_hash, + height: cancel_event.block_number.parse::()?, + is_local: false, + processing_status: GoatTxProcessingStatus::Skipped.to_string(), + extra: None, + created_at: current_time_secs(), + }) + .await? + } + } + } + Ok(()) +} + +async fn handle_proceed_withdraw_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + proceed_withdraw_events: Vec, +) -> anyhow::Result<()> { + // The record only marks the event as seen; nothing reads its status. + let processing_status = GoatTxProcessingStatus::Skipped.to_string(); + for event in proceed_withdraw_events { + let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id))?; + let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + let height = event.block_number.parse::()?; + if !storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(instance_id, graph_id) + .with_proceed_withdraw_height(height), + ) + .await? + { + warn!("ignore ProceedWithdraw for missing graph {instance_id}:{graph_id}"); + continue; + } + storage_processor + .upsert_goat_tx_record(&GoatTxRecord { + instance_id, + graph_id, + tx_type: GoatTxType::ProceedWithdraw.to_string(), + tx_hash: event.transaction_hash, + height, + is_local: false, + processing_status: processing_status.clone(), + extra: None, + created_at: current_time_secs(), + }) + .await?; + } + Ok(()) +} + +async fn apply_gateway_graph_status<'a>( + storage_processor: &mut StorageProcessor<'a>, + instance_id: Uuid, + graph_id: Uuid, + target: GraphStatus, +) -> anyhow::Result { + match storage_processor + .transition_graph_status(instance_id, graph_id, target, GraphStatusSource::GoatEvent, None) + .await? + { + outcome @ (GraphStatusTransitionOutcome::Applied + | GraphStatusTransitionOutcome::AlreadyCurrent) => Ok(outcome), + GraphStatusTransitionOutcome::Rejected { current } => { + warn!( + "ignore stale gateway graph event: graph={graph_id}, target={target}, current={current}" + ); + Ok(GraphStatusTransitionOutcome::Rejected { current }) + } + GraphStatusTransitionOutcome::NotFound => { + warn!("no local row for graph {graph_id}; gateway event ignored: target={target}"); + Ok(GraphStatusTransitionOutcome::NotFound) + } + } +} + +async fn graph_belongs_to_instance<'a>( + storage_processor: &mut StorageProcessor<'a>, + instance_id: Uuid, + graph_id: Uuid, +) -> anyhow::Result { + match storage_processor.find_graph(&graph_id).await? { + Some(graph) if graph.instance_id == instance_id => Ok(true), + Some(graph) => { + warn!( + "ignore gateway event with mismatched graph instance: graph={graph_id}, event_instance={instance_id}, stored_instance={}", + graph.instance_id + ); + Ok(false) + } + None => { + warn!("ignore gateway event for missing graph {instance_id}:{graph_id}"); + Ok(false) + } + } +} + +async fn handle_withdraw_paths_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + withdraw_paths_events: Vec, +) -> anyhow::Result<()> { + for event in withdraw_paths_events { + let (graph_id, instance_id, tx_type, status) = match event.clone() { + WithdrawPathsEvent::WithdrawHappyEvent(v) => ( + v.graph_id.clone(), + v.instance_id.clone(), + GoatTxType::WithdrawHappyPath.to_string(), + GraphStatus::OperatorTake1, + ), + WithdrawPathsEvent::WithdrawUnhappyEvent(v) => ( + v.graph_id.clone(), + v.instance_id.clone(), + GoatTxType::WithdrawUnhappyPath.to_string(), + GraphStatus::OperatorTake2, + ), + }; + let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&graph_id))?; + let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&instance_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + + let reward_add = U256::from_str(&event.reward_amount_str()).unwrap_or_default(); + let (flag, goat_addr) = reflect_goat_address(Some(event.operator_addr())); + if !flag { + warn!( + "handle_withdraw_paths_events failed as cast operator address failed, detail: {}, {}", + event.tx_hash(), + event.operator_addr() + ); + continue; + } + let outcome = + apply_gateway_graph_status(storage_processor, instance_id, graph_id, status).await?; + if !matches!( + outcome, + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent + ) { + continue; + } + let obsoleted_graph_ids = + obsolete_instance_graphs_except(storage_processor, instance_id, Some(graph_id)).await?; + if !obsoleted_graph_ids.is_empty() { + info!( + instance_id = %instance_id, + completed_graph_id = %graph_id, + obsoleted_graph_ids = ?obsoleted_graph_ids, + "marked other instance graphs obsolete after completed withdrawal" + ); + } + let is_new_event = storage_processor + .find_graph_goat_tx_record(&instance_id, &graph_id, &tx_type) + .await? + .is_none(); + storage_processor + .upsert_goat_tx_record(&GoatTxRecord { + instance_id, + graph_id, + tx_type: tx_type.clone(), + tx_hash: event.tx_hash(), + height: event.get_block_number(), + is_local: false, + processing_status: GoatTxProcessingStatus::Pending.to_string(), + extra: None, + created_at: current_time_secs(), + }) + .await?; + if is_new_event { + add_node_reward(storage_processor, &goat_addr.unwrap(), reward_add).await?; + } + // The aggregate mirrors the settlement and releases the next slot. + record_event_in(storage_processor, goat_status_event(graph_id, status, &event.tx_hash())?) + .await?; + } + Ok(()) +} + +async fn handle_withdraw_disproved_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + withdraw_disproved_events: Vec, +) -> anyhow::Result<()> { + for event in withdraw_disproved_events { + let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id))?; + let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; + if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { + continue; + } + let (flag, verifier_addr) = reflect_goat_address(Some(event.challenger_addr.clone())); + if !flag { + warn!( + "handle_withdraw_disproved_events failed as cast verifier address failed, detail: {}, {}", + event.transaction_hash, event.challenger_addr + ); + continue; + } + let (flag, disprover_addr) = reflect_goat_address(Some(event.disprover_addr.clone())); + if !flag { + warn!( + "handle_withdraw_disproved_events failed as cast disprover address failed, detail: {}, {}", + event.transaction_hash, event.disprover_addr + ); + continue; + } + + let outcome = apply_gateway_graph_status( + storage_processor, + instance_id, + graph_id, + GraphStatus::Disprove, + ) + .await?; + if !matches!( + outcome, + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent + ) { + continue; + } + let tx_type = GoatTxType::WithdrawDisproved.to_string(); + let is_new_event = storage_processor + .find_graph_goat_tx_record(&instance_id, &graph_id, &tx_type) + .await? + .is_none(); + storage_processor + .upsert_goat_tx_record(&GoatTxRecord { + instance_id, + graph_id, + tx_type, + tx_hash: event.transaction_hash.clone(), + height: event.block_number.parse::()?, + is_local: false, + processing_status: GoatTxProcessingStatus::Pending.to_string(), + extra: None, + created_at: current_time_secs(), + }) + .await?; + if is_new_event { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_pegout_disprove(); + } + add_node_reward( + storage_processor, + &verifier_addr.unwrap(), + U256::from_str(&event.challenger_amount_sats).unwrap_or_default(), + ) + .await?; + add_node_reward( + storage_processor, + &disprover_addr.unwrap(), + U256::from_str(&event.disprover_amount_sats).unwrap_or_default(), + ) + .await?; + } + record_event_in( + storage_processor, + goat_status_event(graph_id, GraphStatus::Disprove, &event.transaction_hash)?, + ) + .await?; + } + Ok(()) +} + +async fn handle_bridge_in_request_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + bridge_in_request_events: Vec, +) -> anyhow::Result<()> { + for event in bridge_in_request_events { + storage_processor + .upsert_goat_tx_record(&GoatTxRecord { + instance_id: Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?, + graph_id: Uuid::nil(), + tx_type: GoatTxType::BridgeInRequest.to_string(), + tx_hash: event.transaction_hash.clone(), + height: event.block_number.parse::()?, + is_local: false, + processing_status: GoatTxProcessingStatus::Pending.to_string(), + extra: Some(serde_json::to_string(&event)?), + created_at: current_time_secs(), + }) + .await?; + } + Ok(()) +} + +async fn handle_committee_response_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + committee_response_events: Vec, +) -> anyhow::Result<()> { + for event in committee_response_events { + if let Ok(instance_id) = &Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)) { + // Convert hex string to [u8; 32] + if let Ok(pubkey_bytes) = hex::decode(&event.committee_pubkey) { + if pubkey_bytes.len() == 33 { + let mut pubkey = [0u8; 33]; + pubkey.copy_from_slice(&pubkey_bytes); + storage_processor + .update_instance_committee_answer( + instance_id, + &event.committee_address, + pubkey.to_vec(), + ) + .await?; + } else { + warn!("committee_pubkey length is not 33 bytes: {}", event.committee_pubkey); + } + } else { + warn!("failed to decode committee_pubkey: {}", event.committee_pubkey); + } + } else { + warn!("failed to parse instance id:{event:?}"); + } + } + Ok(()) +} + +async fn handle_bridge_in_events<'a>( + storage_processor: &mut StorageProcessor<'a>, + btc_client: Arc, + goat_client: Arc, + bridge_in_events: Vec, +) -> anyhow::Result<()> { + for event in bridge_in_events { + let post_pegin_txhash = event.transaction_hash.clone(); + let Ok(instance_id) = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)) else { + warn!("failed to parse instance id of BridgeIn event: {event:?}"); + continue; + }; + let mut row_exists = storage_processor.find_instance(&instance_id).await?.is_some(); + if !row_exists + && let Some(tx_record) = storage_processor + .find_graph_goat_tx_record( + &instance_id, + &Uuid::nil(), + &GoatTxType::BridgeInRequest.to_string(), + ) + .await? + { + // A request seen in history whose instance was never built: build + // it in its final state. + warn!("Instance {instance_id} is finished but not find in db. we will create it"); + if let Some(extra) = tx_record.extra.as_ref() + && let Ok(bridge_event) = serde_json::from_str::(extra) + { + match generate_instance_from_bridge_in_request_event( + btc_client.as_ref(), + goat_client.as_ref(), + &bridge_event, + ) + .await + { + Ok(mut instance) => { + info!( + "Instance {instance_id} is created and set status to RelayerL2Minted" + ); + instance.status = InstanceBridgeInStatus::RelayerL2Minted.to_string(); + instance.post_pegin_txhash = Some(post_pegin_txhash.clone()); + if !storage_processor.insert_instance_if_absent(&instance).await? { + info!( + "Instance {instance_id} was created concurrently; applying mint event" + ); + } + row_exists = true; + } + Err(error) => warn!( + event = "event_watch", + instance_id = %instance_id, + error = %error, + "could not build the instance of a BridgeIn; mint event skipped" + ), + } + } + + if tx_record.processing_status == GoatTxProcessingStatus::Pending.to_string() { + info!( + "Instance {instance_id} related goat tx BridgeInRequest set event processing status skipped" + ); + storage_processor + .update_goat_tx_record_processing_status( + &Uuid::nil(), + &instance_id, + &GoatTxType::BridgeInRequest.to_string(), + &GoatTxProcessingStatus::Skipped.to_string(), + ) + .await?; + } + } + if !row_exists { + // Without a row the event would park forever; nothing local to update. + warn!( + event = "event_watch", + instance_id = %instance_id, + tx_hash = %post_pegin_txhash, + "BridgeIn for an instance this node never saw the request of; mint event skipped" + ); + continue; + } + // The Instance aggregate applies the mint, or alerts when it holds the instance in a + // failure state. + record_event_in(storage_processor, l2_minted_event(instance_id, &post_pegin_txhash)?) + .await?; + } + Ok(()) +} + +async fn handle_post_graph_data_events<'a>( + actor: &Actor, + storage_processor: &mut StorageProcessor<'a>, + post_graph_data_events: Vec, +) -> anyhow::Result<()> { + for event in post_graph_data_events { + match ( + Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)), + Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id)), + ) { + (Ok(instance_id), Ok(graph_id)) => { + let outcome = apply_gateway_graph_status( + storage_processor, + instance_id, + graph_id, + GraphStatus::OperatorDataPushed, + ) + .await?; + // Watching roles fetch a finalized definition when the posted graph is absent + // locally. + if matches!(outcome, GraphStatusTransitionOutcome::NotFound) + && matches!(actor, Actor::Verifier | Actor::Watchtower) + { + record_event_in( + storage_processor, + graph_pull_event( + graph_id, + GraphLookup::Id { instance_id, graph_id, finalized: true }, + )?, + ) + .await?; + } + // Posted: the Graph aggregate closes every delivery of the graph. + record_event_in(storage_processor, graph_posted_event(graph_id)?).await?; + } + _ => warn!("failed to parse graph event identifiers: {event:?}"), + } + } + Ok(()) +} + +async fn add_node_reward( + storage_processor: &mut StorageProcessor<'_>, + goat_addr: &str, + add_value: U256, +) -> anyhow::Result<()> { + let (nodes, _) = storage_processor + .find_nodes(&NodeQuery::default().with_goat_addr(goat_addr.to_string())) + .await?; + for node in nodes { + let mut reward = U256::from_str(&node.reward).unwrap_or_default(); + reward.add_assign(&add_value); + storage_processor.update_node_reward_by_peer_id(&node.peer_id, &reward.to_string()).await? + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use store::localdb::LocalDB; + + async fn db() -> (LocalDB, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("watch.db"); + let db = LocalDB::new_with_busy_timeout( + &format!("sqlite:{}", path.display()), + false, + store::DEFAULT_BUSY_TIMEOUT, + ) + .await; + db.migrate().await; + (db, dir) + } + + fn posted(instance_id: Uuid, graph_id: Uuid) -> PostGraphDataEvent { + PostGraphDataEvent { + id: "1".to_string(), + transaction_hash: "0xaa".to_string(), + block_number: "1".to_string(), + instance_id: format!("0x{}", instance_id.simple()), + graph_id: format!("0x{}", graph_id.simple()), + } + } + + #[tokio::test] + async fn a_posted_graph_unknown_locally_is_pulled_by_the_watching_roles() { + let (db, _dir) = db().await; + for (actor, pulled) in [ + (Actor::Verifier, true), + (Actor::Watchtower, true), + (Actor::Operator, false), + (Actor::Committee, false), + ] { + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + let mut tx = db.start_transaction().await.unwrap(); + handle_post_graph_data_events(&actor, &mut tx, vec![posted(instance_id, graph_id)]) + .await + .unwrap(); + tx.commit().await.unwrap(); + let mut storage = db.acquire().await.unwrap(); + let pull = storage + .find_engine_inbox_event(&format!("graph:{graph_id}:pull:finalized")) + .await + .unwrap(); + assert_eq!(pull.is_some(), pulled, "{actor}"); + // The posted event is recorded for every role. + assert!( + storage + .find_engine_inbox_event(&format!("graph:{graph_id}:posted")) + .await + .unwrap() + .is_some(), + "{actor}" + ); + } + } +} diff --git a/node/src/scheduled_tasks/event_watch/mod.rs b/node/src/scheduled_tasks/event_watch/mod.rs new file mode 100644 index 000000000..9d028c720 --- /dev/null +++ b/node/src/scheduled_tasks/event_watch/mod.rs @@ -0,0 +1,636 @@ +//! Goat event watching: the history sync and the live monitor over the +//! gateway and swap contracts, with the handlers in `gateway` and +//! `bridge_out`. + +mod bridge_out; +mod gateway; + +use crate::clock::current_time_secs; +use crate::env::{ + ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, ENV_GOAT_SWAP_CONTRACT_ADDRESS, + LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS, get_goat_address_from_env, + get_goat_gateway_event_filter_from_from_env, get_goat_gateway_event_filter_gap_from_env, + get_goat_gateway_the_graph_urls_from_env, get_goat_swap_event_filter_from_from_env, + get_goat_swap_event_filter_gap_from_env, get_goat_swap_the_graph_urls_from_env, +}; +use crate::metrics_service::{EventWatchState, MetricsState}; +use alloy::primitives::Address as EvmAddress; +use bitvm_lib::actors::Actor; +use client::btc_chain::BTCClient; +use client::goat_chain::{GOATClient, GoatInitConfig}; +use client::graphs::GraphQueryClient; +use client::graphs::graph_query::{ + GatewayEventEntity, SwapConfig, SwapEventEntity, TheGraphConfig, WatchContractType, + WatchEventConfig, +}; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::{Duration, Instant}; +use store::localdb::{LocalDB, StorageProcessor}; +use store::{WatchContract, WatchContractStatus}; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; +use tracing::{info, warn}; + +use bridge_out::fetch_and_handle_bridge_out_events; +use gateway::fetch_and_handle_gateway_events; + +/// How long a history sync waits when the indexer tip is below its cursor. +const HISTORY_SYNC_TIP_BEHIND_WAIT: Duration = Duration::from_secs(5); + +/// Contracts whose history sync task runs in this process. +static HISTORY_SYNC_IN_FLIGHT: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + +/// A contract's history-sync slot; dropping it, also when the task panics, +/// frees the slot. +struct HistorySyncGuard(String); + +impl HistorySyncGuard { + fn try_acquire(contract_addr: &str) -> Option { + let mut in_flight = + HISTORY_SYNC_IN_FLIGHT.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + in_flight.insert(contract_addr.to_string()).then(|| Self(contract_addr.to_string())) + } +} + +impl Drop for HistorySyncGuard { + fn drop(&mut self) { + HISTORY_SYNC_IN_FLIGHT + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&self.0); + } +} + +async fn refresh_event_watch_metrics( + local_db: &LocalDB, + goat_client: &GOATClient, + metrics_state: &MetricsState, +) -> anyhow::Result<()> { + let finalized_height = goat_client.get_finalized_block_number().await.inspect_err(|error| { + metrics_state.backend_probe_failed("goat", "get_finalized_block_number", error); + })?; + metrics_state.record_goat_backend_probe(true); + + let snapshot = async { + let mut storage = local_db.acquire().await?; + storage.event_watch_metrics_snapshot(finalized_height).await + } + .await + .map_err(|error| { + metrics_state.record_db_error(&error); + warn!(event = "metrics_event_watch", error = %error, "failed to collect event watcher state"); + error + })?; + + metrics_state.apply_event_watch_lag(snapshot.lag_blocks); + let state = if snapshot.failed_count > 0 { + EventWatchState::Failed + } else if snapshot.syncing_count > 0 { + EventWatchState::Syncing + } else { + EventWatchState::Healthy + }; + metrics_state.set_event_watch_state(state); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub async fn fetch_and_handle_block_range_events<'a>( + actor: Actor, + btc_client: Arc, + goat_client: Arc, + client: &GraphQueryClient, + storage_processor: &mut StorageProcessor<'a>, + watch_events_config: &WatchEventConfig, + from_height: i64, + to_height: i64, +) -> anyhow::Result<()> { + match watch_events_config { + WatchEventConfig::Gateway(config) => { + fetch_and_handle_gateway_events( + actor, + btc_client, + goat_client.clone(), + client, + storage_processor, + &config.the_graph_url, + &config.event_entities, + from_height, + to_height, + ) + .await?; + } + WatchEventConfig::Swap(config) => { + fetch_and_handle_bridge_out_events( + goat_client.clone(), + client, + storage_processor, + &config.graph_config.address, + &config.graph_config.the_graph_url, + &config.graph_config.event_entities, + from_height, + to_height, + config.peg_btc_address, + ) + .await?; + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub async fn fetch_history_events( + actor: Actor, + btc_client: Arc, + goat_client: Arc, + local_db: &LocalDB, + query_client: &GraphQueryClient, + watch_contract: WatchContract, + watch_events_config: WatchEventConfig, +) -> Result<(), Box> { + let task_name = watch_events_config.get_watch_contract_type().to_string(); + info!("Start into fetch_history_events from:{} for {task_name}", watch_contract.from_height); + // let goat_client = GOATClient::new(env::goat_config_from_env().await, env::get_goat_network()); + let mut watch_contract = watch_contract.clone(); + let local_db_clone = local_db.clone(); + let contract_addr = watch_contract.contract_addr.clone(); + let async_fn = || async move { + let task_name = watch_events_config.get_watch_contract_type().to_string(); + loop { + // let current_finalized = goat_client.get_finalized_block_number().await; + let current_finalized = match query_client + .get_sync_block_height(&watch_contract.the_graph_url) + .await + { + Ok(Some(v)) => v, + Ok(None) => { + warn!( + "fetch_history_events:fail to get graph sync block height, will try later, empty value returned" + ); + sleep(Duration::from_millis(500)).await; + continue; + } + Err(e) => { + warn!( + "fetch_history_events:fail to get graph sync block height, will try later, err:{e:#}" + ); + sleep(Duration::from_millis(500)).await; + continue; + } + }; + + if watch_contract.from_height > current_finalized { + // The indexer tip fell below the cursor (reindex or replica lag): + // wait for it to catch up instead of spinning on the RPC. + info!( + "Contract {task_name} history sync waits for the indexer: finalized height {current_finalized} is below the watch from height {}", + watch_contract.from_height, + ); + sleep(HISTORY_SYNC_TIP_BEHIND_WAIT).await; + continue; + } + + let to_height = current_finalized.min(watch_contract.from_height + watch_contract.gap); + let mut tx = local_db.start_transaction().await?; + + fetch_and_handle_block_range_events( + actor.clone(), + btc_client.clone(), + goat_client.clone(), + query_client, + &mut tx, + &watch_events_config, + watch_contract.from_height, + to_height, + ) + .await?; + info!( + "Contract {task_name} finish load history event from: {}, to: {to_height}", + watch_contract.from_height + ); + watch_contract.from_height = to_height + 1; + watch_contract.status = WatchContractStatus::Syncing.to_string(); + watch_contract.updated_at = current_time_secs(); + + if to_height >= current_finalized { + info!("Contract {task_name} Finish load history at {to_height}"); + watch_contract.status = WatchContractStatus::Synced.to_string(); + tx.upsert_watch_contract(&watch_contract).await?; + tx.commit().await?; + break; + } + tx.upsert_watch_contract(&watch_contract).await?; + tx.commit().await?; + } + Ok::<(), Box>(()) + }; + let err = match async_fn().await { + Ok(_) => false, + Err(err) => { + warn!("{task_name} fetch_history_events failed,err:{:?}", err); + true + } + }; + if err { + let mut storage_processor = local_db_clone.acquire().await?; + let _ = storage_processor + .update_watch_contract_status( + &contract_addr, + &WatchContractStatus::Failed.to_string(), + current_time_secs(), + ) + .await; + } + Ok(()) +} + +pub async fn monitor_events( + actor: Actor, + btc_client: Arc, + goat_client: Arc, + local_db: &LocalDB, + watch_configs: Vec, +) -> anyhow::Result<()> { + for watch_config in watch_configs { + if watch_config.get_watch_events_len() == 0 { + continue; + } + monitor_events_item( + actor.clone(), + btc_client.clone(), + goat_client.clone(), + local_db, + watch_config, + ) + .await?; + } + + Ok(()) +} + +pub async fn monitor_events_item( + actor: Actor, + btc_client: Arc, + goat_client: Arc, + local_db: &LocalDB, + watch_events_config: WatchEventConfig, +) -> anyhow::Result<()> { + info!("start tick monitor_events"); + let mut storage_processor = local_db.acquire().await?; + let mut watch_contract = get_watch_contract( + &mut storage_processor, + &watch_events_config.get_watch_contract().to_string(), + watch_events_config.get_watch_contract_type(), + ) + .await?; + + let query_client = GraphQueryClient::new(); + // let current_finalized = goat_client.get_finalized_block_number().await?; + let current_finalized = match query_client + .get_sync_block_height(&watch_contract.the_graph_url) + .await + { + Ok(Some(v)) => { + crate::metrics_service::record_backend_ok("graph"); + v + } + Ok(None) => { + warn!( + "monitor_events_item:fail to get graph sync block height, will try later, empty value returned" + ); + return Ok(()); + } + Err(e) => { + crate::metrics_service::record_backend_failure( + "graph", + "get_sync_block_height", + &format!("{e:#}"), + ); + return Ok(()); + } + }; + + if watch_contract.from_height == 0 || watch_contract.from_height >= current_finalized { + // A misconfigured (zero) or not yet reachable start height: say it once at + // WARN, then at debug on every later tick. + static WATCH_RANGE_WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + let message = format!( + "watch_contract {} start height {} is zero or not below the current height {}, not do watch jobs", + watch_contract.contract_addr, watch_contract.from_height, current_finalized + ); + if WATCH_RANGE_WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) { + tracing::debug!("{message}"); + } else { + warn!("{message}"); + } + return Ok(()); + } + if watch_contract.status == WatchContractStatus::Syncing.to_string() + && watch_contract.updated_at + LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS > current_time_secs() + { + info!("Event sync not finished "); + return Ok(()); + } + if watch_contract.from_height + watch_contract.gap < current_finalized { + // One history sync per contract. The gate above sees `Syncing` only + // after the task commits its first batch, so claim the contract here, + // in this process and in its row, before the task starts. + let Some(sync_guard) = HistorySyncGuard::try_acquire(&watch_contract.contract_addr) else { + info!("Event sync not finished "); + return Ok(()); + }; + watch_contract.status = WatchContractStatus::Syncing.to_string(); + watch_contract.updated_at = current_time_secs(); + storage_processor.upsert_watch_contract(&watch_contract).await?; + let watch_contract_clone = watch_contract.clone(); + let local_db_clone = local_db.clone(); + let query_client_clone = query_client.clone(); + let watch_events_config_clone = watch_events_config.clone(); + tokio::spawn(async move { + let _sync_guard = sync_guard; + if let Err(error) = fetch_history_events( + actor.clone(), + btc_client.clone(), + goat_client, + &local_db_clone, + &query_client_clone, + watch_contract_clone, + watch_events_config_clone, + ) + .await + { + warn!(event = "event_watch_history_sync", error = %error, "event watcher history sync task exited unexpectedly"); + } + }); + return Ok(()); + } + + let to_height = current_finalized.min(watch_contract.from_height + watch_contract.gap); + let mut tx = local_db.start_transaction().await?; + fetch_and_handle_block_range_events( + actor.clone(), + btc_client, + goat_client, + &query_client, + &mut tx, + &watch_events_config, + watch_contract.from_height, + to_height, + ) + .await?; + info!("finish monitor event from: {}, to: {to_height}", watch_contract.from_height); + watch_contract.from_height = to_height + 1; + watch_contract.status = WatchContractStatus::Synced.to_string(); + watch_contract.updated_at = current_time_secs(); + tx.upsert_watch_contract(&watch_contract).await?; + tx.commit().await?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub async fn run_watch_event_task( + actor: Actor, + local_db: LocalDB, + btc_client: Arc, + goat_client: Arc, + interval: u64, + cancellation_token: CancellationToken, + goat_init_config: GoatInitConfig, + metrics_state: MetricsState, +) -> anyhow::Result { + let gateway_contract: EvmAddress = get_goat_address_from_env(ENV_GOAT_GATEWAY_CONTRACT_ADDRESS) + .ok_or(anyhow::anyhow!("need to set gateway contract address"))?; + let swap_contract: EvmAddress = get_goat_address_from_env(ENV_GOAT_SWAP_CONTRACT_ADDRESS) + .ok_or(anyhow::anyhow!("need to set swap contract address"))?; + let events_map: HashMap> = HashMap::from([ + ( + Actor::Committee, + vec![ + WatchEventConfig::Gateway(TheGraphConfig { + address: gateway_contract, + the_graph_url: get_goat_gateway_the_graph_urls_from_env(), + event_entities: vec![ + GatewayEventEntity::InitWithdraws, + GatewayEventEntity::CancelWithdraws, + GatewayEventEntity::ProceedWithdraws, + GatewayEventEntity::WithdrawHappyPaths, + GatewayEventEntity::WithdrawUnhappyPaths, + GatewayEventEntity::WithdrawDisproveds, + GatewayEventEntity::BridgeInRequests, + GatewayEventEntity::BridgeIns, + GatewayEventEntity::PostGraphDatas, + ], + }), + WatchEventConfig::Swap(SwapConfig { + graph_config: TheGraphConfig { + address: swap_contract, + the_graph_url: get_goat_swap_the_graph_urls_from_env(), + event_entities: vec![ + SwapEventEntity::Initializes, + SwapEventEntity::Claims, + SwapEventEntity::Refunds, + ], + }, + peg_btc_address: goat_init_config.peg_btc_address.expect( + "peg_btc_address must be set (requires GOAT_GATEWAY_CONTRACT_ADDRESS)", + ), + }), + ], + ), + ( + Actor::Operator, + vec![WatchEventConfig::Gateway(TheGraphConfig { + address: gateway_contract, + the_graph_url: get_goat_gateway_the_graph_urls_from_env(), + event_entities: vec![ + GatewayEventEntity::InitWithdraws, + GatewayEventEntity::CancelWithdraws, + GatewayEventEntity::ProceedWithdraws, + GatewayEventEntity::WithdrawHappyPaths, + GatewayEventEntity::WithdrawUnhappyPaths, + GatewayEventEntity::WithdrawDisproveds, + GatewayEventEntity::BridgeInRequests, + GatewayEventEntity::BridgeIns, + GatewayEventEntity::PostGraphDatas, + ], + })], + ), + ( + Actor::Verifier, + vec![WatchEventConfig::Gateway(TheGraphConfig { + address: gateway_contract, + the_graph_url: get_goat_gateway_the_graph_urls_from_env(), + event_entities: vec![ + GatewayEventEntity::InitWithdraws, + GatewayEventEntity::CancelWithdraws, + GatewayEventEntity::ProceedWithdraws, + GatewayEventEntity::WithdrawHappyPaths, + GatewayEventEntity::WithdrawUnhappyPaths, + GatewayEventEntity::WithdrawDisproveds, + GatewayEventEntity::BridgeInRequests, + GatewayEventEntity::BridgeIns, + GatewayEventEntity::PostGraphDatas, + ], + })], + ), + ( + Actor::Watchtower, + vec![WatchEventConfig::Gateway(TheGraphConfig { + address: gateway_contract, + the_graph_url: get_goat_gateway_the_graph_urls_from_env(), + event_entities: vec![ + GatewayEventEntity::InitWithdraws, + GatewayEventEntity::CancelWithdraws, + GatewayEventEntity::ProceedWithdraws, + GatewayEventEntity::WithdrawHappyPaths, + GatewayEventEntity::WithdrawUnhappyPaths, + GatewayEventEntity::WithdrawDisproveds, + GatewayEventEntity::BridgeInRequests, + GatewayEventEntity::BridgeIns, + GatewayEventEntity::PostGraphDatas, + ], + })], + ), + ]); + loop { + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(interval)) => { + let started_at = Instant::now(); + // Execute the normal monitoring logic + match monitor_events( + actor.clone(), + btc_client.clone(), + goat_client.clone(), + &local_db, + events_map.get(&actor).cloned().unwrap_or_default(), + ) + .await + { + Ok(_) => { + metrics_state.record_task_run( + "event_watcher", + "success", + started_at.elapsed(), + ); + let local_db = local_db.clone(); + let goat_client = goat_client.clone(); + let metrics_state = metrics_state.clone(); + tokio::spawn(async move { + if let Ok(Err(error)) = tokio::time::timeout( + Duration::from_secs(30), + refresh_event_watch_metrics( + &local_db, + goat_client.as_ref(), + &metrics_state, + ), + ) + .await + { + metrics_state.set_event_watch_state(EventWatchState::Failed); + if metrics_state.backend_unavailable("goat") { + tracing::debug!(event = "metrics_event_watch", error = %error, "failed to refresh event watcher metrics (Goat unavailable)"); + } else { + warn!(event = "metrics_event_watch", error = %error, "failed to refresh event watcher metrics"); + } + } + }); + } + Err(e) => { + metrics_state.set_event_watch_state(EventWatchState::Failed); + metrics_state.record_task_run( + "event_watcher", + "failed", + started_at.elapsed(), + ); + warn!("fail to monitor events: {e}"); + } + } + } + _ = cancellation_token.cancelled() => { + info!("Watch event task received shutdown signal"); + return Ok("watch_shutdown".to_string()); + } + } + } +} + +async fn get_watch_contract<'a>( + storage_processor: &mut StorageProcessor<'a>, + contract_addr: &str, + contract_type: WatchContractType, +) -> anyhow::Result { + let (from_height, the_graph_url, gap) = match contract_type { + WatchContractType::Gateway => ( + get_goat_gateway_event_filter_from_from_env(), + get_goat_gateway_the_graph_urls_from_env(), + get_goat_gateway_event_filter_gap_from_env(), + ), + WatchContractType::Swap => ( + get_goat_swap_event_filter_from_from_env(), + get_goat_swap_the_graph_urls_from_env(), + get_goat_swap_event_filter_gap_from_env(), + ), + }; + + if let Some(mut watch_contract) = storage_processor.find_watch_contract(contract_addr).await? { + if from_height > watch_contract.from_height { + watch_contract.from_height = from_height; + } + if the_graph_url != watch_contract.the_graph_url { + watch_contract.the_graph_url = the_graph_url; + } + + if gap != watch_contract.gap { + watch_contract.gap = gap + } + Ok(watch_contract) + } else { + Ok(WatchContract { + contract_addr: contract_addr.to_string(), + the_graph_url, + gap, + from_height, + status: WatchContractStatus::UnSync.to_string(), + extra: None, + updated_at: current_time_secs(), + created_at: current_time_secs(), + }) + } +} + +pub async fn is_processing_gateway_history_events( + local_db: &LocalDB, + goat_client: &GOATClient, +) -> anyhow::Result { + let gateway_contract: EvmAddress = get_goat_address_from_env(ENV_GOAT_GATEWAY_CONTRACT_ADDRESS) + .ok_or(anyhow::anyhow!("need to set gateway contract address"))?; + let mut storage_processor = local_db.acquire().await?; + // use finalized block height to judge is processing history events + let current_finalized = goat_client.get_finalized_block_number().await?; + let watch_contract = get_watch_contract( + &mut storage_processor, + &gateway_contract.to_string(), + WatchContractType::Gateway, + ) + .await?; + Ok(watch_contract.from_height + watch_contract.gap < current_finalized + || watch_contract.status == WatchContractStatus::Syncing.to_string()) +} + +#[cfg(test)] +mod tests { + use super::HistorySyncGuard; + + #[test] + fn history_sync_guard_is_exclusive_per_contract() { + let first = HistorySyncGuard::try_acquire("0xabc").expect("free slot"); + assert!(HistorySyncGuard::try_acquire("0xabc").is_none()); + let other = HistorySyncGuard::try_acquire("0xdef").expect("another contract is free"); + drop(first); + assert!(HistorySyncGuard::try_acquire("0xabc").is_some()); + drop(other); + } +} diff --git a/node/src/scheduled_tasks/event_watch_task.rs b/node/src/scheduled_tasks/event_watch_task.rs deleted file mode 100644 index 5066d17a9..000000000 --- a/node/src/scheduled_tasks/event_watch_task.rs +++ /dev/null @@ -1,1555 +0,0 @@ -use crate::env; -use crate::env::{ - ENV_GOAT_GATEWAY_CONTRACT_ADDRESS, ENV_GOAT_SWAP_CONTRACT_ADDRESS, - LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS, get_goat_address_from_env, - get_goat_gateway_event_filter_from_from_env, get_goat_gateway_event_filter_gap_from_env, - get_goat_gateway_the_graph_urls_from_env, get_goat_swap_event_filter_from_from_env, - get_goat_swap_event_filter_gap_from_env, get_goat_swap_the_graph_urls_from_env, get_network, -}; -use crate::metrics_service::{EventWatchState, MetricsState}; -use crate::rpc_service::current_time_secs; -use crate::scheduled_tasks::get_timestamp_from_contract_data; -use crate::utils::evm_swap_utils::{extract_claim_data_from_tx, extract_escrow_data_from_tx}; -use crate::utils::{ - GenerateInstanceParams, generate_instance, get_bridge_out_global_stats, - obsolete_instance_graphs_except, outpoint_available, reflect_goat_address, - strip_hex_prefix_owned, -}; -use alloy::primitives::{Address as EvmAddress, U256}; -use alloy::sol_types::SolValue; -use bitcoin::address::NetworkUnchecked; -use bitcoin::hashes::Hash; -use bitcoin::{Address, Amount, OutPoint, Txid}; -use bitvm_lib::actors::Actor; -use bitvm_lib::types::UserInfo; -use client::btc_chain::BTCClient; -use client::goat_chain::{GOATClient, GoatInitConfig}; -use client::graphs::GraphQueryClient; -use client::graphs::graph_query::{ - BlockRange, BridgeInEvent, BridgeInRequestEvent, CancelWithdrawEvent, CommitteeResponseEvent, - GatewayEventEntity, InitWithdrawEvent, PostGraphDataEvent, ProceedWithdrawEvent, - SwapClaimEvent, SwapConfig, SwapEventEntity, SwapInitializeEvent, SwapRefundEvent, - TheGraphConfig, UserGraphWithdrawEvent, WatchContractType, WatchEventConfig, - WithdrawDisprovedEvent, WithdrawHappyEvent, WithdrawPathsEvent, WithdrawUnhappyEvent, - get_bridge_out_events_query, get_gateway_events_query, -}; -use goat::transactions::base::Input; -use secp256k1::XOnlyPublicKey; -use std::collections::HashMap; -use std::ops::AddAssign; -use std::str::FromStr; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use store::localdb::{ - GraphRuntimeUpdate, InstanceUpdate, LocalDB, NodeQuery, StorageProcessor, SwapEscrowUpdate, -}; -use store::{ - GoatTxProcessingStatus, GoatTxRecord, GoatTxType, GraphStatus, GraphStatusSource, - GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, SwapEscrow, SwapEscrowStatus, - WatchContract, WatchContractStatus, normalize_escrow_hash, -}; -use tokio::time::sleep; -use tokio_util::sync::CancellationToken; -use tracing::{info, warn}; -use uuid::Uuid; - -async fn refresh_event_watch_metrics( - local_db: &LocalDB, - goat_client: &GOATClient, - metrics_state: &MetricsState, -) -> anyhow::Result<()> { - let finalized_height = goat_client.get_finalized_block_number().await.map_err(|error| { - metrics_state.record_goat_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "goat", operation = "get_finalized_block_number", error = %error, "event watcher lag probe failed"); - error - })?; - metrics_state.record_goat_backend_probe(true); - - let snapshot = async { - let mut storage = local_db.acquire().await?; - storage.event_watch_metrics_snapshot(finalized_height).await - } - .await - .map_err(|error| { - metrics_state.record_db_error(&error); - warn!(event = "metrics_event_watch", error = %error, "failed to collect event watcher state"); - error - })?; - - metrics_state.apply_event_watch_lag(snapshot.lag_blocks); - let state = if snapshot.failed_count > 0 { - EventWatchState::Failed - } else if snapshot.syncing_count > 0 { - EventWatchState::Syncing - } else { - EventWatchState::Healthy - }; - metrics_state.set_event_watch_state(state); - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn fetch_and_handle_block_range_events<'a>( - actor: Actor, - btc_client: Arc, - goat_client: Arc, - client: &GraphQueryClient, - storage_processor: &mut StorageProcessor<'a>, - watch_events_config: &WatchEventConfig, - from_height: i64, - to_height: i64, -) -> anyhow::Result<()> { - match watch_events_config { - WatchEventConfig::Gateway(config) => { - fetch_and_handle_gateway_events( - actor, - btc_client, - goat_client.clone(), - client, - storage_processor, - &config.the_graph_url, - &config.event_entities, - from_height, - to_height, - ) - .await?; - } - WatchEventConfig::Swap(config) => { - fetch_and_handle_bridge_out_events( - goat_client.clone(), - client, - storage_processor, - &config.graph_config.address, - &config.graph_config.the_graph_url, - &config.graph_config.event_entities, - from_height, - to_height, - config.peg_btc_address, - ) - .await?; - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn fetch_and_handle_gateway_events<'a>( - actor: Actor, - btc_client: Arc, - goat_client: Arc, - client: &GraphQueryClient, - storage_processor: &mut StorageProcessor<'a>, - graph_url: &str, - event_entities: &[GatewayEventEntity], - from_height: i64, - to_height: i64, -) -> anyhow::Result<()> { - let query_res = client - .execute_query( - graph_url, - &get_gateway_events_query( - event_entities, - Some(BlockRange::new(from_height, to_height)), - ), - ) - .await?; - - let mut init_withdraw_events: Vec = vec![]; - let mut cancel_withdraw_events = vec![]; - let mut proceed_withdraw_events: Vec = vec![]; - let mut withdraw_paths_events: Vec = vec![]; - let mut withdraw_disproved_events: Vec = vec![]; - let mut bridge_in_request_events: Vec = vec![]; - let mut committee_response_events: Vec = vec![]; - let mut bridge_in_events: Vec = vec![]; - let mut post_graph_data_events: Vec = vec![]; - for event_entity in event_entities { - let entity = event_entity.clone(); - if let Some(value_vec) = query_res[entity.to_string()].as_array() { - match entity { - GatewayEventEntity::InitWithdraws => { - init_withdraw_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::CancelWithdraws => { - cancel_withdraw_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::ProceedWithdraws => { - proceed_withdraw_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::WithdrawHappyPaths => { - let events: Vec = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - let mut events: Vec = - events.into_iter().map(WithdrawPathsEvent::WithdrawHappyEvent).collect(); - withdraw_paths_events.append(&mut events); - } - GatewayEventEntity::WithdrawUnhappyPaths => { - let events: Vec = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - let mut events: Vec = - events.into_iter().map(WithdrawPathsEvent::WithdrawUnhappyEvent).collect(); - withdraw_paths_events.append(&mut events); - } - GatewayEventEntity::WithdrawDisproveds => { - withdraw_disproved_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::BridgeInRequests => { - bridge_in_request_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::CommitteeResponses => { - committee_response_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::BridgeIns => { - bridge_in_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - GatewayEventEntity::PostGraphDatas => { - post_graph_data_events = - serde_json::from_value(serde_json::Value::Array(value_vec.clone()))?; - } - }; - } - } - info!( - "get user init withdraw events: {}, cancel withdraw events: {}, proceed_withdraw_events: {}, \ - withdraw_paths_events: {}, withdraw_disproved_events: {}, bridge_in_request_events: {} \ - committee_response_events: {}, bridge_in_events: {} post_graph_data_events: {} block range {from_height}:{to_height}", - init_withdraw_events.len(), - cancel_withdraw_events.len(), - proceed_withdraw_events.len(), - withdraw_paths_events.len(), - withdraw_disproved_events.len(), - bridge_in_request_events.len(), - committee_response_events.len(), - bridge_in_events.len(), - post_graph_data_events.len(), - ); - handle_user_withdraw_events(storage_processor, init_withdraw_events, cancel_withdraw_events) - .await?; - handle_proceed_withdraw_events(actor.clone(), storage_processor, proceed_withdraw_events) - .await?; - handle_withdraw_paths_events(storage_processor, withdraw_paths_events).await?; - handle_withdraw_disproved_events(storage_processor, withdraw_disproved_events).await?; - handle_bridge_in_request_events(storage_processor, bridge_in_request_events).await?; - handle_committee_response_events(storage_processor, committee_response_events).await?; - handle_bridge_in_events( - storage_processor, - btc_client.clone(), - goat_client.clone(), - bridge_in_events, - ) - .await?; - handle_post_graph_data_events(storage_processor, post_graph_data_events).await?; - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn fetch_and_handle_bridge_out_events<'a>( - goat_client: Arc, - client: &GraphQueryClient, - storage_processor: &mut StorageProcessor<'a>, - contract_addr: &EvmAddress, - graph_url: &str, - event_entities: &[SwapEventEntity], - from_height: i64, - to_height: i64, - gateway_peg_btc_address: EvmAddress, -) -> anyhow::Result<()> { - let query_res = client - .execute_query( - graph_url, - &get_bridge_out_events_query( - event_entities, - Some(BlockRange::new(from_height, to_height)), - ), - ) - .await?; - for event_entity in event_entities { - let entity = event_entity.clone(); - if let Some(value_vec) = query_res[entity.to_string()].as_array() { - match entity { - SwapEventEntity::Initializes => { - handle_swap_init_events( - storage_processor, - goat_client.clone(), - contract_addr, - &gateway_peg_btc_address, - serde_json::from_value::>( - serde_json::Value::Array(value_vec.clone()), - )?, - ) - .await?; - } - SwapEventEntity::Claims => { - handle_swap_claim_events( - storage_processor, - goat_client.clone(), - contract_addr, - &gateway_peg_btc_address, - serde_json::from_value::>(serde_json::Value::Array( - value_vec.clone(), - ))?, - ) - .await?; - } - SwapEventEntity::Refunds => { - handle_swap_refund_events( - storage_processor, - &gateway_peg_btc_address, - serde_json::from_value::>(serde_json::Value::Array( - value_vec.clone(), - ))?, - ) - .await?; - } - }; - } - } - Ok(()) -} - -async fn handle_user_withdraw_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - init_withdraw_events: Vec, - cancel_withdraw_events: Vec, -) -> anyhow::Result<()> { - let mut user_withdraw_events: Vec = - init_withdraw_events.into_iter().map(UserGraphWithdrawEvent::InitWithdraw).collect(); - let mut user_cancel_withdraw_events: Vec = - cancel_withdraw_events.into_iter().map(UserGraphWithdrawEvent::CancelWithdraw).collect(); - user_withdraw_events.append(&mut user_cancel_withdraw_events); - user_withdraw_events.sort_by_key(|v| v.get_block_number()); - for event in user_withdraw_events { - match event { - UserGraphWithdrawEvent::InitWithdraw(init_event) => { - let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&init_event.instance_id))?; - let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&init_event.graph_id))?; - if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { - continue; - } - if !storage_processor - .update_graph_runtime( - &GraphRuntimeUpdate::new(instance_id, graph_id) - .with_bridge_out_start_at(current_time_secs()) - .with_init_withdraw_tx_hash(init_event.transaction_hash.clone()), - ) - .await? - { - warn!("ignore InitWithdraw for missing graph {instance_id}:{graph_id}"); - continue; - } - storage_processor - .upsert_goat_tx_record(&GoatTxRecord { - instance_id, - graph_id, - tx_type: GoatTxType::InitWithdraw.to_string(), - tx_hash: init_event.transaction_hash, - height: init_event.block_number.parse::()?, - is_local: false, - processing_status: GoatTxProcessingStatus::Pending.to_string(), - extra: None, - created_at: current_time_secs(), - }) - .await? - } - UserGraphWithdrawEvent::CancelWithdraw(cancel_event) => { - let instance_id = - Uuid::from_str(&strip_hex_prefix_owned(&cancel_event.instance_id))?; - let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&cancel_event.graph_id))?; - if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { - continue; - } - if !storage_processor - .update_graph_runtime( - &GraphRuntimeUpdate::new(instance_id, graph_id) - .with_bridge_out_start_at(0) - .with_init_withdraw_tx_hash("".to_string()), - ) - .await? - { - warn!("ignore CancelWithdraw for missing graph {instance_id}:{graph_id}"); - continue; - } - storage_processor - .upsert_goat_tx_record(&GoatTxRecord { - instance_id, - graph_id, - tx_type: GoatTxType::CancelWithdraw.to_string(), - tx_hash: cancel_event.transaction_hash, - height: cancel_event.block_number.parse::()?, - is_local: false, - processing_status: GoatTxProcessingStatus::Skipped.to_string(), - extra: None, - created_at: current_time_secs(), - }) - .await? - } - } - } - Ok(()) -} - -async fn handle_proceed_withdraw_events<'a>( - actor: Actor, - storage_processor: &mut StorageProcessor<'a>, - proceed_withdraw_events: Vec, -) -> anyhow::Result<()> { - let processing_status = if actor == Actor::Operator && env::get_proof_server_url().is_none() { - GoatTxProcessingStatus::Pending.to_string() - } else { - GoatTxProcessingStatus::Skipped.to_string() - }; - for event in proceed_withdraw_events { - let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id))?; - let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; - if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { - continue; - } - let height = event.block_number.parse::()?; - if !storage_processor - .update_graph_runtime( - &GraphRuntimeUpdate::new(instance_id, graph_id) - .with_proceed_withdraw_height(height), - ) - .await? - { - warn!("ignore ProceedWithdraw for missing graph {instance_id}:{graph_id}"); - continue; - } - storage_processor - .upsert_goat_tx_record(&GoatTxRecord { - instance_id, - graph_id, - tx_type: GoatTxType::ProceedWithdraw.to_string(), - tx_hash: event.transaction_hash, - height, - is_local: false, - processing_status: processing_status.clone(), - extra: None, - created_at: current_time_secs(), - }) - .await?; - - // for history events - storage_processor - .update_goat_tx_record_processing_status( - &graph_id, - &instance_id, - &GoatTxType::InitWithdraw.to_string(), - &GoatTxProcessingStatus::Processed.to_string(), - ) - .await?; - } - Ok(()) -} - -async fn apply_gateway_graph_status<'a>( - storage_processor: &mut StorageProcessor<'a>, - instance_id: Uuid, - graph_id: Uuid, - target: GraphStatus, -) -> anyhow::Result { - match storage_processor - .transition_graph_status(instance_id, graph_id, target, GraphStatusSource::GoatEvent, None) - .await? - { - outcome @ (GraphStatusTransitionOutcome::Applied - | GraphStatusTransitionOutcome::AlreadyCurrent) => Ok(outcome), - GraphStatusTransitionOutcome::Rejected { current } => { - warn!( - "ignore stale gateway graph event: graph={graph_id}, target={target}, current={current}" - ); - Ok(GraphStatusTransitionOutcome::Rejected { current }) - } - GraphStatusTransitionOutcome::NotFound => { - warn!("ignore gateway graph event for missing graph {graph_id}: target={target}"); - Ok(GraphStatusTransitionOutcome::NotFound) - } - } -} - -async fn graph_belongs_to_instance<'a>( - storage_processor: &mut StorageProcessor<'a>, - instance_id: Uuid, - graph_id: Uuid, -) -> anyhow::Result { - match storage_processor.find_graph(&graph_id).await? { - Some(graph) if graph.instance_id == instance_id => Ok(true), - Some(graph) => { - warn!( - "ignore gateway event with mismatched graph instance: graph={graph_id}, event_instance={instance_id}, stored_instance={}", - graph.instance_id - ); - Ok(false) - } - None => { - warn!("ignore gateway event for missing graph {instance_id}:{graph_id}"); - Ok(false) - } - } -} - -async fn handle_withdraw_paths_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - withdraw_paths_events: Vec, -) -> anyhow::Result<()> { - for event in withdraw_paths_events { - let (graph_id, instance_id, tx_type, status) = match event.clone() { - WithdrawPathsEvent::WithdrawHappyEvent(v) => ( - v.graph_id.clone(), - v.instance_id.clone(), - GoatTxType::WithdrawHappyPath.to_string(), - GraphStatus::OperatorTake1, - ), - WithdrawPathsEvent::WithdrawUnhappyEvent(v) => ( - v.graph_id.clone(), - v.instance_id.clone(), - GoatTxType::WithdrawUnhappyPath.to_string(), - GraphStatus::OperatorTake2, - ), - }; - let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&graph_id))?; - let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&instance_id))?; - if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { - continue; - } - - let reward_add = U256::from_str(&event.reward_amount_str()).unwrap_or_default(); - let (flag, goat_addr) = reflect_goat_address(Some(event.operator_addr())); - if !flag { - warn!( - "handle_withdraw_paths_events failed as cast operator address failed, detail: {}, {}", - event.tx_hash(), - event.operator_addr() - ); - continue; - } - let outcome = - apply_gateway_graph_status(storage_processor, instance_id, graph_id, status).await?; - if !matches!( - outcome, - GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent - ) { - continue; - } - let obsoleted_graph_ids = - obsolete_instance_graphs_except(storage_processor, instance_id, Some(graph_id)).await?; - if !obsoleted_graph_ids.is_empty() { - info!( - instance_id = %instance_id, - completed_graph_id = %graph_id, - obsoleted_graph_ids = ?obsoleted_graph_ids, - "marked other instance graphs obsolete after completed withdrawal" - ); - } - let is_new_event = storage_processor - .find_graph_goat_tx_record(&instance_id, &graph_id, &tx_type) - .await? - .is_none(); - storage_processor - .upsert_goat_tx_record(&GoatTxRecord { - instance_id, - graph_id, - tx_type: tx_type.clone(), - tx_hash: event.tx_hash(), - height: event.get_block_number(), - is_local: false, - processing_status: GoatTxProcessingStatus::Pending.to_string(), - extra: None, - created_at: current_time_secs(), - }) - .await?; - if is_new_event { - add_node_reward(storage_processor, &goat_addr.unwrap(), reward_add).await?; - } - storage_processor.cancel_messages_by_business_id(&graph_id, None).await?; - } - Ok(()) -} - -async fn handle_withdraw_disproved_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - withdraw_disproved_events: Vec, -) -> anyhow::Result<()> { - for event in withdraw_disproved_events { - let graph_id = Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id))?; - let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; - if !graph_belongs_to_instance(storage_processor, instance_id, graph_id).await? { - continue; - } - let (flag, verifier_addr) = reflect_goat_address(Some(event.challenger_addr.clone())); - if !flag { - warn!( - "handle_withdraw_disproved_events failed as cast verifier address failed, detail: {}, {}", - event.transaction_hash, event.challenger_addr - ); - continue; - } - let (flag, disprover_addr) = reflect_goat_address(Some(event.disprover_addr.clone())); - if !flag { - warn!( - "handle_withdraw_disproved_events failed as cast disprover address failed, detail: {}, {}", - event.transaction_hash, event.disprover_addr - ); - continue; - } - - let outcome = apply_gateway_graph_status( - storage_processor, - instance_id, - graph_id, - GraphStatus::Disprove, - ) - .await?; - if !matches!( - outcome, - GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent - ) { - continue; - } - let tx_type = GoatTxType::WithdrawDisproved.to_string(); - let is_new_event = storage_processor - .find_graph_goat_tx_record(&instance_id, &graph_id, &tx_type) - .await? - .is_none(); - storage_processor - .upsert_goat_tx_record(&GoatTxRecord { - instance_id, - graph_id, - tx_type, - tx_hash: event.transaction_hash.clone(), - height: event.block_number.parse::()?, - is_local: false, - processing_status: GoatTxProcessingStatus::Pending.to_string(), - extra: None, - created_at: current_time_secs(), - }) - .await?; - if is_new_event { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_pegout_disprove(); - } - add_node_reward( - storage_processor, - &verifier_addr.unwrap(), - U256::from_str(&event.challenger_amount_sats).unwrap_or_default(), - ) - .await?; - add_node_reward( - storage_processor, - &disprover_addr.unwrap(), - U256::from_str(&event.disprover_amount_sats).unwrap_or_default(), - ) - .await?; - } - storage_processor.cancel_messages_by_business_id(&graph_id, None).await?; - } - Ok(()) -} - -async fn handle_bridge_in_request_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - bridge_in_request_events: Vec, -) -> anyhow::Result<()> { - for event in bridge_in_request_events { - storage_processor - .upsert_goat_tx_record(&GoatTxRecord { - instance_id: Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?, - graph_id: Uuid::nil(), - tx_type: GoatTxType::BridgeInRequest.to_string(), - tx_hash: event.transaction_hash.clone(), - height: event.block_number.parse::()?, - is_local: false, - processing_status: GoatTxProcessingStatus::Pending.to_string(), - extra: Some(serde_json::to_string(&event)?), - created_at: current_time_secs(), - }) - .await?; - } - Ok(()) -} - -async fn handle_committee_response_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - committee_response_events: Vec, -) -> anyhow::Result<()> { - for event in committee_response_events { - if let Ok(instance_id) = &Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)) { - // Convert hex string to [u8; 32] - if let Ok(pubkey_bytes) = hex::decode(&event.committee_pubkey) { - if pubkey_bytes.len() == 33 { - let mut pubkey = [0u8; 33]; - pubkey.copy_from_slice(&pubkey_bytes); - storage_processor - .update_instance_committee_answer( - instance_id, - &event.committee_address, - pubkey.to_vec(), - ) - .await?; - } else { - warn!("committee_pubkey length is not 32 bytes: {}", event.committee_pubkey); - } - } else { - warn!("failed to decode committee_pubkey: {}", event.committee_pubkey); - } - } else { - warn!("failed to parse instance id:{event:?}"); - } - } - Ok(()) -} - -async fn handle_bridge_in_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - btc_client: Arc, - goat_client: Arc, - bridge_in_events: Vec, -) -> anyhow::Result<()> { - for event in bridge_in_events { - let post_pegin_txhash = event.transaction_hash.clone(); - if let Ok(instance_id) = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)) - && !storage_processor - .update_instance( - &InstanceUpdate::new_with_instance_id(instance_id) - .with_status(InstanceBridgeInStatus::RelayerL2Minted.to_string()) - .with_post_pegin(post_pegin_txhash.clone()), - ) - .await? - && let Some(tx_record) = storage_processor - .find_graph_goat_tx_record( - &instance_id, - &Uuid::nil(), - &GoatTxType::BridgeInRequest.to_string(), - ) - .await? - { - // it will happened when handle history events - warn!("Instance {instance_id} is finished but not find in db. we will create it"); - if let Some(extra) = tx_record.extra.as_ref() - && let Ok(bridge_event) = serde_json::from_str::(extra) - && let Ok((mut instance, _)) = generate_instance_from_bridge_in_request_event( - btc_client.as_ref(), - goat_client.as_ref(), - &bridge_event, - false, - ) - .await - { - info!("Instance {instance_id} is created and set status to RelayerL2Minted"); - instance.status = InstanceBridgeInStatus::RelayerL2Minted.to_string(); - instance.post_pegin_txhash = Some(post_pegin_txhash.clone()); - if !storage_processor.insert_instance_if_absent(&instance).await? { - info!("Instance {instance_id} was created concurrently; applying mint event"); - storage_processor - .update_instance( - &InstanceUpdate::new_with_instance_id(instance_id) - .with_status(InstanceBridgeInStatus::RelayerL2Minted.to_string()) - .with_post_pegin(post_pegin_txhash.clone()), - ) - .await?; - } - } - - if tx_record.processing_status == GoatTxProcessingStatus::Pending.to_string() { - info!( - "Instance {instance_id} related goat tx BridgeInRequest set event processing status skipped" - ); - storage_processor - .update_goat_tx_record_processing_status( - &Uuid::nil(), - &instance_id, - &GoatTxType::BridgeInRequest.to_string(), - &GoatTxProcessingStatus::Skipped.to_string(), - ) - .await?; - } - } - } - Ok(()) -} - -async fn handle_swap_init_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - goat_client: Arc, - swap_contract_address: &EvmAddress, - gateway_peg_btc_address: &EvmAddress, - init_events: Vec, -) -> anyhow::Result<()> { - for event in init_events { - let Some(escrow_data) = extract_escrow_data_from_tx( - &goat_client, - &event.transaction_hash, - swap_contract_address, - &hex::decode(strip_hex_prefix_owned(&event.escrow_hash))?.try_into().map_err( - |v: Vec| anyhow::anyhow!("escrow_hash length is {}, expected 32", v.len()), - )?, - ) - .await? - else { - warn!("failed to parse escrow_data for event:{event:?}"); - continue; - }; - let escrow_hash = normalize_escrow_hash(&event.escrow_hash); - let create_time = event.block_timestamp.parse::()?; - let event_height = event.block_number.parse::()?; - let escrow = SwapEscrow { - escrow_hash: escrow_hash.clone(), - network: get_network().to_string(), - status: SwapEscrowStatus::Initialize.to_string(), - offerer_addr: escrow_data.offerer.to_string(), - claimer_addr: escrow_data.claimer.to_string(), - token: escrow_data.token.to_string(), - amount: escrow_data.amount.to_string(), - refund_deadline: get_timestamp_from_contract_data(&escrow_data.refundData.0), - escrow_data: Some(hex::encode(escrow_data.abi_encode())), - init_tx_hash: event.transaction_hash.clone(), - init_tx_height: event_height, - status_updated_at: create_time, - created_at: create_time, - ..Default::default() - }; - // Swap records are derived exclusively from the on-chain Initialize - // event; duplicate event scans must not overwrite the first record. - let initialized = storage_processor.insert_swap_escrow_if_absent(&escrow).await?; - if initialized && escrow_data.token == *gateway_peg_btc_address { - let mut bridge_out_global_stats = - get_bridge_out_global_stats(storage_processor).await?; - let mut initial_amount = - U256::from_str(&bridge_out_global_stats.initial_amount).unwrap_or_default(); - initial_amount.add_assign(&escrow_data.amount); - bridge_out_global_stats.initial_amount = initial_amount.to_string(); - bridge_out_global_stats.initial_txn += 1; - storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; - info!( - "swap initialize stats included: tx_hash={}, escrow_hash={}, token={}, amount={}", - event.transaction_hash, escrow_hash, escrow_data.token, escrow_data.amount, - ); - } else if initialized { - info!( - "swap initialize stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}, token={}, expect_token={}", - event.transaction_hash, escrow_hash, escrow_data.token, gateway_peg_btc_address, - ); - } else { - info!( - "swap initialize ignored for resolved or previously initialized escrow {escrow_hash}" - ); - } - } - Ok(()) -} - -async fn handle_swap_claim_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - goat_client: Arc, - swap_contract_address: &EvmAddress, - gateway_peg_btc_address: &EvmAddress, - claim_events: Vec, -) -> anyhow::Result<()> { - for event in claim_events { - let Some(claim_data) = extract_claim_data_from_tx( - &goat_client, - &event.transaction_hash, - swap_contract_address, - &hex::decode(strip_hex_prefix_owned(&event.escrow_hash))?.try_into().map_err( - |v: Vec| anyhow::anyhow!("escrow_hash length is {}, expected 32", v.len()), - )?, - ) - .await? - else { - warn!("failed to parse claim_data for event:{event:?}"); - continue; - }; - let escrow_hash = normalize_escrow_hash(&event.escrow_hash); - let Some(escrow) = storage_processor.find_swap_escrow(&escrow_hash).await? else { - warn!( - "swap claim ignored for unknown escrow {escrow_hash}, tx_hash={}", - event.transaction_hash - ); - continue; - }; - let btc_addr = Address::from_script( - bitcoin::Script::from_bytes(&claim_data.output_script), - get_network(), - ) - .map(|address| address.to_string()) - .unwrap_or_else(|error| { - warn!( - escrow_hash = %escrow_hash, - tx_hash = %event.transaction_hash, - output_script = %hex::encode(&claim_data.output_script), - %error, - "swap claim output script is not a Bitcoin address; storing an empty display address" - ); - String::new() - }); - // The claim event is authoritative: it also overrides a locally - // derived Timeout. - let transitioned = storage_processor - .update_swap_escrow( - &SwapEscrowUpdate::new(escrow_hash.clone()) - .with_status(SwapEscrowStatus::Claim.to_string()) - .with_claim_tx_hash(event.transaction_hash.clone()) - .with_claim_btc_txid(claim_data.txid.into()) - .with_btc_addr(btc_addr) - .with_only_if_status_in(vec![ - SwapEscrowStatus::Initialize.to_string(), - SwapEscrowStatus::Timeout.to_string(), - ]), - ) - .await?; - if !transitioned { - info!( - "swap claim ignored for resolved escrow {escrow_hash} with status {}", - escrow.status - ); - continue; - } - if escrow.token == gateway_peg_btc_address.to_string() { - let mut bridge_out_global_stats = - get_bridge_out_global_stats(storage_processor).await?; - let mut claim_amount = - U256::from_str(&bridge_out_global_stats.claim_amount).unwrap_or_default(); - claim_amount.add_assign(&U256::from_str(&escrow.amount).unwrap_or_default()); - bridge_out_global_stats.claim_amount = claim_amount.to_string(); - bridge_out_global_stats.claim_txn += 1; - storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; - info!( - "swap claim stats included: tx_hash={}, escrow_hash={}, amount={}", - event.transaction_hash, escrow_hash, escrow.amount, - ); - } else { - info!( - "swap claim stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}", - event.transaction_hash, escrow_hash, - ); - } - } - Ok(()) -} - -async fn handle_swap_refund_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - gateway_peg_btc_address: &EvmAddress, - refund_events: Vec, -) -> anyhow::Result<()> { - for event in refund_events { - let escrow_hash = normalize_escrow_hash(&event.escrow_hash); - let Some(escrow) = storage_processor.find_swap_escrow(&escrow_hash).await? else { - warn!( - "swap refund ignored for unknown escrow {escrow_hash}, tx_hash={}", - event.transaction_hash - ); - continue; - }; - // The refund event is authoritative: it also overrides a locally - // derived Timeout. - let transitioned = storage_processor - .update_swap_escrow( - &SwapEscrowUpdate::new(escrow_hash.clone()) - .with_status(SwapEscrowStatus::Refund.to_string()) - .with_refund_tx_hash(event.transaction_hash.clone()) - .with_only_if_status_in(vec![ - SwapEscrowStatus::Initialize.to_string(), - SwapEscrowStatus::Timeout.to_string(), - ]), - ) - .await?; - if !transitioned { - info!( - "swap refund ignored for resolved escrow {escrow_hash} with status {}", - escrow.status - ); - continue; - } - if escrow.token == gateway_peg_btc_address.to_string() { - let mut bridge_out_global_stats = - get_bridge_out_global_stats(storage_processor).await?; - let mut refund_amount = - U256::from_str(&bridge_out_global_stats.refund_amount).unwrap_or_default(); - refund_amount.add_assign(&U256::from_str(&escrow.amount).unwrap_or_default()); - bridge_out_global_stats.refund_amount = refund_amount.to_string(); - bridge_out_global_stats.refund_txn += 1; - storage_processor.upsert_bridge_out_global_stats(&bridge_out_global_stats).await?; - info!( - "swap refund stats included: tx_hash={}, escrow_hash={}, amount={}", - event.transaction_hash, escrow_hash, escrow.amount, - ); - } else { - info!( - "swap refund stats skipped(non-pegBTC): tx_hash={}, escrow_hash={}", - event.transaction_hash, escrow_hash, - ); - } - } - Ok(()) -} - -pub(super) async fn generate_instance_from_bridge_in_request_event( - btc_client: &BTCClient, - goat_client: &GOATClient, - event: &BridgeInRequestEvent, - is_check_utxos: bool, -) -> anyhow::Result<(Instance, bool)> { - let mut input_utxo_available = true; - let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - let inputs: Vec = pegin_data - .user_inputs - .iter() - .map(|u| Input { - outpoint: OutPoint { txid: Txid::from_byte_array(u.txid), vout: u.vout }, - amount: Amount::from_sat(u.amount_sats), - }) - .collect(); - - if is_check_utxos { - for input in &inputs { - if !outpoint_available(btc_client, &input.outpoint.txid, input.outpoint.vout.into()) - .await? - { - input_utxo_available = false; - break; - } - } - } - - let user_change_address: Address = - Address::from_str(&event.user_change_address)?; - let user_refund_addr: Address = - Address::from_str(&event.user_refund_address)?; - let network = get_network(); - let user_info = UserInfo { - depositor_evm_address: EvmAddress::from_str(&event.depositor_address)?.into_array(), - txn_fees: event.txn_fees.clone().map(|s| s.parse::().unwrap_or(0)), - inputs, - user_xonly_pubkey: XOnlyPublicKey::from_str(&strip_hex_prefix_owned( - &event.user_xonly_pubkey, - ))?, - user_change_address: user_change_address.require_network(network)?, - user_refund_address: user_refund_addr.require_network(network)?, - }; - let instance = generate_instance( - btc_client, - GenerateInstanceParams { - instance_id, - user_info, - pegin_amount: Amount::from_sat(event.pegin_amount_sats.parse::().unwrap_or(0)), - pegin_request_tx_hash: event.transaction_hash.clone(), - pegin_request_height: event.block_number.parse::().unwrap_or(0), - pegin_timestamp: event.block_timestamp.parse::().unwrap_or(current_time_secs()), - }, - ) - .await?; - Ok((instance, input_utxo_available)) -} -async fn handle_post_graph_data_events<'a>( - storage_processor: &mut StorageProcessor<'a>, - post_graph_data_events: Vec, -) -> anyhow::Result<()> { - for event in post_graph_data_events { - match ( - Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id)), - Uuid::from_str(&strip_hex_prefix_owned(&event.graph_id)), - ) { - (Ok(instance_id), Ok(graph_id)) => { - let _ = apply_gateway_graph_status( - storage_processor, - instance_id, - graph_id, - GraphStatus::OperatorDataPushed, - ) - .await?; - } - _ => warn!("failed to parse graph event identifiers: {event:?}"), - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn fetch_history_events( - actor: Actor, - btc_client: Arc, - goat_client: Arc, - local_db: &LocalDB, - query_client: &GraphQueryClient, - watch_contract: WatchContract, - watch_events_config: WatchEventConfig, -) -> Result<(), Box> { - let task_name = watch_events_config.get_watch_contract_type().to_string(); - info!("Start into fetch_history_events from:{} for {task_name}", watch_contract.from_height); - // let goat_client = GOATClient::new(env::goat_config_from_env().await, env::get_goat_network()); - let mut watch_contract = watch_contract.clone(); - let local_db_clone = local_db.clone(); - let contract_addr = watch_contract.contract_addr.clone(); - let async_fn = || async move { - let task_name = watch_events_config.get_watch_contract_type().to_string(); - loop { - // let current_finalized = goat_client.get_finalized_block_number().await; - let current_finalized = match query_client - .get_sync_block_height(&watch_contract.the_graph_url) - .await - { - Ok(Some(v)) => v, - Ok(None) => { - warn!( - "fetch_history_events:fail to get graph sync block height, will try later, empty value returned" - ); - sleep(Duration::from_millis(500)).await; - continue; - } - Err(e) => { - warn!( - "fetch_history_events:fail to get graph sync block height, will try later, err:{e:#}" - ); - sleep(Duration::from_millis(500)).await; - continue; - } - }; - - if watch_contract.from_height > current_finalized { - info!( - "Contract {task_name} fetch history events will finish, as current finalize height: {current_finalized} is litter than watch from height: {}", - watch_contract.from_height, - ); - continue; - } - - let to_height = current_finalized.min(watch_contract.from_height + watch_contract.gap); - let mut tx = local_db.start_transaction().await?; - - fetch_and_handle_block_range_events( - actor.clone(), - btc_client.clone(), - goat_client.clone(), - query_client, - &mut tx, - &watch_events_config, - watch_contract.from_height, - to_height, - ) - .await?; - info!( - "Contract {task_name} finish load history event from: {}, to: {to_height}", - watch_contract.from_height - ); - watch_contract.from_height = to_height + 1; - watch_contract.status = WatchContractStatus::Syncing.to_string(); - watch_contract.updated_at = current_time_secs(); - - if to_height >= current_finalized { - info!("Contract {task_name} Finish load history at {to_height}"); - watch_contract.status = WatchContractStatus::Synced.to_string(); - tx.upsert_watch_contract(&watch_contract).await?; - tx.commit().await?; - break; - } - tx.upsert_watch_contract(&watch_contract).await?; - tx.commit().await?; - } - Ok::<(), Box>(()) - }; - let err = match async_fn().await { - Ok(_) => false, - Err(err) => { - warn!("{task_name} fetch_history_events failed,err:{:?}", err); - true - } - }; - if err { - let mut storage_processor = local_db_clone.acquire().await?; - let _ = storage_processor - .update_watch_contract_status( - &contract_addr, - &WatchContractStatus::Failed.to_string(), - current_time_secs(), - ) - .await; - } - Ok(()) -} - -pub async fn monitor_events( - actor: Actor, - btc_client: Arc, - goat_client: Arc, - local_db: &LocalDB, - watch_configs: Vec, -) -> anyhow::Result<()> { - for watch_config in watch_configs { - if watch_config.get_watch_events_len() == 0 { - continue; - } - monitor_events_item( - actor.clone(), - btc_client.clone(), - goat_client.clone(), - local_db, - watch_config, - ) - .await?; - } - - Ok(()) -} - -pub async fn monitor_events_item( - actor: Actor, - btc_client: Arc, - goat_client: Arc, - local_db: &LocalDB, - watch_events_config: WatchEventConfig, -) -> anyhow::Result<()> { - info!("start tick monitor_events"); - let mut storage_processor = local_db.acquire().await?; - let mut watch_contract = get_watch_contract( - &mut storage_processor, - &watch_events_config.get_watch_contract().to_string(), - watch_events_config.get_watch_contract_type(), - ) - .await?; - - let query_client = GraphQueryClient::new(); - // let current_finalized = goat_client.get_finalized_block_number().await?; - let current_finalized = match query_client - .get_sync_block_height(&watch_contract.the_graph_url) - .await - { - Ok(Some(v)) => v, - Ok(None) => { - warn!( - "monitor_events_item:fail to get graph sync block height, will try later, empty value returned" - ); - return Ok(()); - } - Err(e) => { - warn!( - "monitor_events_item:fail to get graph sync block height, will try later, err:{e:#}" - ); - return Ok(()); - } - }; - - if watch_contract.from_height == 0 || watch_contract.from_height >= current_finalized { - warn!( - "watch_contract start height is zero or bigger than current height, not do watch jobs" - ); - return Ok(()); - } - if watch_contract.status == WatchContractStatus::Syncing.to_string() - && watch_contract.updated_at + LOAD_HISTORY_EVENT_NO_WOKING_MAX_SECS > current_time_secs() - { - info!("Event sync not finished "); - return Ok(()); - } - if watch_contract.from_height + watch_contract.gap < current_finalized { - let watch_contract_clone = watch_contract.clone(); - let local_db_clone = local_db.clone(); - let query_client_clone = query_client.clone(); - let watch_events_config_clone = watch_events_config.clone(); - tokio::spawn(async move { - if let Err(error) = fetch_history_events( - actor.clone(), - btc_client.clone(), - goat_client, - &local_db_clone, - &query_client_clone, - watch_contract_clone, - watch_events_config_clone, - ) - .await - { - warn!(event = "event_watch_history_sync", error = %error, "event watcher history sync task exited unexpectedly"); - } - }); - return Ok(()); - } - - let to_height = current_finalized.min(watch_contract.from_height + watch_contract.gap); - let mut tx = local_db.start_transaction().await?; - fetch_and_handle_block_range_events( - actor.clone(), - btc_client, - goat_client, - &query_client, - &mut tx, - &watch_events_config, - watch_contract.from_height, - to_height, - ) - .await?; - info!("finish monitor event from: {}, to: {to_height}", watch_contract.from_height); - watch_contract.from_height = to_height + 1; - watch_contract.status = WatchContractStatus::Synced.to_string(); - watch_contract.updated_at = current_time_secs(); - tx.upsert_watch_contract(&watch_contract).await?; - tx.commit().await?; - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn run_watch_event_task( - actor: Actor, - local_db: LocalDB, - btc_client: Arc, - goat_client: Arc, - interval: u64, - cancellation_token: CancellationToken, - goat_init_config: GoatInitConfig, - metrics_state: MetricsState, -) -> anyhow::Result { - let gateway_contract: EvmAddress = get_goat_address_from_env(ENV_GOAT_GATEWAY_CONTRACT_ADDRESS) - .ok_or(anyhow::anyhow!("need to set gateway contract address"))?; - let swap_contract: EvmAddress = get_goat_address_from_env(ENV_GOAT_SWAP_CONTRACT_ADDRESS) - .ok_or(anyhow::anyhow!("need to set swap contract address"))?; - let events_map: HashMap> = HashMap::from([ - ( - Actor::Committee, - vec![ - WatchEventConfig::Gateway(TheGraphConfig { - address: gateway_contract, - the_graph_url: get_goat_gateway_the_graph_urls_from_env(), - event_entities: vec![ - GatewayEventEntity::InitWithdraws, - GatewayEventEntity::CancelWithdraws, - GatewayEventEntity::ProceedWithdraws, - GatewayEventEntity::WithdrawHappyPaths, - GatewayEventEntity::WithdrawUnhappyPaths, - GatewayEventEntity::WithdrawDisproveds, - GatewayEventEntity::BridgeInRequests, - GatewayEventEntity::BridgeIns, - GatewayEventEntity::PostGraphDatas, - ], - }), - WatchEventConfig::Swap(SwapConfig { - graph_config: TheGraphConfig { - address: swap_contract, - the_graph_url: get_goat_swap_the_graph_urls_from_env(), - event_entities: vec![ - SwapEventEntity::Initializes, - SwapEventEntity::Claims, - SwapEventEntity::Refunds, - ], - }, - peg_btc_address: goat_init_config.peg_btc_address.expect( - "peg_btc_address must be set (requires GOAT_GATEWAY_CONTRACT_ADDRESS)", - ), - }), - ], - ), - ( - Actor::Operator, - vec![WatchEventConfig::Gateway(TheGraphConfig { - address: gateway_contract, - the_graph_url: get_goat_gateway_the_graph_urls_from_env(), - event_entities: vec![ - GatewayEventEntity::InitWithdraws, - GatewayEventEntity::CancelWithdraws, - GatewayEventEntity::ProceedWithdraws, - GatewayEventEntity::WithdrawHappyPaths, - GatewayEventEntity::WithdrawUnhappyPaths, - GatewayEventEntity::WithdrawDisproveds, - GatewayEventEntity::BridgeInRequests, - GatewayEventEntity::BridgeIns, - GatewayEventEntity::PostGraphDatas, - ], - })], - ), - ( - Actor::Verifier, - vec![WatchEventConfig::Gateway(TheGraphConfig { - address: gateway_contract, - the_graph_url: get_goat_gateway_the_graph_urls_from_env(), - event_entities: vec![ - GatewayEventEntity::InitWithdraws, - GatewayEventEntity::CancelWithdraws, - GatewayEventEntity::ProceedWithdraws, - GatewayEventEntity::WithdrawHappyPaths, - GatewayEventEntity::WithdrawUnhappyPaths, - GatewayEventEntity::WithdrawDisproveds, - GatewayEventEntity::BridgeInRequests, - GatewayEventEntity::BridgeIns, - GatewayEventEntity::PostGraphDatas, - ], - })], - ), - ( - Actor::Verifier, - vec![WatchEventConfig::Gateway(TheGraphConfig { - address: gateway_contract, - the_graph_url: get_goat_gateway_the_graph_urls_from_env(), - event_entities: vec![ - GatewayEventEntity::InitWithdraws, - GatewayEventEntity::CancelWithdraws, - GatewayEventEntity::ProceedWithdraws, - GatewayEventEntity::WithdrawHappyPaths, - GatewayEventEntity::WithdrawUnhappyPaths, - GatewayEventEntity::WithdrawDisproveds, - GatewayEventEntity::BridgeInRequests, - GatewayEventEntity::BridgeIns, - GatewayEventEntity::PostGraphDatas, - ], - })], - ), - ( - Actor::Watchtower, - vec![WatchEventConfig::Gateway(TheGraphConfig { - address: gateway_contract, - the_graph_url: get_goat_gateway_the_graph_urls_from_env(), - event_entities: vec![ - GatewayEventEntity::InitWithdraws, - GatewayEventEntity::CancelWithdraws, - GatewayEventEntity::ProceedWithdraws, - GatewayEventEntity::WithdrawHappyPaths, - GatewayEventEntity::WithdrawUnhappyPaths, - GatewayEventEntity::WithdrawDisproveds, - GatewayEventEntity::BridgeInRequests, - GatewayEventEntity::BridgeIns, - GatewayEventEntity::PostGraphDatas, - ], - })], - ), - ]); - loop { - tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(interval)) => { - let started_at = Instant::now(); - // Execute the normal monitoring logic - match monitor_events( - actor.clone(), - btc_client.clone(), - goat_client.clone(), - &local_db, - events_map.get(&actor).cloned().unwrap_or_default(), - ) - .await - { - Ok(_) => { - metrics_state.record_task_run( - "event_watcher", - "success", - started_at.elapsed(), - ); - let local_db = local_db.clone(); - let goat_client = goat_client.clone(); - let metrics_state = metrics_state.clone(); - tokio::spawn(async move { - if let Ok(Err(error)) = tokio::time::timeout( - Duration::from_secs(30), - refresh_event_watch_metrics( - &local_db, - goat_client.as_ref(), - &metrics_state, - ), - ) - .await - { - metrics_state.set_event_watch_state(EventWatchState::Failed); - warn!(event = "metrics_event_watch", error = %error, "failed to refresh event watcher metrics"); - } - }); - } - Err(e) => { - metrics_state.set_event_watch_state(EventWatchState::Failed); - metrics_state.record_task_run( - "event_watcher", - "failed", - started_at.elapsed(), - ); - warn!("fail to monitor events: {e}"); - } - } - } - _ = cancellation_token.cancelled() => { - info!("Watch event task received shutdown signal"); - return Ok("watch_shutdown".to_string()); - } - } - } -} - -async fn get_watch_contract<'a>( - storage_processor: &mut StorageProcessor<'a>, - contract_addr: &str, - contract_type: WatchContractType, -) -> anyhow::Result { - let (from_height, the_graph_url, gap) = match contract_type { - WatchContractType::Gateway => ( - get_goat_gateway_event_filter_from_from_env(), - get_goat_gateway_the_graph_urls_from_env(), - get_goat_gateway_event_filter_gap_from_env(), - ), - WatchContractType::Swap => ( - get_goat_swap_event_filter_from_from_env(), - get_goat_swap_the_graph_urls_from_env(), - get_goat_swap_event_filter_gap_from_env(), - ), - }; - - if let Some(mut watch_contract) = storage_processor.find_watch_contract(contract_addr).await? { - if from_height > watch_contract.from_height { - watch_contract.from_height = from_height; - } - if the_graph_url != watch_contract.the_graph_url { - watch_contract.the_graph_url = the_graph_url; - } - - if gap != watch_contract.gap { - watch_contract.gap = gap - } - Ok(watch_contract) - } else { - Ok(WatchContract { - contract_addr: contract_addr.to_string(), - the_graph_url, - gap, - from_height, - status: WatchContractStatus::UnSync.to_string(), - extra: None, - updated_at: current_time_secs(), - created_at: current_time_secs(), - }) - } -} - -pub async fn is_processing_gateway_history_events( - local_db: &LocalDB, - goat_client: &GOATClient, -) -> anyhow::Result { - let gateway_contract: EvmAddress = get_goat_address_from_env(ENV_GOAT_GATEWAY_CONTRACT_ADDRESS) - .ok_or(anyhow::anyhow!("need to set gateway contract address"))?; - let mut storage_processor = local_db.acquire().await?; - // use finalized block height to judge is processing history events - let current_finalized = goat_client.get_finalized_block_number().await?; - let watch_contract = get_watch_contract( - &mut storage_processor, - &gateway_contract.to_string(), - WatchContractType::Gateway, - ) - .await?; - Ok(watch_contract.from_height + watch_contract.gap < current_finalized - || watch_contract.status == WatchContractStatus::Syncing.to_string()) -} - -async fn add_node_reward( - storage_processor: &mut StorageProcessor<'_>, - goat_addr: &str, - add_value: U256, -) -> anyhow::Result<()> { - let (nodes, _) = storage_processor - .find_nodes(&NodeQuery::default().with_goat_addr(goat_addr.to_string())) - .await?; - for node in nodes { - let mut reward = U256::from_str(&node.reward).unwrap_or_default(); - reward.add_assign(&add_value); - storage_processor.update_node_reward_by_peer_id(&node.peer_id, &reward.to_string()).await? - } - Ok(()) -} diff --git a/node/src/scheduled_tasks/graph_maintenance_tasks.rs b/node/src/scheduled_tasks/graph_maintenance_tasks.rs deleted file mode 100644 index 83a489667..000000000 --- a/node/src/scheduled_tasks/graph_maintenance_tasks.rs +++ /dev/null @@ -1,1441 +0,0 @@ -use crate::action::{ - AssertReady, AssertSent, ChallengeAssertSent, ChallengeSent, DisproveSent, GOATMessageContent, - KickoffReady, KickoffSent, NackReady, OperatorCommitPubinReady, OperatorCommitPubinTimeout, - PreKickoffSent, Take1Ready, Take1Sent, Take2Ready, Take2Sent, WatchtowerChallengeInitSent, - WatchtowerChallengeSent, WatchtowerChallengeTimeout, WronglyChallengeTimeout, -}; -use crate::env::get_network; -use crate::rpc_service::current_time_secs; -use crate::scheduled_tasks::{ - fetch_all_graphs_by_status, fetch_first_graph_per_operator_by_status, -}; -use crate::utils::{ - SELF_SENDER, load_validated_graph_definition, outpoint_spent_txid, upsert_message, -}; -use bitcoin::Txid; -use bitvm_lib::actors::Actor; -use bitvm_lib::timelocks::{ - connector_f_timelock_blocks, disprove_timelock_blocks, operator_ack_timelock_blocks, - operator_commit_timelock_blocks, take1_timelock_blocks, take2_timelock_blocks, - validate_timelock_config, watchtower_challenge_timelock_blocks, -}; -use client::btc_chain::BTCClient; -use client::goat_chain::DisproveTxType; -use goat::{constants::TimelockConfig, transactions::base::output_topology}; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use store::localdb::{LocalDB, StorageProcessor}; -use store::{ - GoatTxProcessingStatus, GoatTxType, Graph, GraphBtcTxVoutMonitor, GraphStatus, SerializableTxid, -}; -use strum::{Display, EnumString}; -use tracing::{info, trace, warn}; -use uuid::Uuid; - -const MONITE_BTC_TX_NAME_KICKOFF: &str = "kickoff"; -const MONITE_BTC_TX_NAME_WATCHTOWER_INIT: &str = "watchtower_init"; -const MONITE_BTC_TX_NAME_PROVER_ASSERT: &str = "prover_assert"; -const MONITE_BTC_TX_NAME_VERIFIER_ASSERT: &str = "verifier_assert"; -const MAX_PREKICKOFF_SUCCESSORS_PER_SCAN: usize = 32; - -#[derive(Clone, Debug, Eq, PartialEq, Display, EnumString)] -enum OperatorWithdrawType { - Take1, - Take2, -} - -#[derive( - Copy, Clone, Debug, Serialize, Deserialize, Default, Eq, PartialEq, Display, EnumString, -)] -pub enum VerifierChallengeStatus { - #[default] - None, - VerifierAsserted, - ProverAnswered, - Disproved, -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Default)] -pub struct ChallengeSubStatus { - pub watchtower_challenge_status: Vec, // true for challenge connector spend - pub verifier_challenge_status: Vec, - pub disprove_type: Option, - pub disprove_index: i32, -} - -struct DetectedGraphMessage { - actor: Actor, - content: GOATMessageContent, -} - -impl DetectedGraphMessage { - fn new(actor: Actor, content: GOATMessageContent) -> Self { - Self { actor, content } - } -} - -async fn graph_timelock_config( - local_db: &LocalDB, - graph_id: Uuid, -) -> anyhow::Result { - let mut storage_processor = local_db.acquire().await?; - let graph_row = storage_processor.find_graph(&graph_id).await?.ok_or_else(|| { - anyhow::anyhow!("graph {graph_id} is missing while loading its timelock config") - })?; - let graph = - load_validated_graph_definition(&mut storage_processor, graph_row.instance_id, graph_id) - .await? - .ok_or_else(|| anyhow::anyhow!("graph {graph_id} has no validated raw definition"))?; - validate_timelock_config( - graph.parameters.instance_parameters.network, - &graph.parameters.timelock_config, - )?; - Ok(graph.parameters.timelock_config) -} - -impl ChallengeSubStatus { - pub fn is_watchtower_challenge_success(&self, required_watchtower_num: usize) -> bool { - self.watchtower_challenge_status - .iter() - .filter(|&&status| status) - .take(required_watchtower_num) - .count() - == required_watchtower_num - } - - pub fn is_disproved(&self) -> bool { - self.disprove_type.is_some() - } -} - -async fn upsert_detected_messages( - local_db: &LocalDB, - messages: Vec, -) -> anyhow::Result<()> { - if messages.is_empty() { - return Ok(()); - } - - let mut storage_processor = local_db.acquire().await?; - for message in messages { - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - message.actor, - message.content, - 0, - 0, - ) - .await?; - } - Ok(()) -} - -async fn get_confirmed_tx_monitor( - btc_client: &BTCClient, - local_db: &LocalDB, - graph_id: Uuid, - txid: Txid, - tx_name: String, -) -> anyhow::Result> { - let txid_serial = SerializableTxid::from(txid); - let existing = { - let mut storage_processor = local_db.acquire().await?; - storage_processor.find_graph_btc_tx_vout_monitor(&graph_id, &txid_serial).await? - }; - if let Some(existing) = existing - && existing.height > 0 - && existing.vout_len > 0 - { - return Ok(Some(existing)); - } - - let Some(tx_info) = btc_client.get_tx_info(&txid_serial.0).await? else { - return Ok(None); - }; - let height = tx_info.status.block_height.unwrap_or_default() as i64; - if height <= 0 { - return Ok(None); - } - - let current_times = current_time_secs(); - let monitor = GraphBtcTxVoutMonitor { - graph_id, - tx_name, - txid: txid_serial, - height, - vout_len: tx_info.vout.len() as i64, - monitor_data: String::new(), - created_at: current_times, - updated_at: current_times, - }; - let mut storage_processor = local_db.acquire().await?; - storage_processor.upsert_graph_btc_tx_vout_monitor(&monitor).await?; - Ok(Some(monitor)) -} - -async fn detect_watchtower_flow_disprove( - btc_client: &BTCClient, - graph: &Graph, -) -> anyhow::Result> { - for (index, txid) in graph.operator_challenge_nack_txids.iter().enumerate() { - if btc_client.get_tx_status(&txid.0).await?.confirmed { - return Ok(Some(DetectedGraphMessage::new( - Actor::Committee, - GOATMessageContent::DisproveSent(DisproveSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - disprove_type: DisproveTxType::OperatorChallengeNack, - index, - challenge_start_txid: None, - challenge_finish_txid: txid.0, - }), - ))); - } - } - - if let Some(txid) = graph.operator_commit_timeout_txid.clone() - && btc_client.get_tx_status(&txid.0).await?.confirmed - { - return Ok(Some(DetectedGraphMessage::new( - Actor::Committee, - GOATMessageContent::DisproveSent(DisproveSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - disprove_type: DisproveTxType::OperatorCommitTimeout, - index: 0, - challenge_start_txid: None, - challenge_finish_txid: txid.0, - }), - ))); - } - - Ok(None) -} - -fn is_timelock_ready(confirmed_height: i64, lock_blocks: i64, current_height: i64) -> bool { - confirmed_height > 0 && confirmed_height + lock_blocks <= current_height -} - -pub async fn get_user_init_withdraw_graphs<'a>( - storage_processor: &mut StorageProcessor<'a>, -) -> anyhow::Result> { - let goat_tx_records = storage_processor - .get_goat_tx_record_by_processing_status( - &GoatTxType::InitWithdraw.to_string(), - &GoatTxProcessingStatus::Pending.to_string(), - ) - .await?; - Ok(goat_tx_records.iter().map(|v| (v.instance_id, v.graph_id)).collect()) -} - -/// may trigger: KickoffReady -pub async fn detect_init_withdraw_call(local_db: &LocalDB) -> anyhow::Result<()> { - trace!("start tick action: detect_init_withdraw_call"); - let graphs = { - let mut storage_processor = local_db.acquire().await?; - get_user_init_withdraw_graphs(&mut storage_processor).await? - }; - info!("start tick action: detect_init_withdraw_call get graphs:{}", graphs.len()); - for (instance_id, graph_id) in graphs { - let mut tx = local_db.start_transaction().await?; - if let Ok(Some(graph)) = tx.find_graph(&graph_id).await { - if graph.instance_id.ne(&instance_id) { - warn!( - "Graph:{graph_id} recorded instance_id:{} not equal expected instance_id:{instance_id}", - graph.instance_id - ); - continue; - } - upsert_message( - &mut tx, - false, - SELF_SENDER.to_string(), - Actor::Operator, - GOATMessageContent::KickoffReady(KickoffReady { instance_id, graph_id }), - 0, - 0, - ) - .await?; - } else { - warn!( - "instance_id: {instance_id} graph_id: {graph_id} fail to get graph from db or kickoff txid is none" - ); - } - tx.update_goat_tx_record_processing_status( - &graph_id, - &instance_id, - &GoatTxType::InitWithdraw.to_string(), - &GoatTxProcessingStatus::Processed.to_string(), - ) - .await?; - tx.commit().await?; - } - Ok(()) -} - -async fn enqueue_kickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::Result<()> { - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - Actor::All, - GOATMessageContent::KickoffSent(KickoffSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - 0, - 0, - ) - .await - .map(|_| ()) -} - -async fn enqueue_prekickoff_sent(local_db: &LocalDB, graph: &Graph) -> anyhow::Result<()> { - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - Actor::Verifier, - GOATMessageContent::PreKickoffSent(PreKickoffSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - 0, - 0, - ) - .await - .map(|_| ()) -} - -fn is_kickoff_pending_status(status: &str) -> bool { - status == GraphStatus::OperatorDataPushed.to_string() - || status == GraphStatus::PreKickoff.to_string() -} - -async fn detect_graph_kickoff( - local_db: &LocalDB, - btc_client: &BTCClient, - graph: &Graph, -) -> anyhow::Result<()> { - if !is_kickoff_pending_status(&graph.status) { - return Ok(()); - } - - let Some(kickoff_txid) = graph.kickoff_txid.clone() else { - warn!(graph_id = %graph.graph_id, "kickoff txid is missing"); - return Ok(()); - }; - let kickoff_txid: Txid = kickoff_txid.into(); - let tx_status = btc_client.get_tx_status(&kickoff_txid).await?; - if tx_status.confirmed { - enqueue_kickoff_sent(local_db, graph).await?; - } else { - trace!(graph_id = %graph.graph_id, kickoff_txid = %kickoff_txid, "kickoff is not confirmed yet"); - } - Ok(()) -} - -async fn confirmed_prekickoff_successor( - local_db: &LocalDB, - btc_client: &BTCClient, - graph: &Graph, -) -> anyhow::Result> { - let Some(next_prekickoff) = graph.next_prekickoff.clone() else { - return Ok(None); - }; - let next_prekickoff_txid: Txid = next_prekickoff.clone().into(); - if !btc_client.get_tx_status(&next_prekickoff_txid).await?.confirmed { - return Ok(None); - } - - let successor = { - let mut storage_processor = local_db.acquire().await?; - let Some((graph_id, instance_id, cur_prekickoff, _)) = storage_processor - .get_graph_pre_kickoff_chain_by_cur_pre_kickoff(next_prekickoff.clone()) - .await? - else { - warn!( - graph_id = %graph.graph_id, - next_prekickoff_txid = %next_prekickoff_txid, - "confirmed next prekickoff has no successor graph" - ); - return Ok(None); - }; - let successor = storage_processor.find_graph(&graph_id).await?.ok_or_else(|| { - anyhow::anyhow!("pre-kickoff successor graph {graph_id} disappeared from storage") - })?; - if successor.instance_id != instance_id - || successor.cur_prekickoff_txid != Some(cur_prekickoff) - || successor.operator_pubkey != graph.operator_pubkey - || successor.kickoff_index != graph.kickoff_index + 1 - { - anyhow::bail!( - "invalid pre-kickoff successor {} for graph {}", - successor.graph_id, - graph.graph_id - ); - } - successor - }; - - Ok(Some(successor)) -} - -async fn scan_kickoff_chain( - local_db: &LocalDB, - btc_client: &BTCClient, - start_graph: Graph, - visited_graph_ids: &mut HashSet, -) -> anyhow::Result<()> { - let mut graph = start_graph; - for successor_depth in 0..=MAX_PREKICKOFF_SUCCESSORS_PER_SCAN { - if !visited_graph_ids.insert(graph.graph_id) { - return Ok(()); - } - - // A failed lookup for the lower-index kickoff must not hide a - // confirmed successor pre-kickoff in the same scan. - if let Err(error) = detect_graph_kickoff(local_db, btc_client, &graph).await { - warn!( - graph_id = %graph.graph_id, - error = %error, - "failed to scan graph kickoff; continuing pre-kickoff chain" - ); - } - let Some(successor) = confirmed_prekickoff_successor(local_db, btc_client, &graph).await? - else { - return Ok(()); - }; - - if successor_depth == MAX_PREKICKOFF_SUCCESSORS_PER_SCAN { - warn!( - graph_id = %graph.graph_id, - max_depth = MAX_PREKICKOFF_SUCCESSORS_PER_SCAN, - "stopped pre-kickoff successor scan at configured depth limit" - ); - return Ok(()); - } - - enqueue_prekickoff_sent(local_db, &successor).await?; - graph = successor; - } - Ok(()) -} - -/// May trigger PreKickoffSent and KickoffSent. -/// -/// The first OperatorDataPushed graph is the normal scan entry point for an -/// operator. A confirmed `next_prekickoff` proves that the next graph has -/// started, so follow that chain instead of letting the earlier graph hide it. -pub async fn detect_kickoff(local_db: &LocalDB, btc_client: &BTCClient) -> anyhow::Result<()> { - trace!("start tick action: detect_kickoff"); - let graphs = { - let mut storage_processor = local_db.acquire().await?; - let mut graphs = fetch_first_graph_per_operator_by_status( - &mut storage_processor, - &GraphStatus::OperatorDataPushed.to_string(), - ) - .await?; - // A PreKickoff graph must remain eligible until its kickoff confirms. - graphs.extend( - fetch_all_graphs_by_status( - &mut storage_processor, - &GraphStatus::PreKickoff.to_string(), - ) - .await?, - ); - graphs - }; - info!("start tick action: detect_kickoff, roots: {}", graphs.len()); - - let mut visited_graph_ids = HashSet::new(); - for graph in graphs { - let graph_id = graph.graph_id; - if let Err(error) = - scan_kickoff_chain(local_db, btc_client, graph, &mut visited_graph_ids).await - { - warn!(graph_id = %graph_id, error = %error, "failed to scan kickoff chain"); - } - } - Ok(()) -} - -/// may trigger: Take1Ready, Take1Sent, ChallengeSent -pub async fn detect_take1_or_challenge( - local_db: &LocalDB, - btc_client: &BTCClient, -) -> anyhow::Result<()> { - trace!("start tick action: detect_take1_or_challenge"); - - let graphs = { - let mut storage_processor = local_db.acquire().await?; - fetch_all_graphs_by_status( - &mut storage_processor, - &GraphStatus::OperatorKickOff.to_string(), - ) - .await? - }; - let current_height = btc_client.get_height().await? as i64; - info!( - "start tick action: detect_take1_or_challenge, graphs: {}, current_height: {current_height}", - graphs.len() - ); - for graph in graphs { - let graph_id = graph.graph_id; - if let Err(error) = - detect_take1_or_challenge_for_graph(local_db, btc_client, graph, current_height).await - { - warn!(graph_id = %graph_id, error = %error, "failed to scan operator kickoff graph"); - } - } - Ok(()) -} - -async fn detect_take1_or_challenge_for_graph( - local_db: &LocalDB, - btc_client: &BTCClient, - graph: Graph, - current_height: i64, -) -> anyhow::Result<()> { - let timelock_config = graph_timelock_config(local_db, graph.graph_id).await?; - let lock_blocks = take1_timelock_blocks(get_network(), &timelock_config) as i64; - if detect_kickoff_ref_disprove_tx(btc_client, local_db, &graph).await? { - warn!( - "process_graph_challenge detect_kickoff_ref_disprove_tx happened at graph:{}", - graph.graph_id - ); - return Ok(()); - } - - // process_kickoff_graph may trigger Take1Ready, Take1Sent or ChallengeSent - if let Some((actor, message_content)) = - process_kickoff_graph(btc_client, local_db, &graph, lock_blocks, current_height).await? - { - info!("process_kickoff_graph detect take1 ready or take1 sent or challenge sent"); - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - actor, - message_content, - 0, - 0, - ) - .await?; - } - Ok(()) -} - -#[tracing::instrument(level = "info", skip(local_db, btc_client))] -pub async fn process_graph_challenge( - local_db: &LocalDB, - btc_client: &BTCClient, -) -> anyhow::Result<()> { - trace!("start tick action: process_graph_challenge"); - - let graphs = { - let mut storage_processor = local_db.acquire().await?; - fetch_all_graphs_by_status(&mut storage_processor, &GraphStatus::Challenge.to_string()) - .await? - }; - let current_height = btc_client.get_height().await? as i64; - info!( - "start tick action: process_graph_challenge, graphs: {}, current_height: {current_height}", - graphs.len() - ); - - for graph in graphs { - let graph_id = graph.graph_id; - if let Err(error) = - process_graph_challenge_for_graph(local_db, btc_client, graph, current_height).await - { - warn!(graph_id = %graph_id, error = %error, "failed to scan challenge graph"); - } - } - Ok(()) -} - -async fn process_graph_challenge_for_graph( - local_db: &LocalDB, - btc_client: &BTCClient, - graph: Graph, - current_height: i64, -) -> anyhow::Result<()> { - let watchtower_flow_messages = - detect_watchtower_flow(btc_client, local_db, &graph, current_height).await?; - if !watchtower_flow_messages.is_empty() { - info!( - "process_graph_challenge detected {} watchtower/pubin flow messages", - watchtower_flow_messages.len() - ); - upsert_detected_messages(local_db, watchtower_flow_messages).await?; - } - - let assert_sent_messages = detect_assert_sent_flow(btc_client, local_db, &graph).await?; - if !assert_sent_messages.is_empty() { - info!( - "process_graph_challenge detected {} assert/challenge-assert messages", - assert_sent_messages.len() - ); - upsert_detected_messages(local_db, assert_sent_messages).await?; - } - - if let Some((actor, message_content)) = - detect_assert_disprove_ready(btc_client, local_db, &graph, current_height).await? - { - info!("process_graph_challenge detect assert disprove ready"); - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - actor, - message_content, - 0, - 0, - ) - .await?; - } - - // take2 monitor - if let Some((actor, message_content)) = - detect_take2(btc_client, local_db, &graph, current_height).await? - { - info!("process_graph_challenge detect take2 ready or take2 sent or disprove sent"); - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - actor, - message_content, - 0, - 0, - ) - .await?; - } - Ok(()) -} - -/// may trigger: -/// - WatchtowerChallengeInitSent -/// - WatchtowerChallengeSent -/// - WatchtowerChallengeTimeout -/// - NackReady -/// - OperatorCommitPubinReady -/// - OperatorCommitPubinTimeout -/// - AssertReady -async fn detect_watchtower_flow( - btc_client: &BTCClient, - local_db: &LocalDB, - graph: &Graph, - current_height: i64, -) -> anyhow::Result> { - let watchtower_challenge_init_txid: Txid = match graph.watchtower_challenge_init_txid.clone() { - Some(txid) => txid.into(), - None => { - warn!( - "detect_watchtower_challenge graph_id:{} watchtower_challenge_init_txid is none", - graph.graph_id - ); - return Ok(vec![]); - } - }; - - let Some(monitor) = get_confirmed_tx_monitor( - btc_client, - local_db, - graph.graph_id, - watchtower_challenge_init_txid, - MONITE_BTC_TX_NAME_WATCHTOWER_INIT.to_string(), - ) - .await? - else { - trace!( - "detect_watchtower_flow graph_id:{} watchtower challenge init txid {} not confirmed", - graph.graph_id, watchtower_challenge_init_txid - ); - return Ok(vec![]); - }; - let timelock_config = graph_timelock_config(local_db, graph.graph_id).await?; - - let mut messages = vec![DetectedGraphMessage::new( - Actor::Watchtower, - GOATMessageContent::WatchtowerChallengeInitSent(WatchtowerChallengeInitSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - )]; - - let watchtower_num = output_topology::watchtower_challenge_init::watchtower_num( - monitor.vout_len.max(0) as usize, - ); - if watchtower_num == 0 { - return Ok(messages); - } - - if let Some(disprove_message) = detect_watchtower_flow_disprove(btc_client, graph).await? { - return Ok(vec![disprove_message]); - } - - let connector_e_vout = - output_topology::watchtower_challenge_init::connector_e(watchtower_num) as u64; - let connector_f_vout = - output_topology::watchtower_challenge_init::connector_f(watchtower_num) as u64; - let connector_e_spent_txid = - outpoint_spent_txid(btc_client, &watchtower_challenge_init_txid, connector_e_vout).await?; - let connector_f_spent_txid = - outpoint_spent_txid(btc_client, &watchtower_challenge_init_txid, connector_f_vout).await?; - let operator_commit_timeout_txid: Option = - graph.operator_commit_timeout_txid.clone().map(Into::into); - let operator_commit_timeout_on_chain = - match (connector_e_spent_txid.as_ref(), operator_commit_timeout_txid.as_ref()) { - (Some(spent_txid), Some(timeout_txid)) => spent_txid == timeout_txid, - _ => false, - }; - let pubin_commit_completed = - connector_e_spent_txid.is_some() && !operator_commit_timeout_on_chain; - - let mut all_watchtower_branches_resolved = true; - let mut any_watchtower_timeout_ready = false; - let mut any_nack_ready = false; - for watchtower_index in 0..watchtower_num { - let watchtower_vout = - output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) - as u64; - let ack_vout = - output_topology::watchtower_challenge_init::ack_connector(watchtower_index) as u64; - let watchtower_spent_txid = - outpoint_spent_txid(btc_client, &watchtower_challenge_init_txid, watchtower_vout) - .await?; - let ack_spent_txid = - outpoint_spent_txid(btc_client, &watchtower_challenge_init_txid, ack_vout).await?; - let timeout_txid = - graph.watchtower_challenge_timeout_txids.get(watchtower_index).cloned().map(Into::into); - let watchtower_spend_confirmed = match watchtower_spent_txid.as_ref() { - Some(txid) => btc_client.get_tx_status(txid).await?.confirmed, - None => false, - }; - let ack_spend_confirmed = match ack_spent_txid.as_ref() { - Some(txid) => btc_client.get_tx_status(txid).await?.confirmed, - None => false, - }; - let watchtower_timeout_spent = match (watchtower_spent_txid.as_ref(), timeout_txid.as_ref()) - { - (Some(spent_txid), Some(timeout_txid)) => { - spent_txid == timeout_txid && watchtower_spend_confirmed - } - _ => false, - }; - - match watchtower_spent_txid { - Some(_) if !watchtower_spend_confirmed => { - all_watchtower_branches_resolved = false; - } - Some(_) if !watchtower_timeout_spent => { - messages.push(DetectedGraphMessage::new( - Actor::Operator, - GOATMessageContent::WatchtowerChallengeSent(WatchtowerChallengeSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - watchtower_index, - }), - )); - - if !ack_spend_confirmed { - all_watchtower_branches_resolved = false; - if connector_f_spent_txid.is_none() - && ack_spent_txid.is_none() - && is_timelock_ready( - monitor.height, - operator_ack_timelock_blocks(get_network(), &timelock_config) as i64, - current_height, - ) - { - any_nack_ready = true; - } - } - } - Some(_) => {} - None => { - all_watchtower_branches_resolved = false; - if ack_spent_txid.is_none() - && is_timelock_ready( - monitor.height, - watchtower_challenge_timelock_blocks(get_network(), &timelock_config) - as i64, - current_height, - ) - { - any_watchtower_timeout_ready = true; - } - } - } - } - - if any_watchtower_timeout_ready { - messages.push(DetectedGraphMessage::new( - Actor::Operator, - GOATMessageContent::WatchtowerChallengeTimeout(WatchtowerChallengeTimeout { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - )); - } - - if any_nack_ready { - messages.push(DetectedGraphMessage::new( - Actor::Verifier, - GOATMessageContent::NackReady(NackReady { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - )); - } - - if all_watchtower_branches_resolved - && connector_e_spent_txid.is_none() - && connector_f_spent_txid.is_none() - { - messages.push(DetectedGraphMessage::new( - Actor::Operator, - GOATMessageContent::OperatorCommitPubinReady(OperatorCommitPubinReady { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - )); - - if is_timelock_ready( - monitor.height, - operator_commit_timelock_blocks(get_network(), &timelock_config) as i64, - current_height, - ) { - messages.push(DetectedGraphMessage::new( - Actor::Verifier, - GOATMessageContent::OperatorCommitPubinTimeout(OperatorCommitPubinTimeout { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - )); - } - } - - if all_watchtower_branches_resolved - && pubin_commit_completed - && let Some(operator_assert_txid) = graph.operator_assert_txid.clone() - && !btc_client.get_tx_status(&operator_assert_txid.0).await?.confirmed - { - messages.push(DetectedGraphMessage::new( - Actor::Operator, - GOATMessageContent::AssertReady(AssertReady { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - )); - } - - Ok(messages) -} - -/// may trigger: -/// - AssertSent -/// - ChallengeAssertSent -async fn detect_assert_sent_flow( - btc_client: &BTCClient, - local_db: &LocalDB, - graph: &Graph, -) -> anyhow::Result> { - let operator_assert_txid: Txid = match graph.operator_assert_txid.clone() { - Some(txid) => txid.into(), - None => { - warn!( - "detect_assert_sent_flow graph_id:{} operator_assert_txid is none", - graph.graph_id - ); - return Ok(vec![]); - } - }; - - let Some(_) = get_confirmed_tx_monitor( - btc_client, - local_db, - graph.graph_id, - operator_assert_txid, - MONITE_BTC_TX_NAME_PROVER_ASSERT.to_string(), - ) - .await? - else { - trace!( - "detect_assert_sent_flow graph_id:{} operator assert txid {} not confirmed", - graph.graph_id, operator_assert_txid - ); - return Ok(vec![]); - }; - - let mut messages = vec![DetectedGraphMessage::new( - Actor::Verifier, - GOATMessageContent::AssertSent(AssertSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - assert_txid: operator_assert_txid, - }), - )]; - - for (verifier_index, challenge_assert_txid) in graph.verifier_assert_txids.iter().enumerate() { - let challenge_assert_txid: Txid = challenge_assert_txid.clone().into(); - let Some(_) = get_confirmed_tx_monitor( - btc_client, - local_db, - graph.graph_id, - challenge_assert_txid, - format!("{MONITE_BTC_TX_NAME_VERIFIER_ASSERT}_{verifier_index}"), - ) - .await? - else { - continue; - }; - - messages.push(DetectedGraphMessage::new( - Actor::Operator, - GOATMessageContent::ChallengeAssertSent(ChallengeAssertSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - challenge_assert_txid, - verifier_index, - }), - )); - } - - Ok(messages) -} - -// may trigger disprove ready -async fn detect_assert_disprove_ready( - btc_client: &BTCClient, - local_db: &LocalDB, - graph: &Graph, - current_height: i64, -) -> anyhow::Result> { - let operator_assert_txid = match graph.operator_assert_txid.clone() { - Some(operator_assert_txid) => operator_assert_txid.into(), - None => { - warn!( - "detect_assert_disprove_ready graph_id:{} operator_assert_txid has none value", - graph.graph_id - ); - return Ok(None); - } - }; - if graph.verifier_assert_txids.is_empty() { - return Ok(None); - } - - let connector_d_vout = - output_topology::operator_assert::connector_d(graph.verifier_assert_txids.len()) as u64; - if outpoint_spent_txid(btc_client, &operator_assert_txid, connector_d_vout).await?.is_some() { - trace!( - "detect_assert_disprove_ready graph_id:{} connector_d already spent", - graph.graph_id - ); - return Ok(None); - } - - let timelock_config = graph_timelock_config(local_db, graph.graph_id).await?; - let disprove_timelock = disprove_timelock_blocks(get_network(), &timelock_config) as i64; - - for (index, verifier_assert_txid) in graph.verifier_assert_txids.iter().enumerate() { - let verifier_assert_txid: Txid = verifier_assert_txid.clone().into(); - if outpoint_spent_txid(btc_client, &verifier_assert_txid, 0).await?.is_some() { - continue; - } - - let height = { - let mut storage_processor = local_db.acquire().await?; - storage_processor - .find_graph_btc_tx_vout_monitor(&graph.graph_id, &verifier_assert_txid.into()) - .await? - .unwrap_or_default() - .height - }; - let height = if height <= 0 { - let Some(tx_info) = btc_client.get_tx_info(&verifier_assert_txid).await? else { - continue; - }; - let height = tx_info.status.block_height.unwrap_or_default() as i64; - if height <= 0 { - continue; - } - - let current_times = current_time_secs(); - let mut storage_processor = local_db.acquire().await?; - storage_processor - .upsert_graph_btc_tx_vout_monitor(&GraphBtcTxVoutMonitor { - graph_id: graph.graph_id, - tx_name: format!("verifier_assert_{index}"), - txid: verifier_assert_txid.into(), - height, - vout_len: tx_info.vout.len() as i64, - monitor_data: String::new(), - created_at: current_times, - updated_at: current_times, - }) - .await?; - height - } else { - height - }; - - if height + disprove_timelock <= current_height { - info!( - "detect_assert_disprove_ready graph_id:{} verifier_assert index:{} is ready to disprove", - graph.graph_id, index - ); - return Ok(Some(( - Actor::Verifier, - GOATMessageContent::WronglyChallengeTimeout(WronglyChallengeTimeout { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - challenge_assert_txid: verifier_assert_txid, - verifier_index: index, - }), - ))); - } - } - - Ok(None) -} - -/// Check if Take1Ready Take2Ready message needs to be sent -async fn check_operator_withdraw_ready_condition( - btc_client: &BTCClient, - local_db: &LocalDB, - graph_id: Uuid, - check_tx_items: Vec<(Txid, String, OperatorWithdrawType, i64, i64)>, // (txid, tag, height, lock_blocks) - current_height: i64, -) -> anyhow::Result { - info!( - "check_operator_withdraw_ready_condition for graph_id: {graph_id}, check tx size: {}, detail:{check_tx_items:?}", - check_tx_items.len() - ); - let mut ready = true; - for (txid, tx_name, operator_withdraw_type, height, lock_blocks) in check_tx_items { - let height = if height <= 0 { - let current_times = current_time_secs(); - let (height, vout_len) = match btc_client.get_tx_info(&txid).await? { - Some(tx_info) => ( - tx_info.status.block_height.unwrap_or_default() as i64, - tx_info.vout.len() as i64, - ), - None => { - info!("graph_id:{graph_id}, {operator_withdraw_type} txid {txid} not on chain",); - return Ok(false); - } - }; - let txid_serial: SerializableTxid = txid.into(); - let mut storage_processor = local_db.acquire().await?; - let existing = - storage_processor.find_graph_btc_tx_vout_monitor(&graph_id, &txid_serial).await?; - let (monitor_data, created_at, tx_name_to_use) = match existing { - Some(existing) => (existing.monitor_data, existing.created_at, existing.tx_name), - None => ("".to_string(), current_times, tx_name.clone()), - }; - storage_processor - .upsert_graph_btc_tx_vout_monitor(&GraphBtcTxVoutMonitor { - graph_id, - tx_name: tx_name_to_use, - txid: txid_serial, - height, - vout_len, - monitor_data, - created_at, - updated_at: current_times, - }) - .await?; - height - } else { - height - }; - - info!( - "graph_id:{graph_id}, {operator_withdraw_type} txid {txid} at height {height} lock_blocks {lock_blocks}, current height: {current_height} ", - ); - - if height == 0 || height > 0 && height + lock_blocks > current_height { - ready = false; - break; - } - } - Ok(ready) -} - -/// Process graph data in KickOff status -/// may return: Take1Ready, Take1Sent, ChallengeSent -async fn process_kickoff_graph( - btc_client: &BTCClient, - local_db: &LocalDB, - graph: &Graph, - lock_blocks: i64, - current_height: i64, -) -> anyhow::Result> { - trace!("process_kickoff_graph: {}", graph.graph_id); - let (kickoff_txid, take1_txid) = match (graph.kickoff_txid.clone(), graph.take1_txid.clone()) { - (Some(kickoff), Some(take1)) => (kickoff.into(), take1.into()), - _ => { - // NOTE: revert to previous status? - warn!("process_kickoff_graph graph_id:{}, kickoff or take1 is none", graph.graph_id); - return Ok(None); - } - }; - let connector_a_vout = output_topology::kickoff::connector_a() as u64; - let spent_txid = match outpoint_spent_txid(btc_client, &kickoff_txid, connector_a_vout).await? { - Some(txid) => txid, - None => { - // kickoff output not spent, check if we need to send Take1Ready - let height = { - let mut storage_processor = local_db.acquire().await?; - storage_processor - .find_graph_btc_tx_vout_monitor(&graph.graph_id, &kickoff_txid.into()) - .await? - .unwrap_or_default() - .height - }; - if check_operator_withdraw_ready_condition( - btc_client, - local_db, - graph.graph_id, - vec![( - kickoff_txid, - MONITE_BTC_TX_NAME_KICKOFF.to_string(), - OperatorWithdrawType::Take1, - height, - lock_blocks, - )], - current_height, - ) - .await? - { - info!( - "process_kickoff_graph graph_id:{}, take1 is ready to send to btc chain", - graph.graph_id - ); - return Ok(Some(( - Actor::Operator, - GOATMessageContent::Take1Ready(Take1Ready { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - ))); - } else { - trace!("process_kickoff_graph graph_id:{}, take1 not ready", graph.graph_id); - } - return Ok(None); - } - }; - if spent_txid == take1_txid { - info!( - "process_kickoff_graph graph_id:{}, take1 is on chain, will try call contract", - graph.graph_id - ); - Ok(Some(( - Actor::Committee, - GOATMessageContent::Take1Sent(Take1Sent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - ))) - } else { - info!( - "process_kickoff_graph graph_id:{}, challenge txid: {} has been detected.", - graph.graph_id, - spent_txid.to_string() - ); - // Challenge was sent - Ok(Some(( - Actor::Operator, - GOATMessageContent::ChallengeSent(ChallengeSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - challenge_txid: spent_txid, - }), - ))) - } -} - -/// may trigger: DisproveSent(QuickChallenge/ChallengeIncompleteKickoff) -async fn detect_kickoff_ref_disprove_tx( - btc_client: &BTCClient, - local_db: &LocalDB, - graph: &Graph, -) -> anyhow::Result { - let mut detected = false; - let (kickoff_txid, take1_txid, take2_txid, next_pre_kickoff): ( - Txid, - Txid, - Txid, - SerializableTxid, - ) = match ( - graph.kickoff_txid.clone(), - graph.take1_txid.clone(), - graph.take2_txid.clone(), - graph.next_prekickoff.clone(), - ) { - (Some(kickoff_txid), Some(take1_txid), Some(take2_txid), Some(next_pre_kickoff)) => { - (kickoff_txid.into(), take1_txid.into(), take2_txid.into(), next_pre_kickoff) - } - _ => { - warn!("graph:{} kickoff_txid/take1_txid/take2_txid has none value", graph.graph_id); - return Ok(detected); - } - }; - let pre_sents = check_pre_kickoff_sent( - local_db, - btc_client, - next_pre_kickoff, - MAX_PREKICKOFF_SUCCESSORS_PER_SCAN, - ) - .await?; - if pre_sents > 0 { - info!("graph_id:{} next {pre_sents} graphs's pre_kickoff has been sent!", graph.graph_id); - detected = true; - } - let guardian_connector_vout = output_topology::kickoff::guardian_connector() as u64; - if let Some(spend_txid) = - outpoint_spent_txid(btc_client, &kickoff_txid, guardian_connector_vout).await? - && let Some(tx) = btc_client.get_tx(&spend_txid).await? - { - if spend_txid == take1_txid || spend_txid == take2_txid || tx.input.len() < 2 { - return Ok(false); - } - - let disprove_type = if tx.input[1].previous_output.vout == 0 { - DisproveTxType::QuickChallenge - } else { - DisproveTxType::ChallengeIncompleteKickoff - }; - - info!( - "graph_id:{} is disproved, spent txid:{}, disprove_type:{}", - graph.graph_id, spend_txid, disprove_type - ); - let challenge_start_txid: Option = graph.challenge_txid.clone().map(|v| v.into()); - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - Actor::Committee, - GOATMessageContent::DisproveSent(DisproveSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - disprove_type, - index: 0, - challenge_start_txid, - challenge_finish_txid: spend_txid, - }), - 0, - 0, - ) - .await?; - detected = true; - } - Ok(detected) -} - -/// may trigger: Take2Ready, Take2Sent, DisproveSent(Disprove) -async fn detect_take2( - btc_client: &BTCClient, - local_db: &LocalDB, - graph: &Graph, - current_height: i64, -) -> anyhow::Result> { - let (kickoff_txid, watchtower_challenge_init_txid, operator_assert_txid, take2_txid) = match ( - graph.kickoff_txid.clone(), - graph.watchtower_challenge_init_txid.clone(), - graph.operator_assert_txid.clone(), - graph.take2_txid.clone(), - ) { - ( - Some(kickoff_txid), - Some(watchtower_challenge_init_txid), - Some(operator_assert_txid), - Some(take2_txid), - ) => ( - kickoff_txid.into(), - watchtower_challenge_init_txid.into(), - operator_assert_txid.into(), - take2_txid.into(), - ), - _ => { - warn!( - "detect_take2 graph_id:{} kickoff_txid/watchtower_challenge_init_txid/operator_assert_txid/take2_txid has none value", - graph.graph_id - ); - return Ok(None); - } - }; - - let connector_d_vout = - output_topology::operator_assert::connector_d(graph.verifier_assert_txids.len()) as u64; - if let Some(spend_txid) = - outpoint_spent_txid(btc_client, &operator_assert_txid, connector_d_vout).await? - { - if spend_txid == take2_txid { - info!("detect_take2 graph_id:{} take2 is on chain", graph.graph_id); - return Ok(Some(( - Actor::Committee, - GOATMessageContent::Take2Sent(Take2Sent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - ))); - } - - if let Some(tx) = btc_client.get_tx(&spend_txid).await? - && tx.input.len() == 2 - { - let verifier_assert_txid = tx.input[0].previous_output.txid; - if let Some(index) = graph - .verifier_assert_txids - .iter() - .position(|txid| Txid::from(txid.clone()) == verifier_assert_txid) - { - info!( - "detect_take2 graph_id:{} disprove is on chain, spent txid:{}, index:{}", - graph.graph_id, spend_txid, index - ); - return Ok(Some(( - Actor::Committee, - GOATMessageContent::DisproveSent(DisproveSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - disprove_type: DisproveTxType::Disprove, - index, - challenge_start_txid: graph.challenge_txid.clone().map(|v| v.into()), - challenge_finish_txid: spend_txid, - }), - ))); - } - } - - warn!( - "detect_take2 graph_id:{} connector_d spent by pubin-disprove txid:{}", - graph.graph_id, spend_txid - ); - return Ok(Some(( - Actor::Committee, - GOATMessageContent::DisproveSent(DisproveSent { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - disprove_type: DisproveTxType::PubinDisprove, - index: 0, - challenge_start_txid: None, - challenge_finish_txid: spend_txid, - }), - ))); - } - - let guardian_connector_vout = output_topology::kickoff::guardian_connector() as u64; - if outpoint_spent_txid(btc_client, &kickoff_txid, guardian_connector_vout).await?.is_some() { - trace!("detect_take2 graph_id:{} guardian connector already spent", graph.graph_id); - return Ok(None); - } - - let height = { - let mut storage_processor = local_db.acquire().await?; - storage_processor - .find_graph_btc_tx_vout_monitor(&graph.graph_id, &operator_assert_txid.into()) - .await? - .unwrap_or_default() - .height - }; - let timelock_config = graph_timelock_config(local_db, graph.graph_id).await?; - if check_operator_withdraw_ready_condition( - btc_client, - local_db, - graph.graph_id, - vec![ - ( - operator_assert_txid, - MONITE_BTC_TX_NAME_PROVER_ASSERT.to_string(), - OperatorWithdrawType::Take2, - height, - take2_timelock_blocks(get_network(), &timelock_config) as i64, - ), - ( - watchtower_challenge_init_txid, - MONITE_BTC_TX_NAME_WATCHTOWER_INIT.to_string(), - OperatorWithdrawType::Take2, - 0, - connector_f_timelock_blocks(get_network(), &timelock_config) as i64, - ), - ], - current_height, - ) - .await? - { - info!("detect_take2 graph_id:{} take2 is ready to send to btc chain", graph.graph_id); - Ok(Some(( - Actor::Operator, - GOATMessageContent::Take2Ready(Take2Ready { - instance_id: graph.instance_id, - graph_id: graph.graph_id, - }), - ))) - } else { - trace!("detect_take2 graph_id:{} take2 not ready", graph.graph_id); - Ok(None) - } -} - -/// may trigger: PreKickoffSent -async fn check_pre_kickoff_sent( - local_db: &LocalDB, - btc_client: &BTCClient, - pre_kickoff: SerializableTxid, - max_depth: usize, -) -> anyhow::Result { - let check_graphs = { - let mut check_graphs: Vec<(Uuid, Uuid, Txid)> = vec![]; - let mut storage_processor = local_db.acquire().await?; - let mut remaining_depth = max_depth; - let mut pre_kickoff = pre_kickoff; - - while remaining_depth > 0 { - if let Some((graph_id, instance_id, cur_pre_kickoff, next_pre_kickoff)) = - storage_processor - .get_graph_pre_kickoff_chain_by_cur_pre_kickoff(pre_kickoff.clone()) - .await? - { - check_graphs.push((graph_id, instance_id, cur_pre_kickoff.into())); - pre_kickoff = next_pre_kickoff; - remaining_depth -= 1; - } else { - break; - } - } - check_graphs - }; - let mut pre_sents = 0; - for (graph_id, instance_id, cur_pre_kickoff) in check_graphs { - if btc_client.get_tx_status(&cur_pre_kickoff).await?.confirmed { - let mut storage_processor = local_db.acquire().await?; - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - Actor::Verifier, - GOATMessageContent::PreKickoffSent(PreKickoffSent { instance_id, graph_id }), - 0, - 0, - ) - .await?; - pre_sents += 1; - } - } - Ok(pre_sents) -} diff --git a/node/src/scheduled_tasks/instance_maintenance_tasks.rs b/node/src/scheduled_tasks/instance_maintenance_tasks.rs deleted file mode 100644 index 5faa781a2..000000000 --- a/node/src/scheduled_tasks/instance_maintenance_tasks.rs +++ /dev/null @@ -1,860 +0,0 @@ -use crate::action::{ - ConfirmInstance, GOATMessage, GOATMessageContent, MessageDeferReason, PeginConfirmNonce, - PeginConfirmNonceConsensus, PeginConfirmPartialSig, PeginRequest, PostReady, - RetryableDispatchError, RetryableDispatchReason, defer_or_enqueue_message, -}; -use crate::env::{ - COMMITTEE_INSTANCE_KEYS_DIR, get_bitvm_key, get_committee_instance_key_delete_timelock_blocks, - get_instance_maintenance_batch_size, get_instance_presigned_time_expired_secs, - is_enable_committee_instance_key_delete, -}; -use crate::rpc_service::current_time_secs; -use crate::scheduled_tasks::event_watch_task::generate_instance_from_bridge_in_request_event; -use crate::utils::{ - SELF_SENDER, check_bridge_in_uxto_available_or_self_spent, gen_instance_parameters_local, - get_committee_agg_nonce_consensus_for_instance, get_committee_endorse_sigs_for_pegin, - get_committee_partial_sig_for_instance, get_committee_pub_nonce_for_instance, - load_committee_instance_keypair, store_committee_pub_nonce_for_instance, upsert_message, -}; -use bitvm_lib::actors::Actor; -use bitvm_lib::keys::CommitteeMasterKey; -use bitvm_lib::timelocks::default_connector_z_timelock_blocks; -use bitvm_lib::transactions::base::BaseTransaction; -use client::Utxo; -use client::btc_chain::BTCClient; -use client::goat_chain::GOATClient; -use client::graphs::graph_query::BridgeInRequestEvent; -use std::collections::HashMap; -use std::fs; -use std::path::PathBuf; -use std::str::FromStr; -use std::sync::{LazyLock, Mutex}; -use std::vec; -use store::localdb::{InstanceQuery, InstanceUpdate, LocalDB, StorageProcessor}; -use store::{GoatTxProcessingStatus, GoatTxType, Instance, InstanceBridgeInStatus}; -use tracing::{info, warn}; -use uuid::Uuid; - -const TASK_KEY_INSTANCE_WINDOW_EXPIRATION: &str = "instance_window_expiration_monitor"; -const TASK_KEY_INSTANCE_EXPIRATION: &str = "instance_expiration_monitor"; -const TASK_KEY_INSTANCE_BTC_TX: &str = "instance_btc_tx_monitor"; -const TASK_KEY_PEGIN_CONFIRM_RECOVERY: &str = "pegin_confirm_recovery_monitor"; - -#[derive(Clone, Debug)] -struct InstancePageState { - watermark: i64, - cursor: Option<(i64, Uuid)>, -} - -static INSTANCE_PAGE_STATES: LazyLock>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - -fn get_or_init_instance_page_state(task_key: &'static str) -> InstancePageState { - let mut state = INSTANCE_PAGE_STATES.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - state - .entry(task_key) - .or_insert_with(|| InstancePageState { watermark: current_time_secs(), cursor: None }) - .clone() -} - -fn reset_instance_page_state(task_key: &'static str) { - let mut state = INSTANCE_PAGE_STATES.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - state.insert(task_key, InstancePageState { watermark: current_time_secs(), cursor: None }); -} - -fn advance_instance_page_state(task_key: &'static str, watermark: i64, last: (i64, Uuid)) { - let mut state = INSTANCE_PAGE_STATES.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - state.insert(task_key, InstancePageState { watermark, cursor: Some(last) }); -} - -fn finish_recovery_enqueue( - result: anyhow::Result<()>, - instance_id: Uuid, - action: &'static str, -) -> anyhow::Result { - match result { - Ok(()) => Ok(true), - Err(error) - if error.chain().any(|cause| { - cause.downcast_ref::().is_some_and(|retryable| { - retryable.reason == RetryableDispatchReason::ResourceLocked - }) - }) => - { - warn!( - event = "pegin_confirm_recovery", - outcome = "resource_locked", - instance_id = %instance_id, - action, - error = %error, - "skip recovery enqueue while the local message is actively claimed" - ); - Ok(false) - } - Err(error) => Err(error), - } -} - -async fn find_one_instance_page( - local_db: &LocalDB, - task_key: &'static str, - mut query: InstanceQuery, - batch_size: u32, -) -> anyhow::Result> { - let page_state = get_or_init_instance_page_state(task_key); - query = query - .with_order("created_at ASC, instance_id ASC".to_string()) - .with_limit(batch_size) - .with_raw_condition(format!("created_at <= {}", page_state.watermark)); - - if let Some((created_at, instance_id)) = page_state.cursor { - query = query.with_raw_condition(format!( - "(created_at > {created_at} OR (created_at = {created_at} AND instance_id > '{instance_id}'))" - )); - } - - let instances = { - let mut storage_processor = local_db.acquire().await?; - let (instances, _) = storage_processor.find_instances(query).await?; - instances - }; - - if instances.is_empty() { - reset_instance_page_state(task_key); - return Ok(instances); - } - - if instances.len() < batch_size as usize { - reset_instance_page_state(task_key); - } else if let Some(last) = instances.last() { - advance_instance_page_state( - task_key, - page_state.watermark, - (last.created_at, last.instance_id), - ); - } - - Ok(instances) -} - -async fn update_instance<'a>( - storage_processor: &mut StorageProcessor<'a>, - params: &InstanceUpdate, -) -> anyhow::Result { - match storage_processor.update_instance(params).await { - Ok(updated) => { - if updated { - info!("update instance with input: {:?}", params); - } else { - info!("skip stale instance update with input: {:?}", params); - } - Ok(updated) - } - Err(err) => Err(err.context("update instance")), - } -} - -/// for committee -/// handle bridge-in request events on GOAT -pub async fn instance_answers_monitor( - local_db: &LocalDB, - btc_client: &BTCClient, - goat_client: &GOATClient, -) -> anyhow::Result<()> { - let tx_records = { - let mut storage_processor = local_db.acquire().await?; - storage_processor - .get_goat_tx_record_by_processing_status( - &GoatTxType::BridgeInRequest.to_string(), - &GoatTxProcessingStatus::Pending.to_string(), - ) - .await? - }; - let current_height = goat_client.get_finalized_block_number().await?; - let response_window_blocks = goat_client.gateway_get_response_window_blocks().await? as i64; - for tx_record in tx_records { - let event = tx_record - .extra - .as_deref() - .map(serde_json::from_str::) - .transpose()?; - let is_outside_response_window = tx_record.height + response_window_blocks < current_height; - let discarded_instance = if is_outside_response_window { - if let Some(event) = event.as_ref() { - info!( - "instance_answers_monitor: instance_id:{} BridgeInRequest is outside the response window", - tx_record.instance_id - ); - - if let Ok((mut instance, input_utxo_available)) = - generate_instance_from_bridge_in_request_event( - btc_client, - goat_client, - event, - true, - ) - .await - && !input_utxo_available - { - info!( - "instance_answers_monitor: instance_id:{} BridgeInRequest is outside the response window and input utxos not available", - tx_record.instance_id - ); - // for the case: if bridgeIn confirm is broadcast,but L2 not minted, - // instance status will been updated to L2Minted when normal finished - instance.status = InstanceBridgeInStatus::UserDiscarded.to_string(); - Some(instance) - } else { - None - } - } else { - None - } - } else { - None - }; - - let mut tx = local_db.start_transaction().await?; - if let Some(event) = event { - if is_outside_response_window { - if let Some(instance) = discarded_instance - && !tx.insert_instance_if_absent(&instance).await? - && !tx - .update_instance( - &InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(InstanceBridgeInStatus::UserDiscarded.to_string()) - .with_only_if_status_in(vec![ - InstanceBridgeInStatus::UserIniting.to_string(), - InstanceBridgeInStatus::UserInited.to_string(), - ]), - ) - .await? - { - info!("skip stale UserDiscarded update for instance {}", instance.instance_id); - } - } else { - upsert_message( - &mut tx, - false, - SELF_SENDER.to_string(), - Actor::All, - GOATMessageContent::PeginRequest(PeginRequest { - instance_id: tx_record.instance_id, - pegin_request_tx_hash: tx_record.tx_hash, - pegin_request_height: tx_record.height, - pegin_timestamp: event - .block_timestamp - .parse::() - .unwrap_or_else(|_| current_time_secs()), - }), - 0, - 0, - ) - .await?; - } - } - - tx.update_goat_tx_record_processing_status( - &tx_record.graph_id, - &tx_record.instance_id, - &tx_record.tx_type, - &GoatTxProcessingStatus::Processed.to_string(), - ) - .await?; - tx.commit().await?; - } - Ok(()) -} - -/// check committee answers for bridge-in request after response window expired -pub async fn instance_window_expiration_monitor( - local_db: &LocalDB, - goat_client: &GOATClient, -) -> anyhow::Result<()> { - let batch_size = get_instance_maintenance_batch_size(); - let window_blocks = goat_client.gateway_get_response_window_blocks().await? as i64; - let current_height = goat_client.get_latest_block_number().await?; - let instances = find_one_instance_page( - local_db, - TASK_KEY_INSTANCE_WINDOW_EXPIRATION, - InstanceQuery::default() - .with_status(InstanceBridgeInStatus::UserInited.to_string()) - .with_pegin_request_height_threshold(current_height - window_blocks), - batch_size, - ) - .await?; - - let committee_quorum_size = goat_client.committee_mana_quorum_size().await?; - for snapshot in instances { - let pegin_data = match goat_client.gateway_get_pegin_data(&snapshot.instance_id).await { - Ok(pegin_data) => pegin_data, - Err(err) => { - warn!( - "failed to get pegin data for instance {}, err: {}", - snapshot.instance_id, err - ); - continue; - } - }; - - // The RPC call above can take arbitrarily long. Re-read while holding - // SQLite's write lock, then make the decision and its update in the - // same short transaction so a late committee response cannot be - // overwritten by the stale page snapshot. - let mut storage_processor = local_db.start_immediate_transaction().await?; - let Some(mut instance) = storage_processor.find_instance(&snapshot.instance_id).await? - else { - storage_processor.commit().await?; - continue; - }; - if instance.status != InstanceBridgeInStatus::UserInited.to_string() { - info!( - "skip expired response-window reconciliation for instance {} in status {}", - instance.instance_id, instance.status - ); - storage_processor.commit().await?; - continue; - } - - for (committee_addr, pubkey) in - pegin_data.committee_addresses.iter().zip(pegin_data.committee_pubkeys) - { - instance.committees_answers.insert(committee_addr.to_string(), pubkey); - } - - let reached_quorum = committee_quorum_size <= instance.committees_answers.len() as u64; - let next_status = if reached_quorum { - if let Err(err) = update_pegin_txids(&mut instance) { - warn!( - "instance_window_expiration_monitor failed to update pegin txids for instance {}, err: {err:?}", - instance.instance_id - ); - } - InstanceBridgeInStatus::CommitteesAnswered - } else { - InstanceBridgeInStatus::NoEnoughCommitteesAnswered - }; - - let mut update = InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(next_status.to_string()) - .with_committees_answers(instance.committees_answers.clone()) - .with_only_if_status_in(vec![InstanceBridgeInStatus::UserInited.to_string()]); - if reached_quorum { - if let Some(btc_txid) = instance.btc_txid.clone() { - update = update.with_btc_txid(btc_txid); - } - if let Some(pegin_confirm_txid) = instance.pegin_confirm_txid.clone() { - update = update.with_pegin_confirm_txid(pegin_confirm_txid); - } - if let Some(pegin_cancel_txid) = instance.pegin_cancel_txid.clone() { - update = update.with_pegin_cancel_txid(pegin_cancel_txid); - } - } - - if !storage_processor.update_instance(&update).await? { - warn!("skip stale response-window update for instance {}", instance.instance_id); - } - storage_processor.commit().await?; - } - - Ok(()) -} - -fn update_pegin_txids(instance: &mut Instance) -> anyhow::Result<()> { - let (pegin_deposit_tx, pegin_confirm_tx, pegin_refund_tx) = - gen_instance_parameters_local(instance)?.build_pegin_tx()?; - instance.btc_txid = Some(pegin_deposit_tx.tx().compute_txid().into()); - instance.pegin_confirm_txid = Some(pegin_confirm_tx.finalize().compute_txid().into()); - instance.pegin_cancel_txid = Some(pegin_refund_tx.finalize().compute_txid().into()); - Ok(()) -} - -/// check pegin-cancel timelock -pub async fn instance_expiration_monitor( - local_db: &LocalDB, - btc_client: &BTCClient, -) -> anyhow::Result<()> { - let batch_size = get_instance_maintenance_batch_size(); - let current_time = current_time_secs(); - let current_height = btc_client.get_height().await? as i64; - let instances = { - let mut storage_processor = local_db.acquire().await?; - let expired_num = storage_processor - .update_expired_instance( - &InstanceBridgeInStatus::UserBroadcastPeginPrepare.to_string(), - &InstanceBridgeInStatus::PresignedFailed.to_string(), - current_time - get_instance_presigned_time_expired_secs(), - ) - .await?; - info!("Presigned expired instances is {expired_num}"); - let _ = storage_processor; - find_one_instance_page( - local_db, - TASK_KEY_INSTANCE_EXPIRATION, - InstanceQuery::default().with_statuses(vec![ - InstanceBridgeInStatus::Presigned.to_string(), - InstanceBridgeInStatus::PresignedFailed.to_string(), - ]), - batch_size, - ) - .await? - }; - - let lock_height = default_connector_z_timelock_blocks(btc_client.network()) as i64; - let mut storage_processor = local_db.acquire().await?; - for instance in instances { - if instance.btc_height > 0 && current_height > instance.btc_height + lock_height { - update_instance( - &mut storage_processor, - &InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(InstanceBridgeInStatus::Timeout.to_string()) - .with_only_if_status_in(vec![instance.status.clone()]), - ) - .await?; - } else { - info!("instance;{} not expired", instance.instance_id); - } - } - Ok(()) -} - -/// check if pegin-deposit/cancel/confirm tx confirmed on chain, and update instance status accordingly -pub async fn instance_btc_tx_monitor( - local_db: &LocalDB, - btc_client: &BTCClient, -) -> anyhow::Result<()> { - info!("check user broadcast Pegin-Prepare"); - let batch_size = get_instance_maintenance_batch_size(); - - let instances = find_one_instance_page( - local_db, - TASK_KEY_INSTANCE_BTC_TX, - InstanceQuery::default().with_statuses(vec![ - InstanceBridgeInStatus::UserInited.to_string(), - InstanceBridgeInStatus::CommitteesAnswered.to_string(), - InstanceBridgeInStatus::Presigned.to_string(), - InstanceBridgeInStatus::Timeout.to_string(), - ]), - batch_size, - ) - .await?; - for instance in instances { - let (txid_op, next_status) = match InstanceBridgeInStatus::from_str(&instance.status) { - Ok(status) => match status { - InstanceBridgeInStatus::UserInited => { - (None, InstanceBridgeInStatus::CommitteesAnswered) - } - InstanceBridgeInStatus::CommitteesAnswered => { - (instance.btc_txid.clone(), InstanceBridgeInStatus::UserBroadcastPeginPrepare) - } - InstanceBridgeInStatus::Presigned => ( - instance.pegin_confirm_txid.clone(), - InstanceBridgeInStatus::RelayerL1Broadcasted, - ), - InstanceBridgeInStatus::Timeout => { - (instance.pegin_cancel_txid.clone(), InstanceBridgeInStatus::UserCanceled) - } - _ => (None, status), - }, - Err(err) => { - warn!( - "failed to parse instance:{}, {} status: {}", - instance.instance_id, - instance.status, - err.to_string() - ); - continue; - } - }; - if let Some(txid) = txid_op.clone() - && let Ok(status) = btc_client.get_tx_status(&txid.0).await - && status.confirmed - { - let mut tx = local_db.start_transaction().await?; - let mut instance_update = InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(next_status.to_string()) - .with_only_if_status_in(vec![instance.status.clone()]); - if next_status == InstanceBridgeInStatus::UserBroadcastPeginPrepare { - instance_update = - instance_update.with_btc_height(status.block_height.unwrap_or_default() as i64); - } - - if update_instance(&mut tx, &instance_update).await? { - match next_status { - InstanceBridgeInStatus::UserBroadcastPeginPrepare => { - upsert_message( - &mut tx, - false, - SELF_SENDER.to_string(), - Actor::All, - GOATMessageContent::ConfirmInstance(ConfirmInstance { - instance_id: instance.instance_id, - }), - 0, - 0, - ) - .await?; - } - InstanceBridgeInStatus::RelayerL1Broadcasted => { - upsert_message( - &mut tx, - false, - SELF_SENDER.to_string(), - Actor::All, - GOATMessageContent::PostReady(PostReady { - instance_id: instance.instance_id, - }), - 0, - 0, - ) - .await?; - } - _ => {} - } - } - tx.commit().await?; - } else { - warn!( - "instance:{}, status{}, check tx_id:{:?} is not chain ", - instance.instance_id, instance.status, txid_op - ); - if [ - InstanceBridgeInStatus::CommitteesAnswered, - InstanceBridgeInStatus::UserBroadcastPeginPrepare, - ] - .contains(&next_status) - && let utxos = serde_json::from_str::>(&instance.input_utxos)? - && let Some(user_prepare_tx) = instance.btc_txid - && !check_bridge_in_uxto_available_or_self_spent( - btc_client, - Some(user_prepare_tx.0.to_string()), - &utxos, - ) - .await? - { - warn!( - "instance:{}, pegin prepare tx input utxos has been spent in other tx", - instance.instance_id - ); - let mut storage_processor = local_db.acquire().await?; - update_instance( - &mut storage_processor, - &InstanceUpdate::new_with_instance_id(instance.instance_id) - .with_status(InstanceBridgeInStatus::UserDiscarded.to_string()) - .with_only_if_status_in(vec![instance.status.clone()]), - ) - .await?; - } - } - } - Ok(()) -} - -/// Re-publish persisted PeginConfirm signing material until the transaction is visible on Bitcoin. -/// This never derives a new partial signature: it only reuses the deterministic nonce or stored signature. -pub async fn pegin_confirm_recovery_monitor( - local_db: &LocalDB, - btc_client: &BTCClient, - actor: &Actor, -) -> anyhow::Result<()> { - if actor != &Actor::Committee { - return Ok(()); - } - - let instances = find_one_instance_page( - local_db, - TASK_KEY_PEGIN_CONFIRM_RECOVERY, - InstanceQuery::default().with_status(InstanceBridgeInStatus::Presigned.to_string()), - get_instance_maintenance_batch_size(), - ) - .await?; - if instances.is_empty() { - return Ok(()); - } - - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - for instance in instances { - let Some(pegin_confirm_txid) = instance.pegin_confirm_txid.clone() else { - warn!( - instance_id = %instance.instance_id, - "skip PeginConfirm recovery: instance has no expected PeginConfirm txid" - ); - continue; - }; - match btc_client.get_tx(&pegin_confirm_txid.0).await { - Ok(Some(_)) => continue, - Ok(None) => {} - Err(error) => { - warn!( - instance_id = %instance.instance_id, - error = %error, - "defer PeginConfirm recovery: unable to query Bitcoin transaction" - ); - continue; - } - } - - let instance_id = instance.instance_id; - let instance_parameters = gen_instance_parameters_local(&instance)?; - let instance_keypair = load_committee_instance_keypair(&committee_master_key, instance_id)?; - let local_committee_pubkey = instance_keypair.public_key().into(); - - if let Some(partial_sig) = - get_committee_partial_sig_for_instance(local_db, instance_id, &local_committee_pubkey) - .await? - { - let Some((_, endorse_sig)) = - get_committee_endorse_sigs_for_pegin(local_db, instance_id) - .await? - .into_iter() - .find(|(pubkey, _)| *pubkey == local_committee_pubkey) - else { - warn!( - instance_id = %instance_id, - "skip PeginConfirm partial signature recovery: local endorsement signature is missing" - ); - continue; - }; - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::PeginConfirmPartialSig(PeginConfirmPartialSig { - instance_id, - committee_pubkey: local_committee_pubkey, - partial_sig, - endorse_sig, - }), - ); - if !finish_recovery_enqueue( - defer_or_enqueue_message( - local_db, - &message, - 0, - MessageDeferReason::RecoveryRepublish, - "re-publishing persisted pegin-confirm partial signature", - ) - .await, - instance_id, - "republish_partial_signature", - )? { - continue; - } - tracing::info!( - event = "pegin_confirm_recovery", - action = "republish_partial_signature", - instance_id = %instance_id, - "queued persisted PeginConfirm partial signature for re-publication" - ); - continue; - } - - let (_, derived_pub_nonce, nonce_sig) = committee_master_key - .nonce_for_instance_job_with_keypair(&instance_parameters, instance_keypair)?; - let pub_nonce = match get_committee_pub_nonce_for_instance( - local_db, - instance_id, - &local_committee_pubkey, - ) - .await? - { - Some(stored_pub_nonce) => { - if stored_pub_nonce != derived_pub_nonce { - anyhow::bail!( - "stored PeginConfirm nonce differs from deterministic nonce for instance {instance_id}" - ); - } - stored_pub_nonce - } - None => { - store_committee_pub_nonce_for_instance( - local_db, - instance_id, - local_committee_pubkey, - derived_pub_nonce.clone(), - ) - .await?; - derived_pub_nonce - } - }; - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::PeginConfirmNonce(PeginConfirmNonce { - instance_id, - committee_pubkey: local_committee_pubkey, - pub_nonce, - nonce_sig, - }), - ); - if !finish_recovery_enqueue( - defer_or_enqueue_message( - local_db, - &message, - 0, - MessageDeferReason::RecoveryRepublish, - "re-publishing persisted pegin-confirm nonce", - ) - .await, - instance_id, - "republish_nonce", - )? { - continue; - } - tracing::info!( - event = "pegin_confirm_recovery", - action = "republish_nonce", - instance_id = %instance_id, - "queued persisted PeginConfirm nonce for re-publication" - ); - if let Some((_, consensus_hash, signature)) = - get_committee_agg_nonce_consensus_for_instance(local_db, instance_id) - .await? - .into_iter() - .find(|(pubkey, _, _)| *pubkey == local_committee_pubkey) - { - let message = GOATMessage::new( - Actor::Committee, - GOATMessageContent::PeginConfirmNonceConsensus(PeginConfirmNonceConsensus { - instance_id, - committee_pubkey: local_committee_pubkey, - consensus_hash, - signature, - }), - ); - if !finish_recovery_enqueue( - defer_or_enqueue_message( - local_db, - &message, - 0, - MessageDeferReason::RecoveryRepublish, - "re-publishing persisted PeginConfirm nonce consensus", - ) - .await, - instance_id, - "republish_nonce_consensus", - )? { - continue; - } - tracing::info!( - event = "pegin_confirm_recovery", - action = "republish_nonce_consensus", - instance_id = %instance_id, - "queued persisted PeginConfirm nonce consensus for re-publication" - ); - } - } - Ok(()) -} - -/// Mark still-initializing swap escrows whose refund deadline has passed as -/// timed out. -pub async fn swap_escrow_timeout_monitor(local_db: &LocalDB) -> anyhow::Result<()> { - let mut storage_processor = local_db.acquire().await?; - let timed_out = storage_processor.timeout_expired_swap_escrows(current_time_secs()).await?; - if timed_out > 0 { - info!("marked {timed_out} swap escrows as timed out"); - } - Ok(()) -} - -fn list_committee_instance_key_envelopes() -> anyhow::Result> { - let dir = PathBuf::from(COMMITTEE_INSTANCE_KEYS_DIR); - if !dir.exists() { - return Ok(vec![]); - } - - let mut envelopes = Vec::new(); - for entry in fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - if !path.is_file() { - continue; - } - if path.extension().and_then(|ext| ext.to_str()) != Some("json") { - continue; - } - let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else { - warn!("invalid committee key envelope filename: {}", path.display()); - continue; - }; - match Uuid::parse_str(stem) { - Ok(instance_id) => envelopes.push((instance_id, path)), - Err(err) => { - warn!("skip non-instance envelope file {}: {}", path.display(), err); - } - } - } - Ok(envelopes) -} - -pub async fn instance_committee_key_cleanup_monitor( - local_db: &LocalDB, - btc_client: &BTCClient, -) -> anyhow::Result<()> { - if !is_enable_committee_instance_key_delete() { - return Ok(()); - } - - let envelopes = list_committee_instance_key_envelopes()?; - if envelopes.is_empty() { - return Ok(()); - } - - let current_height = btc_client.get_height().await? as i64; - let delete_timelock_blocks = get_committee_instance_key_delete_timelock_blocks(); - let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); - let mut storage_processor = local_db.acquire().await?; - - for (instance_id, envelope_path) in envelopes { - let Some(instance) = storage_processor.find_instance(&instance_id).await? else { - warn!( - "committee key envelope exists but instance missing: {} ({})", - instance_id, - envelope_path.display() - ); - continue; - }; - - let Some(pegin_confirm_txid) = instance.pegin_confirm_txid else { - continue; - }; - - let tx_status = match btc_client.get_tx_status(&pegin_confirm_txid.0).await { - Ok(v) => v, - Err(err) => { - warn!( - "failed to query pegin-confirm tx status for instance {}: {}", - instance_id, err - ); - continue; - } - }; - if !tx_status.confirmed { - continue; - } - let Some(confirmed_height) = tx_status.block_height else { - continue; - }; - if (confirmed_height as i64 + delete_timelock_blocks) > current_height { - continue; - } - - match committee_master_key.delete_instance_keypair_envelope(instance_id, &envelope_path) { - Ok(()) => { - info!( - "deleted committee instance key envelope for {} at {}", - instance_id, - envelope_path.display() - ); - } - Err(err) => { - warn!( - "failed to delete committee instance key envelope for {} at {}: {}", - instance_id, - envelope_path.display(), - err - ); - } - } - } - - Ok(()) -} diff --git a/node/src/scheduled_tasks/mod.rs b/node/src/scheduled_tasks/mod.rs index 719415d9a..50aba5552 100644 --- a/node/src/scheduled_tasks/mod.rs +++ b/node/src/scheduled_tasks/mod.rs @@ -1,40 +1,35 @@ mod babe_setup_state_cleanup_task; -mod event_watch_task; -pub mod graph_maintenance_tasks; -pub mod instance_maintenance_tasks; +mod event_watch; mod node_maintenance_tasks; mod sequencer_set_hash_monitor_task; mod spv_maintenance_tasks; +mod swap_maintenance_tasks; +use crate::aggregates::graph_scan::{abandon_unfinished_setups, scan_graphs_on_chain}; +use crate::aggregates::instance_scan::{ + pegin_request_monitor, scan_answer_windows, scan_instances_on_bitcoin, +}; use crate::env::{ actor_runs_babe_setup_state_cleanup, get_maintenance_run_timeout_secs, get_network, get_node_goat_address, get_node_pubkey, is_enable_babe_setup_state_cleanup, is_enable_update_spv_contract, is_relayer, }; use crate::metrics_service::MetricsState; -use crate::rpc_service::current_time_secs; use crate::scheduled_tasks::babe_setup_state_cleanup_task::babe_setup_state_cleanup_monitor; -use crate::scheduled_tasks::graph_maintenance_tasks::{ - detect_init_withdraw_call, detect_kickoff, detect_take1_or_challenge, process_graph_challenge, -}; -use crate::scheduled_tasks::instance_maintenance_tasks::{ - instance_answers_monitor, instance_btc_tx_monitor, instance_committee_key_cleanup_monitor, - instance_expiration_monitor, instance_window_expiration_monitor, - pegin_confirm_recovery_monitor, swap_escrow_timeout_monitor, -}; use crate::scheduled_tasks::node_maintenance_tasks::node_available_pbtc_update_monitor; use crate::scheduled_tasks::spv_maintenance_tasks::spv_header_hash_update; +use crate::scheduled_tasks::swap_maintenance_tasks::swap_escrow_timeout_monitor; use crate::utils::node_p2wsh_address; use bitvm_lib::actors::Actor; use client::btc_chain::BTCClient; use client::goat_chain::GOATClient; -pub use event_watch_task::{is_processing_gateway_history_events, run_watch_event_task}; +use engine::EngineHandle; +pub use event_watch::{is_processing_gateway_history_events, run_watch_event_task}; pub use sequencer_set_hash_monitor_task::run_sequencer_set_hash_monitor_task; use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; -use store::Graph; -use store::localdb::{LocalDB, StorageProcessor}; +use store::localdb::LocalDB; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -57,15 +52,14 @@ async fn refresh_alert_health( Some(height) } Err(error) => { - metrics_state.record_btc_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "btc", error = %error, "BTC backend health probe failed"); + metrics_state.backend_probe_failed("btc", "get_height", &error); None } }; let goat_height_healthy = goat_height.is_ok(); - metrics_state.record_goat_backend_probe(goat_height_healthy); - if let Err(error) = goat_height { - warn!(event = "metrics_backend_probe", backend = "goat", operation = "get_latest_block_number", error = %error, "Goat backend health probe failed"); + match &goat_height { + Ok(_) => metrics_state.record_goat_backend_probe(true), + Err(error) => metrics_state.backend_probe_failed("goat", "get_latest_block_number", error), } let goat_spv_height = match goat_spv_height { Ok(height) => { @@ -73,8 +67,7 @@ async fn refresh_alert_health( Some(height) } Err(error) => { - metrics_state.record_goat_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "goat", operation = "btc_spv_latest_height", error = %error, "Goat SPV health probe failed"); + metrics_state.backend_probe_failed("goat", "btc_spv_latest_height", &error); None } }; @@ -84,8 +77,7 @@ async fn refresh_alert_health( metrics_state.set_peg_btc_decimals(decimals); } Err(error) => { - metrics_state.record_goat_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "goat", operation = "peg_btc_decimals", error = %error, "failed to load pegBTC decimals"); + metrics_state.backend_probe_failed("goat", "peg_btc_decimals", &error); } } metrics_state.apply_chain_health(btc_height, goat_spv_height); @@ -105,8 +97,7 @@ async fn refresh_alert_health( ); } Err(error) => { - metrics_state.record_btc_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "btc", operation = "get_address_utxo", error = %error, "BTC fee wallet probe failed"); + metrics_state.backend_probe_failed("btc", "get_address_utxo", &error); } } } @@ -122,8 +113,7 @@ async fn refresh_alert_health( metrics_state.apply_goat_gas_balance(balance); } Err(error) => { - metrics_state.record_goat_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "goat", operation = "native_balance", error = %error, "Goat gas balance probe failed"); + metrics_state.backend_probe_failed("goat", "native_balance", &error); } } } @@ -148,8 +138,7 @@ async fn refresh_alert_health( metrics_state.apply_required_stake(sufficient); } Err(error) => { - metrics_state.record_goat_backend_probe(false); - warn!(event = "metrics_backend_probe", backend = "goat", operation = "required_stake", error = %error, "required stake probe failed"); + metrics_state.backend_probe_failed("goat", "required_stake", &error); } } } else { @@ -157,33 +146,6 @@ async fn refresh_alert_health( } } -/// Return every graph in the requested status. Time-sensitive flows must not -/// let a lower-index graph hide another graph owned by the same operator. -pub(super) async fn fetch_all_graphs_by_status<'a>( - storage_processor: &mut StorageProcessor<'a>, - graph_status: &str, -) -> anyhow::Result> { - storage_processor.find_graphs_by_status_group_by_operator(graph_status).await -} - -/// Return the lowest-index graph for each operator in the requested status. -/// This is only suitable as the entry point for a chain-aware scan. -pub(super) async fn fetch_first_graph_per_operator_by_status<'a>( - storage_processor: &mut StorageProcessor<'a>, - graph_status: &str, -) -> anyhow::Result> { - let graphs_ori = fetch_all_graphs_by_status(storage_processor, graph_status).await?; - let mut graphs: Vec = vec![]; - let mut pre_operator_pubkey = "".to_string(); - for graph in graphs_ori { - if graph.operator_pubkey != pre_operator_pubkey { - pre_operator_pubkey = graph.operator_pubkey.clone(); - graphs.push(graph); - } - } - Ok(graphs) -} - async fn run_maintenance_subtask( metrics_state: &MetricsState, task: &'static str, @@ -221,6 +183,9 @@ async fn run_maintenance_subtask( enum MaintenanceRunOutcome { Completed, DeferredHistorySync, + /// The history-sync check itself failed (Goat read): protocol work is + /// deferred to the next tick, not reported as a tick error. + DeferredHistorySyncUnknown, } async fn run( @@ -229,6 +194,7 @@ async fn run( btc_client: Arc, goat_client: Arc, metrics_state: &MetricsState, + engine: &EngineHandle, ) -> anyhow::Result { let btc_client = btc_client.as_ref(); let goat_client = goat_client.as_ref(); @@ -260,7 +226,23 @@ async fn run( .await; } - if is_processing_gateway_history_events(local_db, goat_client).await? { + let history_sync_in_progress = match is_processing_gateway_history_events(local_db, goat_client) + .await + { + Ok(in_progress) => in_progress, + Err(error) => { + warn!( + event = "maintenance_subtask_result", + task = "gateway_history_sync", + outcome = "failed", + error_class = "maintenance", + error = %error, + "could not check the gateway history sync; protocol work deferred to the next tick" + ); + return Ok(MaintenanceRunOutcome::DeferredHistorySyncUnknown); + } + }; + if history_sync_in_progress { info!( event = "maintenance_subtask_result", task = "gateway_history_sync", @@ -272,40 +254,24 @@ async fn run( return Ok(MaintenanceRunOutcome::DeferredHistorySync); } + // New peg-in requests become instance rows and `pegin_request` events. run_maintenance_subtask( metrics_state, - "instance_answers_monitor", - instance_answers_monitor(local_db, btc_client, goat_client), - ) - .await; - run_maintenance_subtask( - metrics_state, - "instance_window_expiration_monitor", - instance_window_expiration_monitor(local_db, goat_client), + "pegin_request_monitor", + pegin_request_monitor(local_db, btc_client, goat_client, engine), ) .await; + // Chain facts for the Instance aggregate; the apply decides every status. run_maintenance_subtask( metrics_state, - "instance_expiration_monitor", - instance_expiration_monitor(local_db, btc_client), + "instance_answer_window_scan", + scan_answer_windows(local_db, goat_client, engine), ) .await; run_maintenance_subtask( metrics_state, - "instance_btc_tx_monitor", - instance_btc_tx_monitor(local_db, btc_client), - ) - .await; - run_maintenance_subtask( - metrics_state, - "pegin_confirm_recovery_monitor", - pegin_confirm_recovery_monitor(local_db, btc_client, &actor), - ) - .await; - run_maintenance_subtask( - metrics_state, - "instance_committee_key_cleanup_monitor", - instance_committee_key_cleanup_monitor(local_db, btc_client), + "instance_btc_scan", + scan_instances_on_bitcoin(local_db, btc_client, engine, &actor), ) .await; run_maintenance_subtask( @@ -314,29 +280,32 @@ async fn run( swap_escrow_timeout_monitor(local_db), ) .await; + // Rejected and stale unregistered network events leave the inbox quota. run_maintenance_subtask( metrics_state, - "detect_init_withdraw_call", - detect_init_withdraw_call(local_db), - ) - .await; - run_maintenance_subtask(metrics_state, "detect_kickoff", detect_kickoff(local_db, btc_client)) - .await; - run_maintenance_subtask( - metrics_state, - "detect_take1_or_challenge", - detect_take1_or_challenge(local_db, btc_client), + "inbox_quota_purge", + crate::p2p_admission::purge_stale_network_events(local_db), ) .await; + // Chain facts for the Graph aggregate's peg-out phases. run_maintenance_subtask( metrics_state, - "process_graph_challenge", - process_graph_challenge(local_db, btc_client), + "graph_chain_scan", + scan_graphs_on_chain(local_db, btc_client, goat_client, engine), ) .await; + if actor == Actor::Operator { + run_maintenance_subtask( + metrics_state, + "graph_setup_abandon_scan", + abandon_unfinished_setups(local_db, btc_client, engine), + ) + .await; + } Ok(MaintenanceRunOutcome::Completed) } +#[allow(clippy::too_many_arguments)] pub async fn run_maintenance_tasks( actor: Actor, local_db: LocalDB, @@ -345,6 +314,7 @@ pub async fn run_maintenance_tasks( interval: u64, cancellation_token: CancellationToken, metrics_state: MetricsState, + engine: EngineHandle, ) -> anyhow::Result { let mut tick: u64 = 0; let maintenance_run_timeout = Duration::from_secs(get_maintenance_run_timeout_secs()); @@ -369,6 +339,7 @@ pub async fn run_maintenance_tasks( btc_client.clone(), goat_client.clone(), &metrics_state, + &engine, ), ).await { Ok(Ok(MaintenanceRunOutcome::Completed)) => { @@ -392,6 +363,17 @@ pub async fn run_maintenance_tasks( "maintenance protocol work was deferred" ); } + Ok(Ok(MaintenanceRunOutcome::DeferredHistorySyncUnknown)) => { + metrics_state.record_task_run("maintenance", "failed", tick_start.elapsed()); + warn!( + event = "maintenance_tick_result", + tick, + outcome = "deferred", + reason = "history_sync_check_failed", + elapsed_ms = tick_start.elapsed().as_millis() as u64, + "maintenance protocol work was deferred" + ); + } Ok(Err(error)) => { metrics_state.record_task_run("maintenance", "failed", tick_start.elapsed()); error!( @@ -436,40 +418,6 @@ pub async fn run_maintenance_tasks( warn!(event = "metrics_alert_health", "alert health probe timed out"); } }); - let queue_started_at = Instant::now(); - match local_db.acquire().await { - Ok(mut storage) => match storage - .get_message_queue_stats(&actor.to_string(), current_time_secs()) - .await - { - Ok(stats) => info!( - event = "message_queue_snapshot", - role = %actor, - pending_ready = stats.pending_ready, - pending_locked = stats.pending_locked, - failed = stats.failed, - oldest_pending_at = ?stats.oldest_pending_at, - elapsed_ms = queue_started_at.elapsed().as_millis() as u64, - "local message queue snapshot" - ), - Err(error) => error!( - event = "db_operation_result", - operation = "get_message_queue_stats", - outcome = "failed", - elapsed_ms = queue_started_at.elapsed().as_millis() as u64, - error = %error, - "failed to collect local message queue snapshot" - ), - }, - Err(error) => error!( - event = "db_operation_result", - operation = "acquire_db_for_message_queue_snapshot", - outcome = "failed", - elapsed_ms = queue_started_at.elapsed().as_millis() as u64, - error = %error, - "failed to acquire local database for message queue snapshot" - ), - } } } _ = cancellation_token.cancelled() => { diff --git a/node/src/scheduled_tasks/swap_maintenance_tasks.rs b/node/src/scheduled_tasks/swap_maintenance_tasks.rs new file mode 100644 index 000000000..10481b206 --- /dev/null +++ b/node/src/scheduled_tasks/swap_maintenance_tasks.rs @@ -0,0 +1,17 @@ +//! Swap escrow maintenance: escrows whose refund deadline has passed are +//! marked timed out. + +use crate::clock::current_time_secs; +use store::localdb::LocalDB; +use tracing::info; + +/// Mark still-initializing swap escrows whose refund deadline has passed as +/// timed out. +pub async fn swap_escrow_timeout_monitor(local_db: &LocalDB) -> anyhow::Result<()> { + let mut storage_processor = local_db.acquire().await?; + let timed_out = storage_processor.timeout_expired_swap_escrows(current_time_secs()).await?; + if timed_out > 0 { + info!("marked {timed_out} swap escrows as timed out"); + } + Ok(()) +} diff --git a/node/src/utils.rs b/node/src/utils.rs deleted file mode 100644 index e427b354a..000000000 --- a/node/src/utils.rs +++ /dev/null @@ -1,6710 +0,0 @@ -use crate::action::{ - BusinessRef, ChallengeSent, DisproveSent, GOATMessage, GOATMessageContent, HasBusinessRef, - KickoffSent, LocalMessageKey, MessageKind, NodeInfo, PreKickoffSent, SolderingProofReady, - Take1Sent, Take2Sent, send_to_peer, -}; -use crate::env::*; -use crate::error::SpecialError; -use crate::middleware::AllBehaviours; -use crate::rpc_service::current_time_secs; -use crate::soldering_payload_store::{ - delete_soldering_proof_store_payload, is_soldering_proof_s3_path, - soldering_proof_payload_store_path, write_soldering_proof_store_payload, -}; -use alloy::primitives::{Address as EvmAddress, Signature as EvmSignature}; -use alloy::signers::Signer; -use alloy::signers::local::PrivateKeySigner; -use anyhow::{Context, Result, anyhow, bail}; -use bitcoin::address::NetworkUnchecked; -use bitcoin::consensus::encode::deserialize; -use bitcoin::hashes::Hash; -use bitcoin::key::Keypair; -use bitcoin::{ - Address, Amount, BlockHash, CompressedPublicKey, EcdsaSighashType, Network, OutPoint, - PrivateKey, PublicKey, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, Witness, - XOnlyPublicKey, -}; -use bitcoin_light_client_circuit::{ - VK_HASH_SIZE, build_watchtower_commitment, decode_operator_public_outputs, -}; -use bitvm::treepp::*; -use bitvm_lib::actors::Actor; -use bitvm_lib::committee::*; -use bitvm_lib::keys::{ - CommitteeMasterKey, OperatorMasterKey, VerifierMasterKey, WatchtowerMasterKey, -}; -use bitvm_lib::operator::*; -use bitvm_lib::timelocks::{ - connector_f_timelock_blocks, default_timelock_config, estimated_block_interval_secs, -}; -use bitvm_lib::types::{ - BitvmGcCircuitData, BitvmGcGraph, BitvmGcGraphParameters, BitvmGcInstanceParameters, - PrekickoffParameters, SimplifiedBitvmGcGraph, UserInfo, -}; -use bitvm_lib::verifier::*; -use bitvm_lib::watchtower::*; -use client::Utxo as ClientUtxo; -use client::{btc_chain::BTCClient, goat_chain::GOATClient}; -use esplora_client::Utxo; -use goat::connectors::{ - base::TaprootConnector, - kickoff_connectors::{ForceSkipConnector, KickoffConnector, PrekickoffConnector}, -}; -use goat::constants::TimelockConfig; -use goat::contexts::base::generate_n_of_n_public_key; -use goat::scripts::{generate_opreturn_script, p2a_output}; -use goat::transactions::base::{Input, output_topology}; -use goat::transactions::pre_signed::PreSignedTransaction; -use goat::transactions::prekickoff::PrekickoffTransaction; -use goat::transactions::signing::populate_p2wsh_witness; -use indexmap::IndexMap; -use libp2p::{PeerId, Swarm}; -use musig2::{PartialSignature, PubNonce}; -use p3_bn254_fr::Bn254Fr; -use p3_field::{PrimeCharacteristicRing, PrimeField}; -use rand::Rng; -use reqwest::Url; -use secp256k1::schnorr::Signature as SchnorrSignature; -use secp256k1::{Message as SecpMessage, SECP256K1, Secp256k1}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use store::localdb::{GraphQuery, GraphRuntimeUpdate, LocalDB, StorageProcessor}; - -use crate::env; -use crate::rpc_service::routes::v1::{ - NODES_OPERATOR_BASE, NODES_WATCHTOWER_BASE, PROOFS_WATCHTOWER_PROOF_TIMEOUT, -}; - -use crate::scheduled_tasks::graph_maintenance_tasks::{ - ChallengeSubStatus, VerifierChallengeStatus, -}; -use bitcoin_light_client_circuit::hash_operator_constant; -use bitvm_lib::babe_adapter::{ - BABE_M_CC, BabeVerifierPrivateState, CACSetupPackage, DYNAMIC_SLOTS, FinalizedInstanceData, - NUM_PUBLIC_INPUTS, STATIC_SLOTS, Seed, SolderingData, StaticInputs, - compact_soldering_proof_payload, ziren_static_inputs, -}; -use bitvm_lib::transactions::base::BaseTransaction; -use client::goat_chain::{DisproveTxType, GraphData, PeginStatus, WithdrawStatus}; -use client::http_client::async_client::HttpAsyncClient; -use proof_builder::api_auth::{ProofBuilderAuthRole, sign_proof_builder_request}; -use proof_builder::{ - OperatorProofRequest, OperatorProofResponse, ProofData, WatchtowerProofRequest, - WatchtowerProofResponse, WatchtowerProofTimeoutUpdateRequest, - WatchtowerProofTimeoutUpdateResponse, -}; -use store::{ - BridgeOutGlobalStats, ByteArray32, Graph, GraphRawData, GraphStatus, GraphStatusSource, - GraphStatusTransitionOutcome, Instance, InstanceBridgeInStatus, Message, MessageState, Node, - PeginGraphProcessData, PeginInstanceProcessData, SerializableTxid, UInt64Array3, -}; -use stun_client::{Attribute, Class, Client}; -use tracing::{error, info, warn}; -use uuid::Uuid; -use zkm_pcs::PartStarkVerifyingKey; -use zkm_recursion_core::stark::KoalaBearPoseidon2Outer; -use zkm_sdk::ZKMProofWithPublicValues; -use zkm_verifier::{ - Groth16Verifier, IMM_GROTH16_VK_BYTES, VK_ROOT_BYTES, convert_ark_imm_wrap_vk, - decode_zkm_vkey_hash, -}; - -pub const SELF_SENDER: &str = "self"; -const BABE_SETUP_STATE_ORPHAN_RETENTION: Duration = Duration::from_secs(24 * 60 * 60); - -pub(crate) fn load_committee_instance_keypair( - committee_master_key: &CommitteeMasterKey, - instance_id: Uuid, -) -> Result { - let mut envelope_path = PathBuf::from(COMMITTEE_INSTANCE_KEYS_DIR); - envelope_path.push(format!("{instance_id}.json")); - committee_master_key.load_instance_keypair(instance_id, &envelope_path).with_context(|| { - format!( - "load committee instance keypair failed for {} at {}", - instance_id, - envelope_path.display() - ) - }) -} - -pub(crate) const BRIDGE_OUT_GLOBAL_STATS_ID: i64 = 1; - -pub type VerifyingKey = ark_groth16::VerifyingKey; -pub type Groth16Proof = ark_groth16::Proof; -pub type PublicInputs = Vec; - -#[derive(Clone)] -pub struct ValidatedOperatorProof { - pub proof: Groth16Proof, - pub public_inputs: PublicInputs, - pub verifying_key: VerifyingKey, - pub public_values: Vec, - pub vk_hash: String, - pub zkm_version: String, -} - -#[derive(Clone)] -pub struct OperatorStatement { - pub static_input: StaticInputs, - pub vk_hash: [u8; 32], - pub zkm_version: String, - pub constant: [u8; 32], -} - -// other operations -pub fn avg_block_time_secs(network: Network) -> u64 { - estimated_block_interval_secs(network) as u64 -} -pub fn min_required_operator() -> usize { - // todo!("get min required operator number") - 1 -} -pub fn min_required_watchtower() -> usize { - // todo!("get min required watchtower number") - 1 -} -pub fn min_required_verifier() -> usize { - get_min_required_verifier() - .unwrap_or_else(|error| panic!("invalid verifier count configuration: {error}")) -} - -/// Checks graph Watchtower membership, uniqueness, and supported count. -fn validate_watchtower_registry_selection( - selected: &[XOnlyPublicKey], - registered: &[XOnlyPublicKey], -) -> Result<()> { - use std::collections::HashSet; - - if selected.len() < min_required_watchtower() { - bail!(SpecialError::InvalidGraph(format!( - "insufficient watchtowers: have {}, required {}", - selected.len(), - min_required_watchtower() - ))); - } - if selected.len() > 256 { - bail!(SpecialError::InvalidGraph(format!( - "too many watchtowers: {}, max 256", - selected.len() - ))); - } - - let mut seen = HashSet::new(); - for key in selected { - if !seen.insert(*key) { - bail!(SpecialError::InvalidGraph("duplicate watchtower pubkey in graph".to_string())); - } - if !registered.contains(key) { - bail!(SpecialError::InvalidGraph(format!("watchtower {} is not registered", key))); - } - } - Ok(()) -} - -/// Validates the graph's ordered watchtower selection and its bound Operator constant. -fn validate_watchtower_selection( - selected: &[XOnlyPublicKey], - registered: &[XOnlyPublicKey], - graph_id: [u8; 16], - genesis_txid: [u8; 32], - challenge_init_txid: [u8; 32], - constant: [u8; 32], -) -> Result<()> { - validate_watchtower_registry_selection(selected, registered)?; - let key_bytes = selected.iter().map(XOnlyPublicKey::serialize).collect::>(); - let expected = hash_operator_constant(graph_id, genesis_txid, challenge_init_txid, &key_bytes); - if constant != expected { - bail!(SpecialError::InvalidGraph( - "operator constant mismatch for graph watchtower list".to_string() - )); - } - Ok(()) -} - -pub async fn validate_graph_instance_parameters( - btc_client: &BTCClient, - goat_client: &GOATClient, - parameters: &BitvmGcInstanceParameters, -) -> Result<()> { - let expected = read_instance_info_from_goat(goat_client, parameters.instance_id) - .await - .context("failed to load instance parameters from GoatChain")?; - if parameters != &expected { - bail!(SpecialError::InvalidGraph( - "instance parameters mismatch with GoatChain peg-in data".to_string() - )); - } - - for input in &expected.user_info.inputs { - let funding_tx = btc_client.get_tx(&input.outpoint.txid).await.with_context(|| { - format!("failed to load peg-in funding transaction {}", input.outpoint.txid) - })?; - let Some(funding_tx) = funding_tx else { - bail!(SpecialError::InvalidGraph(format!( - "peg-in funding transaction {} not found on Bitcoin", - input.outpoint.txid - ))); - }; - let funding_output = - funding_tx.output.get(input.outpoint.vout as usize).ok_or_else(|| { - SpecialError::InvalidGraph(format!( - "peg-in funding outpoint {}:{} does not exist", - input.outpoint.txid, input.outpoint.vout - )) - })?; - if funding_output.value != input.amount { - bail!(SpecialError::InvalidGraph(format!( - "peg-in funding amount mismatch for {}:{}: graph={}, bitcoin={}", - input.outpoint.txid, - input.outpoint.vout, - input.amount.to_sat(), - funding_output.value.to_sat() - ))); - } - } - - let pegin_deposit_txid = expected - .build_pegin_tx() - .map_err(|e| { - SpecialError::InvalidGraph(format!("failed to reconstruct peg-in deposit: {e}")) - })? - .0 - .tx() - .compute_txid(); - if btc_client - .get_tx(&pegin_deposit_txid) - .await - .with_context(|| format!("failed to load peg-in deposit transaction {pegin_deposit_txid}"))? - .is_none() - { - bail!(SpecialError::InvalidGraph(format!( - "peg-in deposit transaction {pegin_deposit_txid} not found on Bitcoin" - ))); - } - - Ok(()) -} - -fn validate_graph_policy(parameters: &BitvmGcGraphParameters) -> Result<()> { - let verifier_num = parameters.gc_data.len(); - if verifier_num < min_required_verifier() { - bail!(SpecialError::InvalidGraph(format!( - "insufficient verifier GC slots: have {}, required {}", - verifier_num, - min_required_verifier() - ))); - } - - let mut verifier_pubkeys = std::collections::HashSet::new(); - for gc_data in ¶meters.gc_data { - if !verifier_pubkeys.insert(XOnlyPublicKey::from(gc_data.verifier_pubkey)) { - bail!(SpecialError::InvalidGraph(format!( - "duplicate verifier pubkey in GC data: {}", - gc_data.verifier_pubkey - ))); - } - if gc_data.final_msg_hashlocks.len() != BABE_M_CC { - bail!(SpecialError::InvalidGraph(format!( - "verifier {} has {} final message hashlocks, expected {}", - gc_data.verifier_pubkey, - gc_data.final_msg_hashlocks.len(), - BABE_M_CC - ))); - } - } - - let watchtower_num = parameters.watchtower_pubkeys.len(); - if watchtower_num < min_required_watchtower() { - bail!(SpecialError::InvalidGraph(format!( - "insufficient watchtower slots: have {}, required {}", - watchtower_num, - min_required_watchtower() - ))); - } - let mut watchtower_pubkeys = std::collections::HashSet::new(); - for pubkey in ¶meters.watchtower_pubkeys { - if !watchtower_pubkeys.insert(*pubkey) { - bail!(SpecialError::InvalidGraph(format!( - "duplicate watchtower pubkey in graph: {pubkey}" - ))); - } - } - let mut watchtower_hashlocks = std::collections::HashSet::new(); - for hashlock in ¶meters.watchtower_ack_hashlocks { - if !watchtower_hashlocks.insert(*hashlock) { - bail!(SpecialError::InvalidGraph( - "duplicate watchtower ack hashlock in graph".to_string() - )); - } - } - - Ok(()) -} - -async fn validate_prekickoff_continuity( - local_db: &LocalDB, - full_graph: &BitvmGcGraph, -) -> Result<()> { - if full_graph.parameters.graph_nonce == 0 { - return Ok(()); - } - - let mut storage_processor = local_db.acquire().await?; - let previous_nonce = full_graph.parameters.graph_nonce - 1; - let graphs = storage_processor - .get_operator_graphs( - GraphQuery::default() - .with_operator_pubkey(full_graph.parameters.operator_pubkey.to_string()) - .with_kickoff_index(previous_nonce as i64) - .with_limit(2), - ) - .await?; - if let Some(previous_graph) = graphs.first() { - let simplified_graph = load_validated_graph_definition( - &mut storage_processor, - previous_graph.instance_id, - previous_graph.graph_id, - ) - .await? - .ok_or_else(|| { - SpecialError::InvalidGraph(format!( - "previous graph raw data is missing for graph nonce {previous_nonce}" - )) - })?; - let expected_cur_prekickoff = - BitvmGcGraph::from_simplified(&simplified_graph)?.next_prekickoff; - let expected_txid = expected_cur_prekickoff.finalize().compute_txid(); - let actual_txid = full_graph.cur_prekickoff.finalize().compute_txid(); - if actual_txid != expected_txid { - bail!(SpecialError::InvalidGraph(format!( - "cur prekickoff continuity mismatch: graph={actual_txid}, expected={expected_txid}" - ))); - } - Ok(()) - } else { - bail!(SpecialError::InvalidGraph(format!( - "previous graph is missing for graph nonce {previous_nonce}" - ))); - } -} - -pub async fn validate_init_graph_base( - local_db: &LocalDB, - btc_client: &BTCClient, - goat_client: &GOATClient, - graph: &SimplifiedBitvmGcGraph, -) -> Result<()> { - if !graph.operator_pre_signed() { - bail!(SpecialError::InvalidGraph("graph is missing operator pre-signatures".to_string())); - } - // Basic structural and on-chain consistency checks for an incoming graph proposal. - // Return SpecialError::InvalidGraph on any validation failure. - // 1) Rebuild full graph (ensures signatures present if flags are set and tx graph is coherent) - let full_graph = BitvmGcGraph::from_simplified(graph) - .map_err(|e| SpecialError::InvalidGraph(format!("invalid graph structure: {e}")))?; - validate_graph_policy(&graph.parameters)?; - validate_prekickoff_continuity(local_db, &full_graph).await?; - verify_graph_operator_pre_signatures(&full_graph) - .map_err(|e| SpecialError::InvalidGraph(format!("invalid operator pre-signatures: {e}")))?; - - // 2) Bind all instance and peg-in parameters to GoatChain and Bitcoin. - validate_graph_instance_parameters( - btc_client, - goat_client, - &graph.parameters.instance_parameters, - ) - .await?; - - // 3) Challenge amount and assert-commit count must match local constants - if graph.parameters.challenge_amount != challenge_amount() { - bail!(SpecialError::InvalidGraph("unexpected challenge amount".to_string())); - } - - // 4) Watchtower config sanity: number of watchtowers should match number of hashlocks and registry size - let watchtowers_on_chain = goat_client.committee_mana_get_watchtowers().await.map_err(|e| { - SpecialError::InvalidGraph(format!("failed to load watchtowers from chain: {e}")) - })?; - - validate_watchtower_selection( - &graph.parameters.watchtower_pubkeys, - &watchtowers_on_chain, - *graph.parameters.graph_id.as_bytes(), - get_genesis_sequencer_commit_id(), - full_graph.watchtower_challenge_init.tx().compute_txid().to_byte_array(), - graph.parameters.pubin_disprove_constant, - )?; - - // 5) Operator stake sanity: verify operator is registered and has enough locked stake. - validate_operator_stake(goat_client, &graph.parameters.operator_pubkey) - .await - .map_err(|error| SpecialError::InvalidGraph(error.to_string()))?; - - Ok(()) -} - -/// Operator stake verdict; RPC failures are returned as errors. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum OperatorStakeStatus { - Staked, - NotRegistered, - Insufficient { locked: u64, min: u64 }, -} - -pub async fn operator_stake_status( - goat_client: &GOATClient, - operator_pubkey: &PublicKey, -) -> Result { - let operator_xonly_pubkey = XOnlyPublicKey::from(*operator_pubkey).serialize(); - let operator_addr = goat_client - .stake_mana_pubkey_to_address(&operator_xonly_pubkey) - .await - .context("query operator address")?; - if operator_addr == [0; 20] { - return Ok(OperatorStakeStatus::NotRegistered); - } - let min_stake_amount = - goat_client.gateway_get_min_stake_amount().await.context("query minimum operator stake")?; - let locked_stake = goat_client - .stake_mana_lock_stake_of(&operator_addr) - .await - .context("query operator locked stake")?; - if locked_stake < min_stake_amount { - return Ok(OperatorStakeStatus::Insufficient { - locked: locked_stake, - min: min_stake_amount, - }); - } - Ok(OperatorStakeStatus::Staked) -} - -pub async fn validate_operator_stake( - goat_client: &GOATClient, - operator_pubkey: &PublicKey, -) -> Result<()> { - match operator_stake_status(goat_client, operator_pubkey).await? { - OperatorStakeStatus::Staked => Ok(()), - OperatorStakeStatus::NotRegistered => bail!("operator not registered"), - OperatorStakeStatus::Insufficient { locked, min } => { - bail!("insufficient operator stake: locked={locked}, min={min}") - } - } -} - -pub async fn validate_verifier_graph_params_endorsements( - goat_client: &GOATClient, - graph: &SimplifiedBitvmGcGraph, - verifier_endorsements: &[(PublicKey, usize, String, SchnorrSignature)], -) -> Result<()> { - let expected_verifier_num = graph.parameters.gc_data.len(); - if verifier_endorsements.len() != expected_verifier_num { - bail!(SpecialError::InvalidGraph(format!( - "verifier params endorsement count {} does not match GC slot count {}", - verifier_endorsements.len(), - expected_verifier_num - ))); - } - - let mut seen_pubkeys = std::collections::HashSet::new(); - let mut seen_indices = std::collections::HashSet::new(); - let mut seen_peers = std::collections::HashSet::new(); - for (verifier_pubkey, verifier_index, verifier_peer_id, signature) in verifier_endorsements { - let peer = - PeerId::from_str(verifier_peer_id).context("invalid verifier endorsement peer")?; - if !seen_peers.insert(peer) { - bail!(SpecialError::InvalidGraph("duplicate verifier endorsement peer".into())); - } - if *verifier_index >= expected_verifier_num { - bail!(SpecialError::InvalidGraph(format!( - "verifier params endorsement index {verifier_index} out of range" - ))); - } - let slot_pubkey = graph.parameters.gc_data[*verifier_index].verifier_pubkey; - if slot_pubkey != *verifier_pubkey { - bail!(SpecialError::InvalidGraph(format!( - "verifier params endorsement slot mismatch at index {verifier_index}: signature pubkey={}, graph pubkey={slot_pubkey}", - verifier_pubkey - ))); - } - if !seen_pubkeys.insert(XOnlyPublicKey::from(*verifier_pubkey)) { - bail!(SpecialError::InvalidGraph(format!( - "duplicate verifier params endorsement pubkey: {verifier_pubkey}" - ))); - } - if !seen_indices.insert(*verifier_index) { - bail!(SpecialError::InvalidGraph(format!( - "duplicate verifier params endorsement index: {verifier_index}" - ))); - } - let ok = verify_verifier_graph_params_endorsement(verifier_pubkey, graph, signature)?; - if !ok { - bail!(SpecialError::InvalidGraph(format!( - "invalid verifier params endorsement signature for {verifier_pubkey}" - ))); - } - } - - for (_, _, peer, _) in verifier_endorsements { - let peer = PeerId::from_str(peer)?; - if !goat_client - .committee_mana_is_verifier(&peer.to_bytes()) - .await - .context("recheck verifier registration before committee setup")? - { - bail!(SpecialError::InvalidGraph( - "endorsement peer is not a registered verifier".into() - )); - } - } - Ok(()) -} - -pub async fn validate_init_graph( - local_db: &LocalDB, - btc_client: &BTCClient, - goat_client: &GOATClient, - graph: &SimplifiedBitvmGcGraph, - verifier_endorsements: &[(PublicKey, usize, String, SchnorrSignature)], -) -> Result<()> { - validate_init_graph_base(local_db, btc_client, goat_client, graph).await?; - validate_verifier_graph_params_endorsements(goat_client, graph, verifier_endorsements).await -} -pub async fn validate_finalized_graph( - btc_client: &BTCClient, - goat_client: &GOATClient, - graph: &SimplifiedBitvmGcGraph, - endorse_sigs: &[(PublicKey, EvmAddress, Vec)], - params_endorse_sigs: &[(PublicKey, EvmAddress, Vec)], -) -> Result<()> { - if !graph.operator_pre_signed() || !graph.committee_pre_signed() { - bail!(SpecialError::InvalidGraph( - "finalized graph is missing operator or committee pre-signatures".to_string() - )); - } - // 1) Rebuild full graph to ensure structure is coherent and txns derivable - let full_graph = BitvmGcGraph::from_simplified(graph) - .map_err(|e| SpecialError::InvalidGraph(format!("invalid graph structure: {e}")))?; - validate_graph_policy(&graph.parameters)?; - verify_graph_operator_pre_signatures(&full_graph) - .map_err(|e| SpecialError::InvalidGraph(format!("invalid operator pre-signatures: {e}")))?; - verify_graph_committee_pre_signatures(&full_graph).map_err(|e| { - SpecialError::InvalidGraph(format!("invalid committee pre-signatures: {e}")) - })?; - - // 2) Repeat the full instance and peg-in binding for nodes that did not see CreateGraph. - validate_graph_instance_parameters( - btc_client, - goat_client, - &graph.parameters.instance_parameters, - ) - .await?; - - let instance_id = graph.parameters.instance_parameters.instance_id; - if graph.parameters.challenge_amount != challenge_amount() { - bail!(SpecialError::InvalidGraph("unexpected challenge amount".to_string())); - } - - let watchtowers_on_chain = goat_client.committee_mana_get_watchtowers().await.map_err(|e| { - SpecialError::InvalidGraph(format!("failed to load watchtowers from chain: {e}")) - })?; - validate_watchtower_selection( - &graph.parameters.watchtower_pubkeys, - &watchtowers_on_chain, - *graph.parameters.graph_id.as_bytes(), - get_genesis_sequencer_commit_id(), - full_graph.watchtower_challenge_init.tx().compute_txid().to_byte_array(), - graph.parameters.pubin_disprove_constant, - )?; - - // 3) Validate endorsements: unique, from legitimate committee members, and signatures recover to the provided EVM address - use std::collections::HashSet; - let mut seen_committee: HashSet = HashSet::new(); - let mut seen_evm: HashSet = HashSet::new(); - let pegin_data = goat_client - .gateway_get_pegin_data(&instance_id) - .await - .map_err(|e| SpecialError::InvalidGraph(format!("failed to load instance data: {e}")))?; - if pegin_data.committee_pubkeys.len() != pegin_data.committee_addresses.len() { - bail!(SpecialError::InvalidGraph( - "on-chain committee pubkey and address counts differ".to_string() - )); - } - if endorse_sigs.len() != pegin_data.committee_pubkeys.len() { - bail!(SpecialError::InvalidGraph(format!( - "endorsement count {} does not match instance committee count {}", - endorse_sigs.len(), - pegin_data.committee_pubkeys.len() - ))); - } - if params_endorse_sigs.len() != pegin_data.committee_pubkeys.len() { - bail!(SpecialError::InvalidGraph(format!( - "params endorsement count {} does not match instance committee count {}", - params_endorse_sigs.len(), - pegin_data.committee_pubkeys.len() - ))); - } - - for (pk, evm_addr, sig) in endorse_sigs.iter() { - // no duplicates - if !seen_committee.insert(*pk) { - bail!(SpecialError::InvalidGraph( - "duplicate committee pubkey in endorsements".to_string() - )); - } - if !seen_evm.insert(*evm_addr) { - bail!(SpecialError::InvalidGraph("duplicate evm address in endorsements".to_string())); - } - - // map pubkey -> expected evm address from GoatChain - let mut found = false; - for i in 0..pegin_data.committee_pubkeys.len() { - let on_chain_pk = - PublicKey::from_slice(&pegin_data.committee_pubkeys[i]).map_err(|e| { - SpecialError::InvalidGraph(format!("invalid committee pubkey on-chain: {e}")) - })?; - if &on_chain_pk == pk { - found = true; - let expected_addr = pegin_data.committee_addresses[i]; - if &expected_addr != evm_addr { - bail!(SpecialError::InvalidGraph("committee evm address mismatch".to_string())); - } - break; - } - } - if !found { - bail!(SpecialError::InvalidGraph("endorser not in committee set".to_string())); - } - - // cryptographically verify the endorsement against the graph digest - let ok = verify_graph_endorsement(goat_client, evm_addr, &full_graph, sig).await.map_err( - |e| SpecialError::InvalidGraph(format!("failed to verify endorsement: {e}")), - )?; - if !ok { - bail!(SpecialError::InvalidGraph("invalid endorsement signature".to_string())); - } - } - - let mut seen_committee: HashSet = HashSet::new(); - let mut seen_evm: HashSet = HashSet::new(); - for (pk, evm_addr, sig) in params_endorse_sigs.iter() { - if !seen_committee.insert(*pk) { - bail!(SpecialError::InvalidGraph( - "duplicate committee pubkey in params endorsements".to_string() - )); - } - if !seen_evm.insert(*evm_addr) { - bail!(SpecialError::InvalidGraph( - "duplicate evm address in params endorsements".to_string() - )); - } - - let mut found = false; - for i in 0..pegin_data.committee_pubkeys.len() { - let on_chain_pk = - PublicKey::from_slice(&pegin_data.committee_pubkeys[i]).map_err(|e| { - SpecialError::InvalidGraph(format!("invalid committee pubkey on-chain: {e}")) - })?; - if &on_chain_pk == pk { - found = true; - let expected_addr = pegin_data.committee_addresses[i]; - if &expected_addr != evm_addr { - bail!(SpecialError::InvalidGraph( - "committee evm address mismatch in params endorsements".to_string() - )); - } - break; - } - } - if !found { - bail!(SpecialError::InvalidGraph("params endorser not in committee set".to_string())); - } - - let ok = verify_graph_params_endorsement(evm_addr, &full_graph, sig).map_err(|e| { - SpecialError::InvalidGraph(format!("failed to verify params endorsement: {e}")) - })?; - if !ok { - bail!(SpecialError::InvalidGraph("invalid params endorsement signature".to_string())); - } - } - - Ok(()) -} -pub fn prekickoff_replenishment_amount() -> Amount { - Amount::from_sat(500000) -} -pub fn min_prekickoff_input_amount() -> Amount { - Amount::from_sat(200000) -} -pub fn challenge_amount() -> Amount { - Amount::from_sat(20000) -} -pub fn prekickoff_fee_amount(replenish_fee_inputs_num: usize) -> Amount { - let tx_vbytes = PRE_KICKOFF_BASE_VBYTES - + (replenish_fee_inputs_num as u64 * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE); - Amount::from_sat(tx_vbytes) -} -pub mod evm_swap_utils { - use super::*; - use crate::utils::evm_swap_utils::IEscrowManager::{EscrowData, IEscrowManagerCalls}; - use alloy::primitives::{B256, keccak256}; - use alloy::rpc::types::trace::geth::{CallConfig, CallFrame, GethDebugTracingOptions}; - use alloy::sol; - use alloy::sol_types::SolInterface; - use alloy::sol_types::SolValue; - - sol! { - interface IEscrowManager { - event Initialize(address indexed offerer, address indexed claimer, bytes32 indexed escrowHash, address claimHandler, address refundHandler); - event Claim(address indexed offerer, address indexed claimer, bytes32 indexed escrowHash, address claimHandler, bytes witnessResult); // for BitcoinNoncedOutputClaimHandler, Claim.witnessResult = payout_btc_txid - event Refund(address indexed offerer, address indexed claimer, bytes32 indexed escrowHash, address refundHandler, bytes witnessResult); - event ExecutionError(bytes32 indexed escrowHash, bytes error); - #[derive(Debug)] - struct EscrowData { - //Account funding the escrow - address offerer; - //Account entitled to claim the funds from the escrow - address claimer; - - //Amount of tokens in the escrow - uint256 amount; - //Token of the escrow - address token; - - //Misc escrow data flags, currently defined: payIn, payOut, reputation. - //It is recommended to randomize the other unused bits in the flags to act as a salt, - // such that no 2 escrow data are the same, even if all the other data in them match. - uint256 flags; - - //Address of the IClaimHandler deciding if this escrow is claimable - // use BitcoinNoncedOutputClaimHandler for Goat -> Bitcoin swaps - address claimHandler; - //Data provided to the claim handler along with the witness to check claimability - // for BitcoinNoncedOutputClaimHandler, this is the hash commitment of the claim data, see hash_claim_commitment - bytes32 claimData; - - //Address of the IRefundHandler deciding if this escrow is refundable - // use TimelockRefundHandler for Goat -> Bitcoin swaps - address refundHandler; - //Data provided to the refund handler along with the witness to check for refundability - // for TimelockRefundHandler, this is the timestamp after which refund is possible - bytes32 refundData; - - //Security deposit taken by the offerer if swap expires without claimer claiming (i.e. options premium) - uint256 securityDeposit; - //Claimer bounty that can be claimed by a 3rd party claimer if he were to claim this swap on behalf of claimer - uint256 claimerBounty; - //Deposit token of the swap used for securityDeposit and claimerBounty - address depositToken; - - //ExecutionAction hash commitment to be executed on claim, left 0x0 if no execution should happen on claim - bytes32 successActionCommitment; - } - function initialize(EscrowData calldata escrow, bytes calldata signature, uint256 timeout, bytes memory _extraData) external payable {} - function claim(EscrowData calldata escrow, bytes calldata witness) external {} - } - } - - #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] - pub struct ClaimData { - pub txid: Txid, - pub nonce: u64, - pub output_amount: u64, - pub output_script: Vec, - pub confirmations: u32, - pub btc_relay_contract: EvmAddress, - pub witness: String, - } - - pub fn hash_escrow_data(escrow: &EscrowData) -> B256 { - let encoded = escrow.abi_encode(); - keccak256(&encoded) - } - - pub fn hash_claim_commitment(claim_data: &ClaimData) -> B256 { - // txoHash = keccak256(uint64 nonce || uint64 outputAmount || keccak256(bytes outputScript)) - // Commitment: C = abi.encodePacked(bytes32 txoHash, uint32 confirmations, address btcRelayContract) - // Witness: W = C || StoredBlockHeader blockheader || uint32 vout || bytes transaction || uint32 position || bytes32[] merkleProof - sol! { - interface IClaimHandlerHelper { - struct Txo { - uint64 nonce; - uint64 outputAmount; - bytes32 outputScriptHash; - } - struct ClaimCommitment { - bytes32 TxoHash; - uint32 confirmations; - address btcRelayContract; - } - } - } - let output_script_hash = keccak256(&claim_data.output_script); - let txo = IClaimHandlerHelper::Txo { - nonce: claim_data.nonce, - outputAmount: claim_data.output_amount, - outputScriptHash: output_script_hash, - }; - let txo_hash = keccak256(txo.abi_encode_packed()); - let claim_commitment = IClaimHandlerHelper::ClaimCommitment { - TxoHash: txo_hash, - confirmations: claim_data.confirmations, - btcRelayContract: claim_data.btc_relay_contract, - }; - keccak256(claim_commitment.abi_encode_packed()) - } - - fn find_escrow_data( - call: &CallFrame, - swap_contract_address: &EvmAddress, - escrow_hash: &[u8; 32], - ) -> anyhow::Result> { - if call.to == Some(*swap_contract_address) - && let Ok(calldata) = IEscrowManagerCalls::abi_decode(&call.input) - && let IEscrowManagerCalls::initialize(args) = calldata - { - let computed_hash = hash_escrow_data(&args.escrow); - if &computed_hash.0 == escrow_hash { - return Ok(Some(args.escrow)); - } - } - - for sub_call in &call.calls { - if let Some(escrow_data) = - find_escrow_data(sub_call, swap_contract_address, escrow_hash)? - { - return Ok(Some(escrow_data)); - } - } - Ok(None) - } - - pub async fn extract_escrow_data_from_tx( - goat_client: &GOATClient, - tx_hash: &str, - swap_contract_address: &EvmAddress, - escrow_hash: &[u8; 32], - ) -> anyhow::Result> { - let trace_opts = GethDebugTracingOptions::call_tracer(CallConfig::default()); - let trace_raw = goat_client.debug_trace_tx(tx_hash, Some(trace_opts)).await?; - let call_trace = trace_raw.try_into_call_frame()?; - if let Some(escrow_data) = - find_escrow_data(&call_trace, swap_contract_address, escrow_hash)? - { - Ok(Some(escrow_data)) - } else { - Ok(None) - } - } - - fn claim_data_from_witness(witness: &[u8]) -> anyhow::Result { - // txoHash = keccak256(uint64 nonce || uint64 outputAmount || keccak256(bytes outputScript)) - // Witness: W = bytes32 txoHash - // || uint32 confirmations - // || address btcRelayContract - // || StoredBlockHeader(160-bytes) blockheader - // || uint32 vout - // || bytes transaction (32-byte length prefix + data) - // || uint32 position || bytes32[] merkleProof - // claimData.nonce = or(shl(24, locktimeSub500M), and(firstNSequence, 0x00FFFFFF)) - // claimData.output_amount = W.transaction.outputs[vout].value - // claimData.output_script = W.transaction.outputs[vout].scriptPubKey - - let mut offset = 0; - - // 1. txoHash (32 bytes) - if witness.len() < 32 { - anyhow::bail!("witness too short for txoHash"); - } - offset += 32; - - // 2. confirmations (4 bytes) - if witness.len() < offset + 4 { - anyhow::bail!("witness too short for confirmations"); - } - let confirmations = u32::from_be_bytes(witness[offset..offset + 4].try_into()?); - offset += 4; - - // 3. btcRelayContract (20 bytes) - if witness.len() < offset + 20 { - anyhow::bail!("witness too short for btcRelayContract"); - } - let btc_relay_contract = EvmAddress::from_slice(&witness[offset..offset + 20]); - offset += 20; - - // 4. StoredBlockHeader (160 bytes) - if witness.len() < offset + 160 { - anyhow::bail!("witness too short for blockheader"); - } - offset += 160; - - // 5. vout (4 bytes) - if witness.len() < offset + 4 { - anyhow::bail!("witness too short for vout"); - } - let vout = u32::from_be_bytes(witness[offset..offset + 4].try_into()?); - offset += 4; - - // 6. transaction (32-byte length prefix + data) - if witness.len() < offset + 32 { - anyhow::bail!("witness too short for transaction length"); - } - let tx_len = alloy::primitives::U256::from_be_slice(&witness[offset..offset + 32]); - offset += 32; - - let tx_len_usize: usize = - tx_len.try_into().map_err(|_| anyhow::anyhow!("tx length too large"))?; - - if witness.len() < offset + tx_len_usize { - anyhow::bail!("witness too short for transaction data"); - } - let tx_bytes = &witness[offset..offset + tx_len_usize]; - - let tx: Transaction = deserialize(tx_bytes)?; - - if vout as usize >= tx.output.len() { - anyhow::bail!("vout index out of bounds"); - } - let output = &tx.output[vout as usize]; - - if tx.input.is_empty() { - anyhow::bail!("transaction has no inputs"); - } - let first_input_sequence = tx.input[0].sequence.to_consensus_u32(); - - let lock_time = tx.lock_time.to_consensus_u32(); - let lock_time_val = - if lock_time >= 500_000_000 { lock_time - 500_000_000 } else { lock_time }; - - let nonce = ((lock_time_val as u64) << 24) | ((first_input_sequence as u64) & 0x00ffffff); - - Ok(ClaimData { - txid: tx.compute_txid(), - nonce, - output_amount: output.value.to_sat(), - output_script: output.script_pubkey.to_bytes(), - confirmations, - btc_relay_contract, - witness: hex::encode(witness), - }) - } - - // for BitcoinNoncedOutputClaimHandler - fn find_claim_data( - tx_hash: &str, - call: &CallFrame, - swap_contract_address: &EvmAddress, - escrow_hash: &[u8; 32], - ) -> anyhow::Result> { - if call.to == Some(*swap_contract_address) - && let Ok(calldata) = IEscrowManagerCalls::abi_decode(&call.input) - && let IEscrowManagerCalls::claim(args) = calldata - { - let computed_hash = hash_escrow_data(&args.escrow); - if &computed_hash.0 == escrow_hash { - return match claim_data_from_witness(&args.witness) { - Ok(claim_data) => Ok(Some(claim_data)), - Err(e) => { - warn!("fail to decode claim data for tx: {tx_hash}, error:{e}"); - Ok(None) - } - }; - } - } - - for sub_call in &call.calls { - if let Some(claim_data) = - find_claim_data(tx_hash, sub_call, swap_contract_address, escrow_hash)? - { - return Ok(Some(claim_data)); - } - } - Ok(None) - } - - pub async fn extract_claim_data_from_tx( - goat_client: &GOATClient, - tx_hash: &str, - swap_contract_address: &EvmAddress, - escrow_hash: &[u8; 32], - ) -> anyhow::Result> { - let trace_opts = GethDebugTracingOptions::call_tracer(CallConfig::default()); - let trace_raw = goat_client.debug_trace_tx(tx_hash, Some(trace_opts)).await?; - let call_trace = trace_raw.try_into_call_frame()?; - if let Some(claim_data) = - find_claim_data(tx_hash, &call_trace, swap_contract_address, escrow_hash)? - { - Ok(Some(claim_data)) - } else { - Ok(None) - } - } - - #[tokio::test] - async fn test_find_escrow_data() { - unsafe { - std::env::set_var(crate::env::ENV_GOAT_CHAIN_URL, "https://rpc.testnet3.goat.network"); - } - let tx_hash = "0x6027024dc57b847120074efed67c3e31534988f70b6b1e5043b248e7740a295c"; - let goat_client = GOATClient::new( - crate::env::goat_config_from_env().await, - client::goat_chain::GoatNetwork::Test, - ); - let swap_contract_address = - EvmAddress::from_str("0xe510D5781C6C849284Fb25Dc20b1684cEC445C8B").unwrap(); - let escrow_hash: [u8; 32] = - hex::decode("521a1d007f9fdf41b18ad6f1ccfeaf8fd67d0b04608ce3d8950526e55e4eca28") - .unwrap() - .try_into() - .unwrap(); - let escrow_data = extract_escrow_data_from_tx( - &goat_client, - tx_hash, - &swap_contract_address, - &escrow_hash, - ) - .await - .unwrap(); - assert!(escrow_data.is_some()); - let expected_claim_hash = - B256::from_str("0xc69e4a62e0c904b341245656ba191790356d771bd4a7a00bed2780a5abad8c63") - .unwrap(); - assert_eq!(escrow_data.unwrap().claimData, expected_claim_hash); - } - - #[test] - fn test_hash_claim_commitment() { - // example data from an actual swap on testnet: - // goat initialize txid: 0x6027024dc57b847120074efed67c3e31534988f70b6b1e5043b248e7740a295c - // goat claim txid: 0xc2b26508a28f349c7ee1e189914dc5815b77d1abaa5ce6a60449f69bd1e7e64a - // btc payout txid: 033d4024aca7f6dda6b01e7f0a2bb0fdd15160cc9b2559b55c6f65962362d74e - let claim_data = ClaimData { - txid: Txid::from_slice(&[0_u8; 32]).unwrap(), - nonce: 17872110975047329u64, - output_amount: 9511u64, - output_script: hex::decode( - "5120a5d06cb76aaf6287b93a8ee73d9678e32b039354e6df4019bbd60087e347f5cc", - ) - .unwrap(), - confirmations: 2u32, - btc_relay_contract: EvmAddress::from_str("0x3887B02217726bB36958Dd595e57293fB63D5082") - .unwrap(), - witness: "".to_string(), - }; - let commitment_hash = hash_claim_commitment(&claim_data); - - let expected_hash = - B256::from_str("0xc69e4a62e0c904b341245656ba191790356d771bd4a7a00bed2780a5abad8c63") - .unwrap(); - assert_eq!(commitment_hash, expected_hash); - } - - #[tokio::test] - async fn test_find_claim_data() { - unsafe { - std::env::set_var(crate::env::ENV_GOAT_CHAIN_URL, "https://rpc.testnet3.goat.network"); - } - let tx_hash = "0xc2b26508a28f349c7ee1e189914dc5815b77d1abaa5ce6a60449f69bd1e7e64a"; - let goat_client = GOATClient::new( - crate::env::goat_config_from_env().await, - client::goat_chain::GoatNetwork::Test, - ); - let swap_contract_address = - EvmAddress::from_str("0xe510D5781C6C849284Fb25Dc20b1684cEC445C8B").unwrap(); - let escrow_hash: [u8; 32] = - hex::decode("521a1d007f9fdf41b18ad6f1ccfeaf8fd67d0b04608ce3d8950526e55e4eca28") - .unwrap() - .try_into() - .unwrap(); - let expected_claim_data = ClaimData { - txid: Txid::from_str("033d4024aca7f6dda6b01e7f0a2bb0fdd15160cc9b2559b55c6f65962362d74e").unwrap(), - nonce: 17872110975047329u64, - output_amount: 9511u64, - output_script: hex::decode( - "5120a5d06cb76aaf6287b93a8ee73d9678e32b039354e6df4019bbd60087e347f5cc", - ) - .unwrap(), - confirmations: 2u32, - btc_relay_contract: EvmAddress::from_str("0x3887B02217726bB36958Dd595e57293fB63D5082") - .unwrap(), - witness: "8977831297fa2d7156898a8d26bb9e276a83cf907d739eb64ff98a0092e9250e000000023887b022\ - 17726bb36958dd595e57293fb63d508200600020bfe2760399ccb567289b120361316911b13e937aa0f2742bb7\ - 0b000000000000efaf19dc26fadb8b1cd69e76c6d1f51123f6d22407755f865b65b446cf863c3c5fbe3769f0ff\ - 0f1ad88a60f10000000000000000000000000000000000000000000017b6e253602b5f4cc25000494e876937b2\ - 636937bd246937bd2a6937bd2d6937bd306937bd6f6937bd8f6937bda66937bde76937bdf96937be07000000000\ - 00000000000000000000000000000000000000000000000000000000000008902000000018c3db50ef29dbdddf9\ - 628cd968667688cd37560ef5e64131efc3a1e4cf4e739c0100000000a1d60dfe022725000000000000225120a5d\ - 06cb76aaf6287b93a8ee73d9678e32b039354e6df4019bbd60087e347f5ccf61a010000000000225120eb3e0c2d\ - d6b344c6efaa771306a29e864c65d536dfac8f89a87680285af7acad1afc4b5d000000060000000000000000000\ - 000000000000000000000000000000000000000000003f7cfe6eed4929eea954c8e1358704c6dec19e2826f7223\ - ff7d0eff92c1addf20fd2b876e05846a100c38bd3c30619c66437abb97bc5fc21e2595fbb8a514f5341ebf1f8dd\ - 26752879de5c581667aaf183b18c1b5a111f4a24061d2c44aea2fd1".to_string(), - }; - let claim_data = - extract_claim_data_from_tx(&goat_client, tx_hash, &swap_contract_address, &escrow_hash) - .await - .unwrap(); - assert!(claim_data.is_some()); - assert_eq!(claim_data.unwrap(), expected_claim_data); - } -} - -#[allow(clippy::too_many_arguments)] -#[allow(clippy::redundant_pattern_matching)] -#[allow(clippy::collapsible_else_if)] -pub(crate) async fn refresh_graph( - local_db: &LocalDB, - btc_client: &BTCClient, - goat_client: &GOATClient, - instance_id: Uuid, - graph_id: Uuid, - graph: &BitvmGcGraph, -) -> Result { - if graph.parameters.instance_parameters.instance_id != instance_id - || graph.parameters.graph_id != graph_id - { - bail!( - "refuse to refresh graph {instance_id}:{graph_id} with mismatched graph parameters {}:{}", - graph.parameters.instance_parameters.instance_id, - graph.parameters.graph_id - ); - } - - let scan = scan_graph_chain_state(btc_client, goat_client, graph).await?; - let outcome = update_graph_status( - local_db, - instance_id, - graph_id, - scan.status, - Some(scan.sub_status.clone()), - ) - .await?; - - match outcome { - GraphStatusTransitionOutcome::Applied => { - if let Some(challenge_txid) = scan.challenge_txid { - update_graph_challenge_txid_if_needed(local_db, graph_id, challenge_txid).await?; - } - Ok(GraphRefreshResult { - status: scan.status, - sub_status: Some(scan.sub_status.clone()), - scan: Some(scan), - status_transition_accepted: true, - }) - } - GraphStatusTransitionOutcome::AlreadyCurrent => { - if let Some(challenge_txid) = scan.challenge_txid { - update_graph_challenge_txid_if_needed(local_db, graph_id, challenge_txid).await?; - } - Ok(GraphRefreshResult { - status: scan.status, - sub_status: Some(scan.sub_status.clone()), - // Preserve the verified scan even for an idempotent status - // replay. The previous attempt may have committed the status - // before it could enqueue the corresponding local message. - scan: Some(scan), - status_transition_accepted: true, - }) - } - GraphStatusTransitionOutcome::Rejected { current } => { - warn!( - "ignore stale chain scan for graph {instance_id}:{graph_id}: candidate={}, current={current}", - scan.status - ); - Ok(GraphRefreshResult { - status: current, - sub_status: None, - scan: None, - status_transition_accepted: false, - }) - } - GraphStatusTransitionOutcome::NotFound => { - warn!("graph {instance_id}:{graph_id} disappeared while applying chain scan"); - Ok(GraphRefreshResult { - status: scan.status, - sub_status: None, - scan: None, - status_transition_accepted: false, - }) - } - } -} - -#[derive(Clone, Debug)] -struct DetectedDisprove { - disprove_type: DisproveTxType, - index: usize, - challenge_start_txid: Option, - challenge_finish_txid: Txid, -} - -#[derive(Clone, Debug)] -pub(crate) struct GraphChainScan { - status: GraphStatus, - sub_status: ChallengeSubStatus, - challenge_txid: Option, - #[allow(dead_code)] - watchtower_challenge_init_on_chain: bool, - #[allow(dead_code)] - operator_assert_on_chain: bool, - disprove: Option, -} - -pub(crate) struct GraphRefreshResult { - pub(crate) status: GraphStatus, - pub(crate) sub_status: Option, - pub(crate) scan: Option, - pub(crate) status_transition_accepted: bool, -} - -fn initial_challenge_sub_status(watchtower_num: usize, verifier_num: usize) -> ChallengeSubStatus { - let mut sub_status = ChallengeSubStatus::default(); - sub_status.watchtower_challenge_status.resize(watchtower_num, false); - sub_status.verifier_challenge_status.resize(verifier_num, VerifierChallengeStatus::None); - sub_status -} - -fn connector_a_outpoint(graph: &BitvmGcGraph) -> Result { - graph - .kickoff - .connector_a_input() - .map(|input| input.outpoint) - .map_err(|e| anyhow!("failed to get connector-a input: {e}")) -} - -fn connector_d_outpoint(graph: &BitvmGcGraph) -> Result { - graph - .operator_assert - .connector_d_input() - .map(|input| input.outpoint) - .map_err(|e| anyhow!("failed to get connector-d input: {e}")) -} - -#[allow(dead_code)] -fn guardian_connector_outpoint(graph: &BitvmGcGraph) -> Result { - graph - .kickoff - .guardian_connector_input() - .map(|input| input.outpoint) - .map_err(|e| anyhow!("failed to get guardian connector input: {e}")) -} - -async fn update_graph_challenge_txid_if_needed( - local_db: &LocalDB, - graph_id: Uuid, - challenge_txid: Txid, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let Some(graph) = storage_processor.find_graph(&graph_id).await? else { - warn!("graph: {graph_id} not found, skip updating challenge txid"); - return Ok(()); - }; - if graph.challenge_txid.as_ref().map(|txid| txid.0) != Some(challenge_txid) { - storage_processor - .update_graph_runtime( - &GraphRuntimeUpdate::new(graph.instance_id, graph_id) - .with_challenge_txid(challenge_txid.into()), - ) - .await?; - } - Ok(()) -} - -#[allow(dead_code)] -async fn detect_guardian_disprove( - btc_client: &BTCClient, - graph: &BitvmGcGraph, - challenge_start_txid: Option, -) -> Result> { - let quick_challenge_txid = graph.quick_challenge.tx().compute_txid(); - if tx_on_chain(btc_client, &quick_challenge_txid).await? { - return Ok(Some(DetectedDisprove { - disprove_type: DisproveTxType::QuickChallenge, - index: 0, - challenge_start_txid, - challenge_finish_txid: quick_challenge_txid, - })); - } - - let challenge_incomplete_kickoff_txid = graph.challenge_incomplete_kickoff.tx().compute_txid(); - if tx_on_chain(btc_client, &challenge_incomplete_kickoff_txid).await? { - return Ok(Some(DetectedDisprove { - disprove_type: DisproveTxType::ChallengeIncompleteKickoff, - index: 0, - challenge_start_txid, - challenge_finish_txid: challenge_incomplete_kickoff_txid, - })); - } - Ok(None) -} - -async fn detect_connector_d_disprove( - btc_client: &BTCClient, - graph: &BitvmGcGraph, - challenge_start_txid: Option, -) -> Result> { - let connector_d = connector_d_outpoint(graph)?; - let Some(spent_txid) = - outpoint_spent_txid(btc_client, &connector_d.txid, connector_d.vout as u64).await? - else { - return Ok(None); - }; - - if spent_txid == graph.take2.tx().compute_txid() { - return Ok(None); - } - - for (index, disprove) in graph.disproves.iter().enumerate() { - if spent_txid == disprove.tx().compute_txid() { - return Ok(Some(DetectedDisprove { - disprove_type: DisproveTxType::Disprove, - index, - challenge_start_txid, - challenge_finish_txid: spent_txid, - })); - } - } - - Ok(Some(DetectedDisprove { - disprove_type: DisproveTxType::PubinDisprove, - index: 0, - challenge_start_txid: None, - challenge_finish_txid: spent_txid, - })) -} - -async fn detect_watchtower_flow_disprove( - btc_client: &BTCClient, - graph: &BitvmGcGraph, -) -> Result> { - for (index, nack) in graph.operator_challenge_nacks.iter().enumerate() { - let txid = nack.tx().compute_txid(); - if tx_on_chain(btc_client, &txid).await? { - return Ok(Some(DetectedDisprove { - disprove_type: DisproveTxType::OperatorChallengeNack, - index, - challenge_start_txid: None, - challenge_finish_txid: txid, - })); - } - } - - let txid = graph.operator_commit_timeout.tx().compute_txid(); - if tx_on_chain(btc_client, &txid).await? { - return Ok(Some(DetectedDisprove { - disprove_type: DisproveTxType::OperatorCommitTimeout, - index: 0, - challenge_start_txid: None, - challenge_finish_txid: txid, - })); - } - - Ok(None) -} - -async fn scan_graph_chain_state( - btc_client: &BTCClient, - goat_client: &GOATClient, - graph: &BitvmGcGraph, -) -> Result { - let instance_id = graph.parameters.instance_parameters.instance_id; - let graph_id = graph.parameters.graph_id; - let watchtower_num = graph.parameters.watchtower_pubkeys.len(); - let verifier_num = graph.verifier_asserts.len(); - let mut sub_status = initial_challenge_sub_status(watchtower_num, verifier_num); - // Derive the candidate from the graph and both chains, rather than using - // the locally persisted status as the scan start. The stored status is a - // projection and can be stale after a restart or missed listener event. - let mut current_status = if graph.committee_pre_signed() { - GraphStatus::CommitteePresigned - } else { - GraphStatus::OperatorPresigned - }; - - let prekickoff_txid = graph.cur_prekickoff.tx().compute_txid(); - let kickoff_txid = graph.kickoff.tx().compute_txid(); - let take1_txid = graph.take1.tx().compute_txid(); - let take2_txid = graph.take2.tx().compute_txid(); - - // GraphData is a Goat-chain fact and must be checked regardless of the - // current local projection. A fully synced node may otherwise remain at - // OperatorPresigned forever after a prior status regression. - let graph_data_on_goat = goat_client.gateway_get_graph_data(&graph_id).await?; - if graph_data_on_goat.operator_pubkey != [0u8; 32] { - current_status = GraphStatus::OperatorDataPushed; - } - // A claimed pegin is terminal, so a graph which did not complete that - // withdrawal can never be used. A Locked pegin is intentionally excluded: - // the active withdrawal may still be cancelled and return to Withdrawable. - if current_status == GraphStatus::OperatorDataPushed { - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - let withdraw_data = goat_client.gateway_get_withdraw_data(&graph_id).await?; - if pegin_data.status == PeginStatus::Claimed && withdraw_data.status == WithdrawStatus::None - { - current_status = GraphStatus::Obsoleted; - } - } - // check Prekickoff - if matches!( - current_status, - GraphStatus::OperatorPresigned - | GraphStatus::CommitteePresigned - | GraphStatus::OperatorDataPushed - | GraphStatus::Obsoleted - ) { - if !tx_on_chain(btc_client, &prekickoff_txid).await? { - return Ok(GraphChainScan { - status: current_status, - sub_status, - challenge_txid: None, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: None, - }); - } else { - current_status = if current_status == GraphStatus::OperatorDataPushed { - GraphStatus::PreKickoff - } else { - // for GraphStatus::OperatorPresigned/CommitteePresigned: - // if prekickoff is on-chain while graph data not yet posted, - // it means this graph will never be posted and operator is going to skip it, - // mark it as Obsoleted so that it can be skipped later - // - // for GraphStatus::Obsoleted: if the graph is obsoleted, - // keep it as Obsoleted so that it can be skipped later - GraphStatus::Obsoleted - }; - } - } - // check Kickoff/SkipKickoff - if matches!(current_status, GraphStatus::PreKickoff | GraphStatus::Obsoleted) { - let kickoff_connector_vout = 1; - if let Some(spent_txid) = - outpoint_spent_txid(btc_client, &prekickoff_txid, kickoff_connector_vout).await? - { - if spent_txid != kickoff_txid { - return Ok(GraphChainScan { - status: GraphStatus::Skipped, - sub_status, - challenge_txid: None, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: None, - }); - } else { - current_status = GraphStatus::OperatorKickOff; - } - } else { - return Ok(GraphChainScan { - status: current_status, - sub_status, - challenge_txid: None, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: None, - }); - } - } - - let mut challenge_txid = None; - if current_status == GraphStatus::OperatorKickOff - && let Some(disprove) = detect_guardian_disprove(btc_client, graph, challenge_txid).await? - { - sub_status.disprove_type = Some(disprove.disprove_type); - sub_status.disprove_index = disprove.index as i32; - return Ok(GraphChainScan { - status: GraphStatus::Disprove, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: Some(disprove), - }); - } - if current_status == GraphStatus::OperatorKickOff { - let connector_a = connector_a_outpoint(graph)?; - if let Some(spent_txid) = - outpoint_spent_txid(btc_client, &connector_a.txid, connector_a.vout as u64).await? - { - if spent_txid != take1_txid { - current_status = GraphStatus::Challenge; - challenge_txid = Some(spent_txid); - } else { - return Ok(GraphChainScan { - status: GraphStatus::OperatorTake1, - sub_status, - challenge_txid: None, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: None, - }); - } - } else { - return Ok(GraphChainScan { - status: GraphStatus::OperatorKickOff, - sub_status, - challenge_txid: None, - watchtower_challenge_init_on_chain: false, - operator_assert_on_chain: false, - disprove: None, - }); - } - } - - if current_status == GraphStatus::Challenge && challenge_txid.is_none() { - let connector_a = connector_a_outpoint(graph)?; - if let Some(spent_txid) = - outpoint_spent_txid(btc_client, &connector_a.txid, connector_a.vout as u64).await? - && spent_txid != take1_txid - { - challenge_txid = Some(spent_txid); - } - } - - let mut watchtower_challenge_init_on_chain = false; - let mut operator_assert_on_chain = false; - if current_status == GraphStatus::Challenge { - if let Some(disprove) = detect_guardian_disprove(btc_client, graph, challenge_txid).await? { - sub_status.disprove_type = Some(disprove.disprove_type); - sub_status.disprove_index = disprove.index as i32; - return Ok(GraphChainScan { - status: GraphStatus::Disprove, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: Some(disprove), - }); - } - - if let Some(disprove) = detect_watchtower_flow_disprove(btc_client, graph).await? { - sub_status.disprove_type = Some(disprove.disprove_type); - sub_status.disprove_index = disprove.index as i32; - return Ok(GraphChainScan { - status: GraphStatus::Disprove, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: Some(disprove), - }); - } - - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - watchtower_challenge_init_on_chain = - tx_on_chain(btc_client, &watchtower_challenge_init_txid).await?; - if watchtower_challenge_init_on_chain { - for watchtower_index in 0..watchtower_num { - let watchtower_vout = - output_topology::watchtower_challenge_init::watchtower_connector( - watchtower_index, - ) as u64; - let spent = outpoint_spent_txid( - btc_client, - &watchtower_challenge_init_txid, - watchtower_vout, - ) - .await? - .is_some(); - if let Some(status) = - sub_status.watchtower_challenge_status.get_mut(watchtower_index) - { - *status = spent; - } - } - } - if !watchtower_challenge_init_on_chain { - return Ok(GraphChainScan { - status: current_status, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain: false, - disprove: None, - }); - } - let operator_assert_txid = graph.operator_assert.tx().compute_txid(); - operator_assert_on_chain = tx_on_chain(btc_client, &operator_assert_txid).await?; - if !operator_assert_on_chain { - return Ok(GraphChainScan { - status: current_status, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: None, - }); - } - // TBD: add GraphStatus::Assert - } - - if current_status == GraphStatus::Challenge { - if let Some(disprove) = detect_guardian_disprove(btc_client, graph, challenge_txid).await? { - sub_status.disprove_type = Some(disprove.disprove_type); - sub_status.disprove_index = disprove.index as i32; - return Ok(GraphChainScan { - status: GraphStatus::Disprove, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: Some(disprove), - }); - } - - if let Some(disprove) = detect_watchtower_flow_disprove(btc_client, graph).await? { - sub_status.disprove_type = Some(disprove.disprove_type); - sub_status.disprove_index = disprove.index as i32; - return Ok(GraphChainScan { - status: GraphStatus::Disprove, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: Some(disprove), - }); - } - - // TBD: add GraphStatus::Assert - let mut detected_disprove = None; - for verifier_index in 0..verifier_num { - let verifier_assert_txid = graph.verifier_asserts[verifier_index].tx().compute_txid(); - let disprove_txid = graph.disproves[verifier_index].tx().compute_txid(); - if tx_on_chain(btc_client, &verifier_assert_txid).await? { - if let Some(spent_txid) = - outpoint_spent_txid(btc_client, &verifier_assert_txid, 0).await? - { - if spent_txid == disprove_txid { - sub_status.verifier_challenge_status[verifier_index] = - VerifierChallengeStatus::Disproved; - sub_status.disprove_type = Some(DisproveTxType::Disprove); - sub_status.disprove_index = verifier_index as i32; - detected_disprove = Some(DetectedDisprove { - disprove_type: DisproveTxType::Disprove, - index: verifier_index, - challenge_start_txid: challenge_txid, - challenge_finish_txid: spent_txid, - }); - current_status = GraphStatus::Disprove; - } else { - sub_status.verifier_challenge_status[verifier_index] = - VerifierChallengeStatus::ProverAnswered; - } - } else { - sub_status.verifier_challenge_status[verifier_index] = - VerifierChallengeStatus::VerifierAsserted; - } - } - } - if current_status == GraphStatus::Disprove { - return Ok(GraphChainScan { - status: current_status, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: detected_disprove, - }); - } - if let Some(disprove) = - detect_connector_d_disprove(btc_client, graph, challenge_txid).await? - { - sub_status.disprove_type = Some(disprove.disprove_type); - sub_status.disprove_index = disprove.index as i32; - return Ok(GraphChainScan { - status: GraphStatus::Disprove, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: Some(disprove), - }); - } - if tx_on_chain(btc_client, &take2_txid).await? { - return Ok(GraphChainScan { - status: GraphStatus::OperatorTake2, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: None, - }); - } - } - - Ok(GraphChainScan { - status: current_status, - sub_status, - challenge_txid, - watchtower_challenge_init_on_chain, - operator_assert_on_chain, - disprove: None, - }) -} - -#[allow(clippy::enum_variant_names)] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum GraphCompensateEventKind { - PreKickoffSent, // OperatorDataPushed -> PreKickoff - KickoffSent, // PreKickoff -> OperatorKickOff - Take1Sent, // OperatorKickOff -> OperatorTake1 - ChallengeSent, // OperatorKickOff -> Challenge - DisproveSent, // Challenge -> Disprove - Take2Sent, // Challenge -> OperatorTake2 -} - -fn map_transition_to_event(from: GraphStatus, to: GraphStatus) -> Option { - use GraphStatus::*; - match (from, to) { - (OperatorDataPushed, PreKickoff) => Some(GraphCompensateEventKind::PreKickoffSent), - (PreKickoff, OperatorKickOff) => Some(GraphCompensateEventKind::KickoffSent), - (OperatorKickOff, OperatorTake1) => Some(GraphCompensateEventKind::Take1Sent), - (OperatorKickOff, Challenge) => Some(GraphCompensateEventKind::ChallengeSent), - (Challenge, Disprove) => Some(GraphCompensateEventKind::DisproveSent), - (Challenge, OperatorTake2) => Some(GraphCompensateEventKind::Take2Sent), - _ => None, - } -} - -/// The canonical protocol ancestry used only for recovering missed local -/// messages. It is deliberately separate from transition authorization: a -/// chain scan can authorize additional recovery edges such as -/// `Obsoleted -> OperatorDataPushed`, but those edges do not imply a P2P -/// phase message should be synthesized. -fn compensation_previous_status(status: GraphStatus) -> Option { - use GraphStatus::*; - - match status { - CommitteePresigned => Some(OperatorPresigned), - OperatorDataPushed => Some(CommitteePresigned), - PreKickoff | Obsoleted | Skipped => Some(OperatorDataPushed), - OperatorKickOff => Some(PreKickoff), - OperatorTake1 | Challenge => Some(OperatorKickOff), - Disprove | OperatorTake2 => Some(Challenge), - OperatorPresigned => None, - } -} - -fn compensation_events_from( - compensation_anchor_status: GraphStatus, - final_status: GraphStatus, -) -> Option> { - if compensation_anchor_status == final_status { - return Some(vec![]); - } - - let mut reverse_path = vec![final_status]; - let mut cursor = final_status; - while cursor != compensation_anchor_status { - let previous = compensation_previous_status(cursor)?; - reverse_path.push(previous); - cursor = previous; - } - reverse_path.reverse(); - - Some( - reverse_path - .windows(2) - .filter_map(|window| match window { - [from, to] => map_transition_to_event(*from, *to), - _ => None, - }) - .collect(), - ) -} - -async fn upsert_graph_compensate_message( - local_db: &LocalDB, - actor: Actor, - message_content: GOATMessageContent, -) -> Result<()> { - let key = LocalMessageKey::from_content(actor.clone(), &message_content)?; - let graph_id = match message_content.business_ref() { - BusinessRef::Graph { graph_id, .. } => graph_id, - _ => bail!("graph compensation message must be graph-scoped"), - }; - let message_id = key.message_id(); - let mut storage_processor = local_db.start_transaction().await?; - if !storage_processor.insert_graph_compensation_marker(graph_id, &message_id).await? { - storage_processor.commit().await?; - return Ok(()); - } - - upsert_message( - &mut storage_processor, - false, - SELF_SENDER.to_string(), - actor, - message_content, - 0, - 0, - ) - .await?; - storage_processor.commit().await -} - -async fn push_graph_compensate_message( - local_db: &LocalDB, - actor: Actor, - message_content: GOATMessageContent, -) -> Result<()> { - // Unlike an action retry, an inferred chain event must not reset an - // existing queued message. This makes compensation safe to retry when the - // status write committed before the message was persisted. - upsert_graph_compensate_message(local_db, actor, message_content).await -} - -#[allow(dead_code)] -async fn should_emit_wrongly_challenge_timeout( - btc_client: &BTCClient, - challenge_assert_txid: Txid, -) -> Result { - let status = btc_client.get_tx_status(&challenge_assert_txid).await?; - let Some(block_height) = status.block_height else { - return Ok(false); - }; - let current_height = btc_client.get_height().await?; - Ok(current_height >= block_height + disprove_timelock(get_network())) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn compensate_graph_events( - local_db: &LocalDB, - _btc_client: &BTCClient, - instance_id: Uuid, - graph_id: Uuid, - _graph: &BitvmGcGraph, - scan: Option<&GraphChainScan>, - compensation_anchor_status: GraphStatus, - final_status: GraphStatus, -) -> Result<()> { - let Some(scan) = scan else { - tracing::debug!( - "Skip graph compensation for {instance_id}:{graph_id}: chain scan result is missing" - ); - return Ok(()); - }; - - // The action anchor is the only input used to build a recovery path. The - // persisted status is a projection and must not manufacture or suppress - // messages after a restart. Every enqueue below is idempotent. - let Some(events) = compensation_events_from(compensation_anchor_status, final_status) else { - tracing::debug!( - "Skip graph compensation without a canonical observed path for {instance_id}:{graph_id}: anchor={compensation_anchor_status:?}, final={final_status:?}" - ); - return Ok(()); - }; - - for event in events { - match event { - GraphCompensateEventKind::PreKickoffSent => { - push_graph_compensate_message( - local_db, - Actor::Verifier, - GOATMessageContent::PreKickoffSent(PreKickoffSent { instance_id, graph_id }), - ) - .await?; - } - GraphCompensateEventKind::KickoffSent => { - push_graph_compensate_message( - local_db, - Actor::All, - GOATMessageContent::KickoffSent(KickoffSent { instance_id, graph_id }), - ) - .await?; - } - GraphCompensateEventKind::Take1Sent => { - push_graph_compensate_message( - local_db, - Actor::Committee, - GOATMessageContent::Take1Sent(Take1Sent { instance_id, graph_id }), - ) - .await?; - } - GraphCompensateEventKind::ChallengeSent => { - if let Some(challenge_txid) = scan.challenge_txid { - push_graph_compensate_message( - local_db, - Actor::Operator, - GOATMessageContent::ChallengeSent(ChallengeSent { - instance_id, - graph_id, - challenge_txid, - }), - ) - .await?; - } - } - GraphCompensateEventKind::DisproveSent => { - let disprove = scan.disprove.clone().ok_or_else(|| { - anyhow!( - "Graph {instance_id}:{graph_id} reached Disprove but no disprove transaction was detected" - ) - })?; - upsert_graph_compensate_message( - local_db, - Actor::Committee, - GOATMessageContent::DisproveSent(DisproveSent { - instance_id, - graph_id, - disprove_type: disprove.disprove_type, - index: disprove.index, - challenge_start_txid: disprove.challenge_start_txid, - challenge_finish_txid: disprove.challenge_finish_txid, - }), - ) - .await?; - } - GraphCompensateEventKind::Take2Sent => { - push_graph_compensate_message( - local_db, - Actor::Committee, - GOATMessageContent::Take2Sent(Take2Sent { instance_id, graph_id }), - ) - .await?; - } - } - } - - Ok(()) -} - -pub fn build_graph_data(graph: &BitvmGcGraph) -> Result { - // operator pubkey: first byte is prefix, next 32 bytes are key - let op_pk_bytes = graph.parameters.operator_pubkey.to_bytes(); - let operator_pubkey_prefix = op_pk_bytes[0]; - let operator_pubkey: [u8; 32] = - op_pk_bytes[1..33].try_into().map_err(|_| anyhow!("invalid operator pubkey length"))?; - - // compute txids for all required transactions - let pegin_txid = graph.pegin.finalize().compute_txid().to_byte_array(); - let kickoff_txid = graph.kickoff.finalize().compute_txid().to_byte_array(); - let take1_txid = graph.take1.finalize().compute_txid().to_byte_array(); - let take2_txid = graph.take2.finalize().compute_txid().to_byte_array(); - let watchtower_challenge_init_txid = - graph.watchtower_challenge_init.finalize().compute_txid().to_byte_array(); - let prover_assert_txid = graph.operator_assert.finalize().compute_txid().to_byte_array(); - let disprove_txids: Vec<[u8; 32]> = - graph.disproves.iter().map(|tx| tx.finalize().compute_txid().to_byte_array()).collect(); - let watchtower_challenge_timeout_txids: Vec<[u8; 32]> = graph - .watchtower_challenge_timeouts - .iter() - .map(|tx| tx.finalize().compute_txid().to_byte_array()) - .collect(); - let operator_challenge_nack_txids: Vec<[u8; 32]> = graph - .operator_challenge_nacks - .iter() - .map(|tx| tx.finalize().compute_txid().to_byte_array()) - .collect(); - let operator_commit_timeout_txid = - graph.operator_commit_timeout.finalize().compute_txid().to_byte_array(); - - Ok(GraphData { - operator_pubkey_prefix, - operator_pubkey, - pegin_txid, - kickoff_txid, - take1_txid, - take2_txid, - watchtower_challenge_init_txid, - prover_assert_txid, - disprove_txids, - watchtower_challenge_timeout_txids, - operator_challenge_nack_txids, - operator_commit_timeout_txid, - }) -} - -pub async fn get_graph_digest(goat_client: &GOATClient, graph: &BitvmGcGraph) -> Result<[u8; 32]> { - let instance_id = graph.parameters.instance_parameters.instance_id; - let graph_id = graph.parameters.graph_id; - let graph_data = build_graph_data(graph)?; - goat_client.gateway_get_post_graph_digest(&instance_id, &graph_id, graph_data).await -} - -pub async fn validate_committee( - goat_client: &GOATClient, - peer_id: &PeerId, - instance_id: Uuid, - committee_pubkey: &PublicKey, -) -> Result<()> { - // return SpecialError::InvalidCommittee if not valid - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - for (i, pk) in pegin_data.committee_pubkeys.iter().enumerate() { - let pk = PublicKey::from_slice(pk)?; - if &pk == committee_pubkey { - let addr = pegin_data.committee_addresses[i]; - let stored_peer_id = goat_client.committee_mana_get_committee_peer_id(&addr).await?; - if stored_peer_id.to_vec() != peer_id.to_bytes() { - bail!(SpecialError::InvalidCommittee( - "committee pubkey & peer id mismatch".to_string() - )); - } - return Ok(()); - } - } - bail!(SpecialError::InvalidCommittee( - "committee pubkey not found in instance's committee pubkeys".to_string() - )) -} -pub async fn validate_committee_with_evm_address( - goat_client: &GOATClient, - peer_id: &PeerId, - instance_id: Uuid, - committee_pubkey: &PublicKey, - committee_evm_address: &EvmAddress, -) -> Result<()> { - // return SpecialError::InvalidCommittee if not valid - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - for i in 0..pegin_data.committee_pubkeys.len() { - let pk = PublicKey::from_slice(&pegin_data.committee_pubkeys[i])?; - let addr = &pegin_data.committee_addresses[i]; - if addr == committee_evm_address { - if &pk != committee_pubkey { - bail!(SpecialError::InvalidCommittee( - "committee evm address & pubkey mismatch".to_string() - )); - } - let stored_peer_id = goat_client.committee_mana_get_committee_peer_id(addr).await?; - if stored_peer_id.to_vec() != peer_id.to_bytes() { - bail!(SpecialError::InvalidCommittee( - "committee evm address & peer id mismatch".to_string() - )); - } - return Ok(()); - } - } - bail!(SpecialError::InvalidCommittee( - "committee evm address not found in instance's committee addresses".to_string() - )) -} - -pub async fn validate_graph_id_on_goat( - goat_client: &GOATClient, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - let graph_data_on_goat = goat_client.gateway_get_graph_data(&graph_id).await?; - if graph_data_on_goat.operator_pubkey == [0u8; 32] { - bail!("Graph {graph_id} not found on GoatChain") - } - let all_instance_graph_ids = - goat_client.gateway_get_graph_ids_by_instance_id(&instance_id).await?; - if !all_instance_graph_ids.contains(&graph_id) { - bail!("graph_id: {graph_id} and instance_id {instance_id} mismatch") - } - Ok(()) -} - -pub async fn read_pegin_request( - btc_client: &BTCClient, - goat_client: &GOATClient, - instance_id: Uuid, -) -> Result<(UserInfo, Amount, Vec)> { - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - if pegin_data.status != PeginStatus::Pending { - bail!("Invalid PeginRequest: expired or already processed"); - } - let network = get_network(); - let user_change_address = Address::from_str(&pegin_data.user_change_addr) - .map_err(|e| SpecialError::InvalidPeginRequest(format!("invalid user_change_addr: {e}")))? - .require_network(network) - .map_err(|e| { - SpecialError::InvalidPeginRequest(format!("invalid user_change_addr network: {e}")) - })?; - let user_refund_address = Address::from_str(&pegin_data.user_refund_addr) - .map_err(|e| SpecialError::InvalidPeginRequest(format!("invalid user_refund_addr: {e}")))? - .require_network(network) - .map_err(|e| { - SpecialError::InvalidPeginRequest(format!("invalid user_refund_addr network: {e}")) - })?; - let user_xonly_pubkey = - XOnlyPublicKey::from_slice(&pegin_data.user_xonly_pubkey).map_err(|e| { - SpecialError::InvalidPeginRequest(format!("invalid user_xonly_pubkey: {e}")) - })?; - let inputs: Vec = pegin_data - .user_inputs - .iter() - .map(|u| Input { - outpoint: OutPoint { txid: Txid::from_byte_array(u.txid), vout: u.vout }, - amount: Amount::from_sat(u.amount_sats), - }) - .collect(); - // TODO: we need to run our own bitcoin node in case of downtime or ddos attack. - for input in &inputs { - if !outpoint_available(btc_client, &input.outpoint.txid, input.outpoint.vout.into()).await? - { - bail!(SpecialError::InvalidPeginRequest(format!( - "input {}:{} is not available", - input.outpoint.txid, input.outpoint.vout - ))); - } - } - let user_info = UserInfo { - depositor_evm_address: pegin_data.depositor_address, - txn_fees: pegin_data.txn_fees, - inputs, - user_change_address, - user_refund_address, - user_xonly_pubkey, - }; - Ok((user_info, Amount::from_sat(pegin_data.pegin_amount_sats), pegin_data.committee_addresses)) -} - -pub async fn read_instance_info_from_goat( - goat_client: &GOATClient, - instance_id: Uuid, -) -> Result { - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - let network = get_network(); - let user_change_address = Address::from_str(&pegin_data.user_change_addr) - .map_err(|e| SpecialError::InvalidPeginData(format!("invalid user_change_addr: {e}")))? - .require_network(network) - .map_err(|e| { - SpecialError::InvalidPeginData(format!("invalid user_change_addr network: {e}")) - })?; - let user_refund_address = Address::from_str(&pegin_data.user_refund_addr) - .map_err(|e| SpecialError::InvalidPeginData(format!("invalid user_refund_addr: {e}")))? - .require_network(network) - .map_err(|e| { - SpecialError::InvalidPeginData(format!("invalid user_refund_addr network: {e}")) - })?; - let user_xonly_pubkey = XOnlyPublicKey::from_slice(&pegin_data.user_xonly_pubkey) - .map_err(|e| SpecialError::InvalidPeginData(format!("invalid user_xonly_pubkey: {e}")))?; - let inputs: Vec = pegin_data - .user_inputs - .iter() - .map(|u| Input { - outpoint: OutPoint { txid: Txid::from_byte_array(u.txid), vout: u.vout }, - amount: Amount::from_sat(u.amount_sats), - }) - .collect(); - let user_info = UserInfo { - depositor_evm_address: pegin_data.depositor_address, - txn_fees: pegin_data.txn_fees, - inputs, - user_change_address, - user_refund_address, - user_xonly_pubkey, - }; - let committee_pubkeys = match goat_client.gateway_get_committee_pubkeys(&instance_id).await { - Ok(pks) => pks, - Err(e) => { - if let Some(msg) = e.downcast_ref::() { - match msg { - SpecialError::EvmReverted(err_msg) => { - bail!(SpecialError::InvalidPeginData(format!( - "fail to get committee pubkeys: {err_msg}" - ))) - } - _ => bail!(e), - } - } else { - bail!(e) - } - } - }; - let committee_agg_pubkey = generate_n_of_n_public_key(&committee_pubkeys).0; - Ok(BitvmGcInstanceParameters { - network, - instance_id, - user_info, - pegin_amount: Amount::from_sat(pegin_data.pegin_amount_sats), - committee_pubkeys, - committee_agg_pubkey, - }) -} - -pub async fn is_take1_timelock_expired( - client: &BTCClient, - kickoff_height: u32, - timelock_config: &TimelockConfig, -) -> Result { - let lock_blocks = take1_timelock_with_config(get_network(), timelock_config); - let current_height = client.get_height().await?; - Ok(current_height >= kickoff_height + lock_blocks) -} - -pub async fn is_take2_timelock_expired( - client: &BTCClient, - operator_assert_height: u32, - watchtower_challenge_init_height: u32, - timelock_config: &TimelockConfig, -) -> Result { - let network = get_network(); - let connector_d_lock_blocks = take2_timelock_with_config(network, timelock_config); - let connector_f_lock_blocks = connector_f_timelock_blocks(network, timelock_config); - let current_height = client.get_height().await?; - Ok(current_height >= operator_assert_height + connector_d_lock_blocks - && current_height >= watchtower_challenge_init_height + connector_f_lock_blocks) -} - -pub async fn get_fee_rate(client: &BTCClient) -> Result { - match client.network() { - //TODO mempool api /fee-estimates failed, fix it latter - Network::Testnet | Network::Testnet4 | Network::Regtest => Ok(5.0), - _ => { - let res = client.get_fee_estimates().await?; - Ok(*res.get(&DEFAULT_CONFIRMATION_TARGET).ok_or(anyhow!( - "fee for {DEFAULT_CONFIRMATION_TARGET} confirmation target not found" - ))?) - } - } -} - -/// Broadcasts a raw transaction to the Bitcoin network using the mempool API. -/// -/// Requirements: -/// - The mempool API URL must be configured. -/// - The transaction should already be fully signed. -pub async fn broadcast_tx(client: &BTCClient, tx: &Transaction) -> Result<()> { - let txid = tx.compute_txid(); - let started_at = Instant::now(); - match client.broadcast(tx).await { - Ok(()) => { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_btc_tx_broadcast(true); - } - tracing::info!( - event = "btc_tx_broadcast", - outcome = "broadcasted", - txid = %txid, - input_count = tx.input.len(), - output_count = tx.output.len(), - elapsed_ms = started_at.elapsed().as_millis() as u64, - "bitcoin transaction broadcast accepted by client" - ); - Ok(()) - } - Err(err) => { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_btc_tx_broadcast(false); - } - tracing::warn!( - event = "btc_tx_broadcast", - outcome = "failed", - txid = %txid, - input_count = tx.input.len(), - output_count = tx.output.len(), - elapsed_ms = started_at.elapsed().as_millis() as u64, - error = %err, - "bitcoin transaction broadcast failed" - ); - Err(err) - } - } -} - -pub async fn broadcast_package( - client: &BTCClient, - txns: &[Transaction], - fallback_on_failure: bool, -) -> Result<()> { - let txids: Vec = txns.iter().map(Transaction::compute_txid).collect(); - let started_at = Instant::now(); - match client.broadcast_package(txns).await { - Ok(_) => { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_btc_tx_broadcast(true); - } - tracing::info!( - event = "btc_tx_package_broadcast", - outcome = "broadcasted", - transaction_count = txids.len(), - txids = ?txids, - elapsed_ms = started_at.elapsed().as_millis() as u64, - "bitcoin transaction package broadcast accepted by client" - ); - } - Err(e) => { - if fallback_on_failure { - tracing::warn!( - event = "btc_tx_package_broadcast", - outcome = "fallback_to_individual_broadcast", - transaction_count = txids.len(), - txids = ?txids, - elapsed_ms = started_at.elapsed().as_millis() as u64, - error = %e, - "bitcoin transaction package broadcast failed; falling back to individual broadcasts" - ); - for tx in txns { - broadcast_tx(client, tx).await?; - } - } else { - if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { - metrics_state.record_btc_tx_broadcast(false); - } - // Surface the original error when fallback is disabled - return Err(e); - } - } - } - Ok(()) -} - -fn gen_watchtower_commitment(graph_id: Uuid, proof_data: ProofData) -> Result> { - let graph_id = graph_id.as_bytes(); - let proof = - proof_data.proof.as_slice().try_into().map_err(|_| anyhow!("invalid proof length"))?; - let public_inputs = proof_data - .public_inputs - .as_slice() - .try_into() - .map_err(|_| anyhow!("invalid public inputs length"))?; - if proof_data.vk.len() != VK_HASH_SIZE { - bail!("invalid vk_hash length"); - } - if proof_data.zkm_version.is_empty() { - bail!("missing zkm_version"); - } - - Ok(build_watchtower_commitment( - graph_id, - proof, - &public_inputs, - &proof_data.vk, - &proof_data.zkm_version, - )) -} - -// proof network -/// Builds fresh Proof Builder authentication headers for one outbound request attempt. -fn proof_builder_auth_headers( - keypair: &Keypair, - role: ProofBuilderAuthRole, - path: &str, - body: &B, -) -> Result> { - Ok(sign_proof_builder_request(keypair, role, "POST", path, body)?.to_header_pairs()) -} - -/// Returns: -/// - `Ok(Some(WatchtowerCommitment), _)` if watchtower proof is available -/// - `Ok(None, wait_secs)` if watchtower proof is not yet available, with suggested wait time -pub async fn get_watchtower_commitment( - local_db: &LocalDB, - btc_client: &BTCClient, - http_client: &HttpAsyncClient, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<(Option>, usize)> { - let mut storage_processor = local_db.acquire().await?; - if let Some(graph) = storage_processor.find_graph(&graph_id).await? - && let Some(challenge_init_txid) = graph.watchtower_challenge_init_txid - { - // check if challenge_txid is confirmed - let tx_status = btc_client.get_tx_status(&challenge_init_txid.0).await?; - if !tx_status.confirmed { - warn!("graph {graph_id} challenge tx is not confirmed"); - return Ok((None, get_watchtower_proof_wait_secs())); - } - - let base_url = Url::parse( - &get_proof_build_rpc_host() - .ok_or_else(|| anyhow::anyhow!("failed to get proof_build_rpc_host"))?, - )?; - let url = base_url.join(NODES_WATCHTOWER_BASE)?; - - let payload = WatchtowerProofRequest { - instance_id: instance_id.to_string(), - graph_id: graph_id.to_string(), - gateway_address: Some(env::get_goat_gateway_contract_from_env().to_string()), - public_key: env::get_node_pubkey()?.to_string(), - challenge_init_txid: challenge_init_txid.0.to_string(), - execution_layer_block_number: graph.proceed_withdraw_height, // NOTE: this number may be zero - }; - let auth_keypair = get_bitvm_key()?; - let response = http_client - .post_response_json_with_dynamic_headers::( - url.as_str(), - &payload, - || { - proof_builder_auth_headers( - &auth_keypair, - ProofBuilderAuthRole::Watchtower, - NODES_WATCHTOWER_BASE, - &payload, - ) - }, - ) - .await?; - match response.proof_data { - Some(proof_data) => { - let watchtower_commitment = gen_watchtower_commitment(graph_id, proof_data)?; - Ok((Some(watchtower_commitment), 0)) - } - None => Ok((None, get_watchtower_proof_wait_secs())), - } - } else { - warn!("graph:{graph_id} not found or related txn is none",); - bail!("No graph in db"); - } -} - -#[derive(Debug, PartialEq, Eq)] -pub struct WatchtowerChallengeInfo { - pub challenge_txids: Vec>, - pub included_watchtowers: Vec, - pub resolved_branch_txids: Vec, -} - -#[tracing::instrument(name = "get_watchtower_challenge_info", skip(btc_client))] -pub async fn get_watchtower_challenge_info( - btc_client: &BTCClient, - watchtower_challenge_init_txid: &SerializableTxid, - watchtower_timeout_txids: &[Txid], - num_watchtowers: usize, -) -> Result { - let mut challenge_txids = Vec::with_capacity(num_watchtowers); - let mut included_watchtowers = Vec::with_capacity(num_watchtowers); - let mut resolved_branch_txids = Vec::with_capacity(num_watchtowers); - for index in 0..num_watchtowers { - let challenge_vout = - output_topology::watchtower_challenge_init::watchtower_connector(index) as u64; - let spent_txid = - outpoint_spent_txid(btc_client, &watchtower_challenge_init_txid.0, challenge_vout) - .await?; - match spent_txid { - Some(txid) => { - let status = btc_client.get_tx_status(&txid).await?; - if !status.confirmed { - bail!("watchtower branch tx {txid} at index {index} is not confirmed yet"); - } - resolved_branch_txids.push(txid); - - let is_timeout = watchtower_timeout_txids - .get(index) - .is_some_and(|timeout_txid| txid == *timeout_txid); - if is_timeout { - challenge_txids.push(None); - included_watchtowers.push(false); - } else { - challenge_txids.push(Some(txid.to_string())); - included_watchtowers.push(true); - } - } - None => { - challenge_txids.push(None); - included_watchtowers.push(false); - } - } - } - Ok(WatchtowerChallengeInfo { challenge_txids, included_watchtowers, resolved_branch_txids }) -} - -/// Returns `(btc_best_block_hash, included_watchtowers_bitmap)` using every finalized -/// challenge-branch resolution for the block hash while only challenges set bitmap bits. -/// -/// The selected hash is signed into the operator's pubin and cannot be replaced after a -/// reorganization. Require every branch that determines the bitmap to be at least as deeply -/// buried as the header/SPV finality window before returning it. -pub async fn compute_operator_pubin_blockhash_and_bitmap( - btc_client: &BTCClient, - resolved_branch_txids: &[Txid], - included_watchtowers_bits: &[bool], -) -> Result<([u8; 32], [u8; 32])> { - let tip_height = btc_client.get_height().await?; - let required_burial_depth = get_btc_block_confirms(btc_client.network()); - let btc_best_block_hash = { - let mut largest: Option<(u32, BlockHash)> = None; - for txid in resolved_branch_txids { - let status = btc_client.get_tx_status(txid).await?; - let (height, hash) = match (status.confirmed, status.block_height, status.block_hash) { - (true, Some(height), Some(hash)) => (height, hash), - _ => bail!("watchtower branch resolution tx {txid} is not confirmed yet"), - }; - // `BTC_BLOCK_CONFIRMS` is the number of blocks that must follow a block before - // the SPV/header pipeline consumes it. Match the proof-builder's strict boundary: - // the anchor must have more than this many blocks after it. - let burial_depth = tip_height.saturating_sub(height); - if burial_depth <= required_burial_depth { - bail!( - "watchtower branch resolution tx {txid} is not final enough for operator pubin: height={height}, tip={tip_height}, burial_depth={burial_depth}, required_burial_depth>{required_burial_depth}" - ); - } - if largest.is_none_or(|(h, _)| height > h) { - largest = Some((height, hash)); - } - } - largest - .map(|(_, hash)| hash.to_byte_array()) - .ok_or_else(|| anyhow!("no confirmed watchtower branch resolution tx available"))? - }; - - let mut included_watchtowers = [0u8; 32]; - for (i, &included) in included_watchtowers_bits.iter().enumerate() { - if included && i < 256 { - included_watchtowers[i / 8] |= 1 << (i % 8); - } - } - - Ok((btc_best_block_hash, included_watchtowers)) -} - -async fn get_operator_committed_blockhash( - btc_client: &BTCClient, - resolved_branch_txids: &[Txid], - included_watchtowers_bits: &[bool], - graph_id: Uuid, -) -> Result> { - match compute_operator_pubin_blockhash_and_bitmap( - btc_client, - resolved_branch_txids, - included_watchtowers_bits, - ) - .await - { - Ok((btc_best_block_hash, _)) => { - Ok(Some(BlockHash::from_byte_array(btc_best_block_hash).to_string())) - } - Err(e) => { - warn!("operator proof inputs are not ready for graph {graph_id}: {e}"); - Ok(None) - } - } -} - -/// Assembles the 96-byte guest pubin: -pub fn build_operator_guest_pubin( - btc_best_block_hash: &[u8; 32], - pubin_disprove_constant: &[u8; 32], - included_watchtowers: &[u8; 32], -) -> [u8; 96] { - let mut pubin = [0u8; 96]; - pubin[0..32].copy_from_slice(btc_best_block_hash); - pubin[32..64].copy_from_slice(pubin_disprove_constant); - pubin[64..96].copy_from_slice(included_watchtowers); - pubin -} - -fn load_part_stark_vk_for_zkm_version(zkm_version: &str) -> Result> { - let part_stark_vk = Groth16Verifier::try_get_part_stark_vk(zkm_version).ok_or_else(|| { - anyhow!("no partial STARK verifying key is bundled for Ziren {zkm_version}") - })?; - Ok(Vec::from(part_stark_vk)) -} - -fn convert_operator_proof_to_ark( - proof: &ZKMProofWithPublicValues, - vk_hash: &str, -) -> Result { - let part_stark_vk = load_part_stark_vk_for_zkm_version(&proof.zkm_version)?; - convert_ark_imm_wrap_vk(proof, vk_hash, &IMM_GROTH16_VK_BYTES, &part_stark_vk) - .map_err(|err| anyhow!("failed to convert operator proof to ark format: {err}")) -} - -fn check_operator_public_input_count(public_inputs: &[ark_bn254::Fr]) -> Result<()> { - if public_inputs.len() != NUM_PUBLIC_INPUTS { - bail!( - "operator proof has {} public inputs; expected {NUM_PUBLIC_INPUTS}", - public_inputs.len() - ); - } - Ok(()) -} - -pub fn operator_assert_dynamic_input(public_inputs: &[ark_bn254::Fr]) -> Result { - check_operator_public_input_count(public_inputs)?; - Ok(public_inputs[DYNAMIC_SLOTS[0]]) -} - -/// The static slots of an operator proof's public inputs, in `STATIC_SLOTS` order. -fn operator_static_inputs(public_inputs: &[ark_bn254::Fr]) -> Result { - check_operator_public_input_count(public_inputs)?; - Ok(STATIC_SLOTS.map(|slot| public_inputs[slot])) -} - -fn combined_operator_vk_hash(operator_vk_hash: &str, zkm_version: &str) -> Result<[u8; 32]> { - if !operator_vk_hash.starts_with("0x") { - bail!("configured operator vk hash must use 0x-prefixed Ziren encoding"); - } - let raw_vk_hash = decode_zkm_vkey_hash(operator_vk_hash) - .map_err(|e| anyhow!("invalid configured operator vk hash: {e:?}"))?; - let part_vk: PartStarkVerifyingKey = - bincode::deserialize(&load_part_stark_vk_for_zkm_version(zkm_version)?) - .context("deserialize operator partial STARK verifying key")?; - let base = Bn254Fr::from_u32(256); - let mut field_hash = Bn254Fr::ZERO; - for byte in raw_vk_hash { - field_hash = field_hash * base + Bn254Fr::from_u32(byte as u32); - } - let combined = zkm_recursion_core::hash_vkey_with_part_vk(&part_vk, field_hash); - let bytes = combined.as_canonical_biguint().to_bytes_be(); - if bytes.len() > 32 { - bail!("combined operator verifying key hash exceeds BN254 field encoding"); - } - let mut encoded = [0u8; 32]; - encoded[32 - bytes.len()..].copy_from_slice(&bytes); - Ok(encoded) -} - -fn operator_identity() -> Result<([u8; 32], String, StaticInputs)> { - let vk_hash = get_operator_vk_hash()?; - let zkm_version = get_operator_zkm_version()?; - let combined_hash = - combined_operator_vk_hash(&format!("0x{}", hex::encode(vk_hash)), &zkm_version)?; - let static_input = ziren_static_inputs(&combined_hash, &VK_ROOT_BYTES); - Ok((vk_hash, zkm_version, static_input)) -} - -pub fn derive_operator_static_input() -> Result { - Ok(operator_identity()?.2) -} - -/// Derives the Operator proof identity and graph constant for one challenge-init transaction. -pub fn derive_operator_statement( - graph_id: Uuid, - challenge_init_txid: [u8; 32], - watchtower_pubkeys: &[XOnlyPublicKey], -) -> Result { - let (vk_hash, zkm_version, static_input) = operator_identity()?; - let key_bytes = watchtower_pubkeys.iter().map(XOnlyPublicKey::serialize).collect::>(); - let constant = hash_operator_constant( - *graph_id.as_bytes(), - get_genesis_sequencer_commit_id(), - challenge_init_txid, - &key_bytes, - ); - Ok(OperatorStatement { static_input, vk_hash, zkm_version, constant }) -} - -/// Returns: -/// - `Ok(Some(OperatorProof), _)` if operator proof is available and valid -/// - `Ok(None, wait_secs)` if operator proof is not yet available -pub async fn get_operator_proof( - local_db: &LocalDB, - http_client: &HttpAsyncClient, - bitvm_graph: &BitvmGcGraph, - btc_client: &BTCClient, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<(Option, usize)> { - let mut storage_processor = local_db.acquire().await?; - let Some(graph) = storage_processor.find_graph(&graph_id).await? else { - warn!("graph:{graph_id} not found"); - bail!("No graph in db"); - }; - drop(storage_processor); - - if graph.proceed_withdraw_height <= 0 { - warn!("graph {graph_id} proceed_withdraw_height <= 0, waiting to been updated"); - return Ok((None, get_operator_proof_wait_secs())); - } - - let watchtower_challenge_init_txid = graph - .watchtower_challenge_init_txid - .ok_or_else(|| anyhow::anyhow!("watchtower_challenge_init_txid is none"))?; - let num_challenger = bitvm_graph.parameters.watchtower_pubkeys.len(); - let watchtower_timeout_txids: Vec = - bitvm_graph.watchtower_challenge_timeouts.iter().map(|tx| tx.tx().compute_txid()).collect(); - let WatchtowerChallengeInfo { - challenge_txids: watchtower_challenge_txids, - included_watchtowers, - resolved_branch_txids, - } = match get_watchtower_challenge_info( - btc_client, - &watchtower_challenge_init_txid, - &watchtower_timeout_txids, - num_challenger, - ) - .await - { - Ok(info) => info, - Err(e) => { - warn!("Failed to get watchtower challenge info: {e}"); - return Ok((None, get_operator_proof_wait_secs())); - } - }; - let Some(operator_committed_blockhash) = get_operator_committed_blockhash( - btc_client, - &resolved_branch_txids, - &included_watchtowers, - graph_id, - ) - .await? - else { - return Ok((None, get_operator_proof_wait_secs())); - }; - - let base_url = Url::parse( - &get_proof_build_rpc_host() - .ok_or_else(|| anyhow::anyhow!("failed to get proof_build_rpc_host"))?, - )?; - let operator_url = base_url.join(NODES_OPERATOR_BASE)?; - let payload = OperatorProofRequest { - instance_id: instance_id.to_string(), - graph_id: graph_id.to_string(), - gateway_address: Some(env::get_goat_gateway_contract_from_env().to_string()), - operator_committed_blockhash, - execution_layer_block_number: graph.proceed_withdraw_height, - watchtower_challenge_txids, - included_watchtowers, - watchtower_challenge_init_txid: watchtower_challenge_init_txid.0.to_string(), - watchtower_challenge_pubkeys: bitvm_graph - .parameters - .watchtower_pubkeys - .iter() - .map(|pk| pk.public_key(secp256k1::Parity::Even).to_string()) - .collect(), - }; - let auth_keypair = get_bitvm_key()?; - let operator_response = http_client - .post_response_json_with_dynamic_headers::( - operator_url.as_str(), - &payload, - || { - proof_builder_auth_headers( - &auth_keypair, - ProofBuilderAuthRole::Operator, - NODES_OPERATOR_BASE, - &payload, - ) - }, - ) - .await?; - - let Some(proof_data) = operator_response.proof_data else { - if let Some(error) = operator_response.error { - info!("operator proof is not ready for graph_id:{graph_id}: {error}"); - } - return Ok((None, get_operator_proof_wait_secs())); - }; - - let statement = derive_operator_statement( - graph_id, - bitvm_graph.watchtower_challenge_init.tx().compute_txid().to_byte_array(), - &bitvm_graph.parameters.watchtower_pubkeys, - )?; - if statement.constant != bitvm_graph.parameters.pubin_disprove_constant { - bail!("graph operator constant does not match its watchtower list"); - } - - let proof: ZKMProofWithPublicValues = bincode::deserialize(proof_data.proof.as_slice()) - .map_err(|err| anyhow!("failed to deserialize operator proof: {err}"))?; - - let operator_vk_hash_raw = decode_zkm_vkey_hash(&proof_data.vk) - .map_err(|e| anyhow!("invalid operator proof vk hash: {e:?}"))?; - if operator_vk_hash_raw != statement.vk_hash { - bail!("operator proof vk hash does not match configured operator identity"); - } - if proof.zkm_version != statement.zkm_version || proof_data.zkm_version != statement.zkm_version - { - bail!("operator proof Ziren version does not match configured operator identity"); - } - - let outputs = decode_operator_public_outputs(&proof.public_values.to_vec()) - .map_err(|e| anyhow!("invalid operator public outputs: {e}"))?; - if outputs.constant != statement.constant { - bail!("operator proof constant does not match graph setup"); - } - - let ark_proof = convert_operator_proof_to_ark(&proof, &proof_data.vk)?; - if operator_static_inputs(&ark_proof.public_inputs)? != statement.static_input { - bail!("operator proof static public inputs do not match graph setup statement"); - } - - Ok(( - Some(ValidatedOperatorProof { - proof: ark_proof.proof, - public_inputs: ark_proof.public_inputs.into(), - verifying_key: ark_proof.groth16_vk.into(), - public_values: proof.public_values.to_vec(), - vk_hash: proof_data.vk, - zkm_version: proof.zkm_version, - }), - 0, - )) -} - -pub async fn verifier_force_skip_kickoff(client: &BTCClient, graph: &BitvmGcGraph) -> Result { - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let verifier_master_keypair = verifier_master_key.master_keypair(); - let verifier_receive_address = - node_p2wsh_address(get_network(), &verifier_master_keypair.public_key().into()); - let fee_rate = get_fee_rate(client).await?; - let (force_skip_kickoff_tx, anchor_added) = - build_force_skip_kickoff_tx(graph, verifier_receive_address, fee_rate)?; - if anchor_added { - let anchor_vout = force_skip_kickoff_tx.output.len() as u64 - 1; - let force_skip_kickoff_tx_total_input_amount = - graph.force_skip_kickoff.prev_outs().iter().map(|o| o.value).sum::(); - let child_tx = build_cpfp_txns( - client, - &force_skip_kickoff_tx, - anchor_vout, - force_skip_kickoff_tx_total_input_amount, - ) - .await?; - match child_tx { - Some(tx) => { - broadcast_package(client, &[force_skip_kickoff_tx.clone(), tx], true).await? - } - None => broadcast_tx(client, &force_skip_kickoff_tx).await?, - } - } else { - broadcast_tx(client, &force_skip_kickoff_tx).await?; - } - Ok(force_skip_kickoff_tx.compute_txid()) -} - -pub async fn verifier_quick_challenge(client: &BTCClient, graph: &BitvmGcGraph) -> Result { - let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); - let verifier_master_keypair = verifier_master_key.master_keypair(); - let verifier_receive_address = - node_p2wsh_address(get_network(), &verifier_master_keypair.public_key().into()); - let fee_rate = get_fee_rate(client).await?; - let (quick_challenge_tx, anchor_added) = - build_quick_challenge_tx(graph, verifier_receive_address, fee_rate)?; - if anchor_added { - let anchor_vout = quick_challenge_tx.output.len() as u64 - 1; - let quick_challenge_tx_total_input_amount = - graph.quick_challenge.prev_outs().iter().map(|o| o.value).sum::(); - let child_tx = build_cpfp_txns( - client, - &quick_challenge_tx, - anchor_vout, - quick_challenge_tx_total_input_amount, - ) - .await?; - match child_tx { - Some(tx) => broadcast_package(client, &[quick_challenge_tx.clone(), tx], true).await?, - None => broadcast_tx(client, &quick_challenge_tx).await?, - } - } else { - broadcast_tx(client, &quick_challenge_tx).await?; - } - Ok(quick_challenge_tx.compute_txid()) -} - -pub async fn fund_address( - client: &BTCClient, - node_keypair: Keypair, - address: Address, - amount: Amount, -) -> Result { - let txins = Vec::new(); - let txouts = vec![TxOut { script_pubkey: address.script_pubkey(), value: amount }]; - let txid = - build_sign_and_broadcast_tx(client, node_keypair, txins, Amount::ZERO, txouts).await?; - Ok(OutPoint { txid, vout: 0 }) -} - -pub async fn build_sign_and_broadcast_tx( - client: &BTCClient, - node_keypair: Keypair, - txins: Vec, - total_input_amount: Amount, - txouts: Vec, -) -> Result { - let txouts = if txouts.is_empty() { - // bitcoin network does not allow a transaction without outputs - vec![TxOut { value: Amount::ZERO, script_pubkey: generate_opreturn_script(vec![]) }] - } else { - txouts - }; - let mut tx = Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: txins, - output: txouts, - }; - let fixed_inputs_num = tx.input.len(); - let total_output_amount: Amount = tx.output.iter().map(|o| o.value).sum(); - let fee_rate = get_fee_rate(client).await?; - let node_address = node_p2wsh_address(get_network(), &node_keypair.public_key().into()); - let shortfall = - Amount::from_sat(total_output_amount.to_sat().saturating_sub(total_input_amount.to_sat())); - match get_proper_utxo_set( - client, - tx.weight().to_vbytes_ceil(), - node_address.clone(), - shortfall, - fee_rate, - ) - .await? - { - Some((inputs, _, change_amount)) => { - for input in &inputs { - tx.input.push(TxIn { - previous_output: input.outpoint, - script_sig: ScriptBuf::new(), - sequence: Sequence::MAX, - witness: Witness::default(), - }); - } - if change_amount > Amount::from_sat(DUST_AMOUNT) { - tx.output.push(TxOut { - script_pubkey: node_address.script_pubkey(), - value: change_amount, - }); - } - for (i, input) in inputs.iter().enumerate() { - node_sign( - &mut tx, - i + fixed_inputs_num, - input.amount, - EcdsaSighashType::All, - &node_keypair, - )?; - } - broadcast_tx(client, &tx).await?; - Ok(tx.compute_txid()) - } - None => { - let current_balance = client - .get_address_utxo(node_address) - .await? - .iter() - .map(|u| u.value) - .sum::(); - bail!(SpecialError::InsufficientBalance(format!( - "Not enough balance to complete the transaction, current_balance: {current_balance} < shortfall: {shortfall}" - ))); - } - } -} - -pub async fn build_cpfp_txns( - btc_client: &BTCClient, - parent_tx: &Transaction, - anchor_vout: u64, - parent_tx_total_input_amount: Amount, -) -> Result> { - let network = get_network(); - if network == Network::Regtest || network == Network::Testnet || network == Network::Testnet4 { - // skip cpfp in test network for testing convenience - return Ok(None); - } - let node_master_keypair = get_bitvm_key()?; - let node_address = node_p2wsh_address(network, &node_master_keypair.public_key().into()); - let total_output_amount: Amount = parent_tx.output.iter().map(|o| o.value).sum(); - let fee_rate = get_fee_rate(btc_client).await?; - let fee_amount = - Amount::from_sat((parent_tx.weight().to_vbytes_ceil() as f64 * fee_rate).ceil() as u64); - if total_output_amount + fee_amount <= parent_tx_total_input_amount { - return Ok(None); - }; - let shortfall = total_output_amount + fee_amount - parent_tx_total_input_amount; - match get_proper_utxo_set( - btc_client, - ANCHOR_CHILD_BASE_VBYTES, - node_address.clone(), - shortfall, - fee_rate, - ) - .await? - { - Some((inputs, _, change_amount)) => { - let mut child_tx = Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: vec![], - output: vec![], - }; - if change_amount > Amount::from_sat(DUST_AMOUNT) { - child_tx.output.push(TxOut { - script_pubkey: node_address.script_pubkey(), - value: change_amount, - }); - } else { - // add an op_return output to avoid no-output transaction - child_tx.output.push(TxOut { - script_pubkey: generate_opreturn_script(vec![]), - value: Amount::ZERO, - }); - } - for input in &inputs { - child_tx.input.push(TxIn { - previous_output: input.outpoint, - script_sig: ScriptBuf::new(), - sequence: Sequence::MAX, - witness: Witness::default(), - }); - } - child_tx.input.push(TxIn { - previous_output: OutPoint { - txid: parent_tx.compute_txid(), - vout: anchor_vout as u32, - }, - script_sig: ScriptBuf::new(), - sequence: Sequence::MAX, - witness: Witness::default(), - }); - for (i, input) in inputs.iter().enumerate() { - node_sign( - &mut child_tx, - i, - input.amount, - EcdsaSighashType::All, - &node_master_keypair, - )?; - } - Ok(Some(child_tx)) - } - None => { - let current_balance = btc_client - .get_address_utxo(node_address) - .await? - .iter() - .map(|u| u.value) - .sum::(); - bail!(SpecialError::InsufficientBalance(format!( - "Not enough balance to complete the transaction, current_balance: {current_balance}" - ))) - } - } -} - -pub async fn broadcast_tx_with_cpfp( - btc_client: &BTCClient, - parent_tx: Transaction, - parent_tx_total_input_amount: Amount, -) -> Result<()> { - let anchor_output = p2a_output(); - let anchor_vout = parent_tx - .output - .iter() - .position(|output| output == &anchor_output) - .ok_or_else(|| anyhow!("cannot CPFP transaction without a P2A anchor output"))?; - if let Some(duplicate_vout) = parent_tx - .output - .iter() - .enumerate() - .skip(anchor_vout + 1) - .find_map(|(vout, output)| if output == &anchor_output { Some(vout) } else { None }) - { - bail!("transaction has multiple P2A anchor outputs at {anchor_vout} and {duplicate_vout}"); - }; - let child_tx = - build_cpfp_txns(btc_client, &parent_tx, anchor_vout as u64, parent_tx_total_input_amount) - .await?; - match child_tx { - Some(tx) => broadcast_package(btc_client, &[parent_tx, tx], true).await?, - None => broadcast_tx(btc_client, &parent_tx).await?, - }; - Ok(()) -} - -/// Returns: -/// - `Ok(None)` if given address does not have enough btc, -/// - `Ok(Some((utxos, fee_amount, change_amount)))` -pub async fn get_proper_utxo_set( - client: &BTCClient, - base_vbytes: u64, - address: Address, - target_amount: Amount, - fee_rate: f64, -) -> Result, Amount, Amount)>> { - fn estimate_tx_vbytes(base_vbytes: u64, extra_inputs: usize, extra_outputs: usize) -> u64 { - // p2wsh inputs/outputs - base_vbytes - + (extra_inputs as u64 * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE) - + (extra_outputs as u64 * P2WSH_OUTPUT_VBYTES) - } - fn to_input(utxos: Vec) -> Vec { - utxos - .into_iter() - .map(|utxo| Input { - outpoint: OutPoint { txid: utxo.txid, vout: utxo.vout }, - amount: utxo.value, - }) - .collect() - } - tracing::debug!("get utxos from: {address}"); - - let utxos = client.get_address_utxo(address).await?; - let mut sorted_utxos = utxos; - sorted_utxos.sort_by(|a, b| b.value.cmp(&a.value)); - - let mut selected = Vec::new(); - let mut total_value = Amount::ZERO; - - for utxo in sorted_utxos.into_iter().take(MAX_CUSTOM_INPUTS) { - selected.push(utxo.clone()); - total_value += utxo.value; - - let num_inputs = selected.len(); - let num_outputs = 1; // change - let tx_vbytes = estimate_tx_vbytes(base_vbytes, num_inputs, num_outputs); - let fee = Amount::from_sat((tx_vbytes as f64 * fee_rate).ceil() as u64); - - if total_value >= target_amount + fee { - let change = total_value - target_amount - fee; - return Ok(Some((to_input(selected), fee, change))); - } - } - - Ok(None) -} - -/// Returns: -/// - `Ok((Empty, None))` if given address does not have enough btc, -/// - `Ok((Empty, Some((SplitTx, Vec))))` if UTXO cannot be properly grouped, and a split transaction is needed, needing user to sign and broadcast -/// - `Ok(Vec>, None))` -pub async fn get_proper_utxo_sets( - client: &BTCClient, - address: Address, - target_amounts: Vec, - fee_rate: f64, -) -> Result<(Vec>, Option<(Transaction, Vec)>)> { - let mut utxos = client.get_address_utxo(address.clone()).await?; - - let total_available_sat: u64 = utxos.iter().map(|u| u.value.to_sat()).sum(); - - let total_target_sat: u64 = target_amounts.iter().map(|a| a.to_sat()).sum(); - - if total_available_sat <= total_target_sat { - return Ok((Vec::new(), None)); - } - - let mut targets_with_idx: Vec<(usize, Amount)> = - target_amounts.clone().into_iter().enumerate().collect(); - targets_with_idx.sort_by(|a, b| b.1.to_sat().cmp(&a.1.to_sat())); - - utxos.sort_by(|a, b| b.value.to_sat().cmp(&a.value.to_sat())); - - let mut available_indices: Vec = (0..utxos.len()).collect(); - - use std::collections::HashMap; - let mut temp_groups: HashMap> = HashMap::new(); - - for (original_idx, target) in targets_with_idx { - let mut group: Vec = Vec::new(); - let mut sum_sat: u64 = 0; - - let i = 0; - while i < available_indices.len() && sum_sat < target.to_sat() { - let utxo_idx = available_indices[i]; - let utxo = utxos[utxo_idx].clone(); - sum_sat += utxo.value.to_sat(); - group.push(Input { - outpoint: OutPoint { txid: utxo.txid, vout: utxo.vout }, - amount: utxo.value, - }); - available_indices.remove(i); - } - - if sum_sat < target.to_sat() { - // if all UTXOs are used but target not met, need to build a split transaction (already checked that total_available > total_target) - let script_pubkey = address.script_pubkey(); - let base_outputs: Vec = target_amounts - .iter() - .map(|amt| TxOut { value: *amt, script_pubkey: script_pubkey.clone() }) - .collect(); - let tx_ins: Vec = utxos - .iter() - .map(|u| TxIn { - previous_output: OutPoint { txid: u.txid, vout: u.vout }, - script_sig: ScriptBuf::new(), - sequence: Sequence::MAX, - witness: bitcoin::Witness::new(), - }) - .collect(); - let txin_amounts: Vec = utxos.iter().map(|u| u.value).collect(); - - let n_inputs = tx_ins.len() as u64; - let n_outputs = base_outputs.len() as u64 + 1; - let est_vbytes = 100u64 - + n_inputs * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE - + n_outputs * P2WSH_OUTPUT_VBYTES; - let est_fee_sat = (est_vbytes as f64 * fee_rate).ceil() as u64; - - if total_available_sat < total_target_sat + est_fee_sat { - return Ok((Vec::new(), None)); - } - - let change_sat = total_available_sat - .checked_sub(total_target_sat + est_fee_sat) - .ok_or_else(|| anyhow!("overflow when calculating change"))?; - - let mut tx_outs = base_outputs; - - if change_sat >= DUST_AMOUNT { - tx_outs.push(TxOut { - value: Amount::from_sat(change_sat), - script_pubkey: script_pubkey.clone(), - }); - } - - let split_tx = Transaction { - version: bitcoin::transaction::Version(2), - lock_time: bitcoin::absolute::LockTime::ZERO, - input: tx_ins, - output: tx_outs, - }; - return Ok((Vec::new(), Some((split_tx, txin_amounts)))); - } - - temp_groups.insert(original_idx, group); - } - - let mut ordered: Vec<(usize, Vec)> = temp_groups.into_iter().collect(); - ordered.sort_by_key(|(idx, _)| *idx); - let grouped: Vec> = ordered.into_iter().map(|(_, g)| g).collect(); - - Ok((grouped, None)) -} - -pub fn node_p2wsh_script(pubkey: &PublicKey) -> ScriptBuf { - script! { - { *pubkey } - OP_CHECKSIG - } - .compile() -} -pub fn node_p2wsh_address(network: Network, pubkey: &PublicKey) -> Address { - Address::p2wsh(&node_p2wsh_script(pubkey), network) -} - -pub fn node_sign( - tx: &mut Transaction, - input_index: usize, - input_value: Amount, - sighash_type: EcdsaSighashType, - node_keypair: &Keypair, -) -> Result<()> { - let node_pubkey = node_keypair.public_key(); - populate_p2wsh_witness( - tx, - input_index, - sighash_type, - &node_p2wsh_script(&node_pubkey.into()), - input_value, - &vec![node_keypair], - ); - Ok(()) -} - -pub async fn build_genesis_prekickoff_tx( - btc_client: &BTCClient, - goat_client: &GOATClient, -) -> Result { - let watchtower_num = goat_client.committee_mana_get_watchtowers().await?.len(); - let verifier_num = min_required_verifier(); - let network = get_network(); - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let node_keypair = operator_master_key.master_keypair(); - let operator_taproot_public_key = node_keypair.x_only_public_key().0; - let cur_prekickoff_connector = PrekickoffConnector::new(network, &operator_taproot_public_key); - let next_force_skip_connector = ForceSkipConnector::new(network, &operator_taproot_public_key); - let next_kickoff_connector = KickoffConnector::new(network, &operator_taproot_public_key); - let next_prekickoff_connector = PrekickoffConnector::new(network, &operator_taproot_public_key); - let init_amount = prekickoff_replenishment_amount(); - let cur_prekickoff_connector_input = genesis_funding_input( - btc_client, - node_keypair, - cur_prekickoff_connector.generate_taproot_address(), - init_amount, - ) - .await?; - let fee_amount = prekickoff_fee_amount(0); - PrekickoffTransaction::new_for_validation( - &cur_prekickoff_connector, - &next_force_skip_connector, - &next_kickoff_connector, - &next_prekickoff_connector, - cur_prekickoff_connector_input, - vec![], - vec![], - fee_amount.to_sat(), - watchtower_num, - verifier_num, - ) - .map_err(|e| anyhow::anyhow!("failed to create pre-kickoff txn: {e}")) -} - -/// Reuse a matching unspent output, or fund the genesis address. -async fn genesis_funding_input( - btc_client: &BTCClient, - node_keypair: Keypair, - address: Address, - amount: Amount, -) -> Result { - let existing = btc_client - .get_address_utxo(address.clone()) - .await? - .into_iter() - .filter(|utxo| utxo.value == amount) - .min_by_key(|utxo| (utxo.txid, utxo.vout)); - let outpoint = match existing { - Some(utxo) => OutPoint { txid: utxo.txid, vout: utxo.vout }, - None => fund_address(btc_client, node_keypair, address, amount).await?, - }; - Ok(Input { outpoint, amount }) -} - -pub async fn build_prekickoff_params( - btc_client: &BTCClient, - graph_nonce: u64, - cur_prekickoff_txn: PrekickoffTransaction, -) -> Result { - let prekickoff_remaining_amount = cur_prekickoff_txn - .prekickoff_connector_input() - .map_err(|e| anyhow!("failed to get pre-kickoff connector input: {e}"))? - .amount; - let (replenish_fee_inputs, replenish_fee_prev_outs, fee_amount) = if prekickoff_remaining_amount - >= min_prekickoff_input_amount() - { - // no need to replenish funds - (vec![], vec![], prekickoff_fee_amount(0)) - } else { - let network = get_network(); - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let master_keypair = operator_master_key.master_keypair(); - let nonce_keypair = operator_master_key.keypair_for_nonce(graph_nonce); - let nonce_address = node_p2wsh_address(network, &nonce_keypair.public_key().into()); - let replenishment_amount = prekickoff_replenishment_amount(); - let mut replenish_fee_inputs: Vec = btc_client - .get_address_utxo(nonce_address.clone()) - .await? - .into_iter() - .map(|u| Input { outpoint: OutPoint { txid: u.txid, vout: u.vout }, amount: u.value }) - .collect(); - let current_balance: Amount = replenish_fee_inputs.iter().map(|i| i.amount).sum(); - if current_balance < replenishment_amount { - let shortfall = replenishment_amount - current_balance; - let extra_input = Input { - outpoint: fund_address( - btc_client, - master_keypair, - nonce_address.clone(), - shortfall, - ) - .await?, - amount: shortfall, - }; - replenish_fee_inputs.push(extra_input); - }; - let replenish_fee_prev_outs: Vec = replenish_fee_inputs - .iter() - .map(|i| TxOut { value: i.amount, script_pubkey: nonce_address.script_pubkey() }) - .collect(); - let fee_amount = prekickoff_fee_amount(replenish_fee_inputs.len()); - (replenish_fee_inputs, replenish_fee_prev_outs, fee_amount) - }; - Ok(PrekickoffParameters { - cur_prekickoff_txn, - replenish_fee_inputs, - replenish_fee_prev_outs, - fee_amount: fee_amount.to_sat(), - }) -} - -pub async fn build_graph_params( - _local_db: &LocalDB, - goat_client: &GOATClient, - instance_parameters: BitvmGcInstanceParameters, - prekickoff_parameters: PrekickoffParameters, - bitvm_gc_circuit_datas: Vec, - graph_nonce: u64, - graph_id: Uuid, -) -> Result { - let network = instance_parameters.network; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let operator_master_keypair = operator_master_key.master_keypair(); - let operator_pubkey = operator_master_keypair.public_key().into(); - let operator_receive_address = - node_p2wsh_address(instance_parameters.network, &operator_pubkey); - let (_, operator_assert_wots_pubkey) = - operator_master_key.assert_wots_keypair_for_graph(graph_id); - let (_, operator_commit_pubin_wots_pubkey) = - operator_master_key.commit_pubin_wots_keypair_for_graph(graph_id); - let watchtower_pubkeys = goat_client.committee_mana_get_watchtowers().await?; - let watchtower_ack_hashlocks = (0..watchtower_pubkeys.len()) - .map(|index| { - bitcoin::hashes::hash160::Hash::hash( - &operator_master_key.preimage_for_graph(graph_id, index), - ) - .to_byte_array() - }) - .collect(); - validate_watchtower_registry_selection(&watchtower_pubkeys, &watchtower_pubkeys)?; - Ok(BitvmGcGraphParameters { - instance_parameters, - prekickoff_parameters, - timelock_config: default_timelock_config(network), - graph_id, - graph_nonce, - challenge_amount: challenge_amount(), - operator_pubkey, - operator_assert_wots_pubkey, - operator_commit_pubin_wots_pubkey, - operator_receive_address, - watchtower_pubkeys, - watchtower_ack_hashlocks, - pubin_disprove_constant: [0u8; 32], - gc_data: bitvm_gc_circuit_datas, - }) -} - -pub async fn operator_skip_graph(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result<()> { - let graph_nonce = graph.parameters.graph_nonce; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let operator_master_keypair = operator_master_key.master_keypair(); - let operator_receive_address = - node_p2wsh_address(get_network(), &operator_master_keypair.public_key().into()); - let operator_graph_keypair = operator_master_key.master_keypair(); - let mut prekickoff_tx = operator_sign_prekickoff_input_0(operator_graph_keypair, graph)?; - if prekickoff_tx.input.len() != 1 { - let operator_nonce_keypair = operator_master_key.keypair_for_nonce(graph_nonce - 1); - for i in 1..prekickoff_tx.input.len() { - let input_value = graph.cur_prekickoff.input_amounts[i]; - node_sign( - &mut prekickoff_tx, - i, - input_value, - bitcoin::EcdsaSighashType::All, - &operator_nonce_keypair, - )?; - } - } - let anchor_vout = prekickoff_tx.output.len() as u64 - 1; - let prekickoff_tx_total_input_amount = - graph.cur_prekickoff.input_amounts.clone().into_iter().sum(); - let child_tx = - build_cpfp_txns(btc_client, &prekickoff_tx, anchor_vout, prekickoff_tx_total_input_amount) - .await?; - let prekickoff_txid = prekickoff_tx.compute_txid(); - match operator_sign_skip_kickoff( - operator_graph_keypair, - graph, - operator_receive_address, - get_fee_rate(btc_client).await?, - )? { - Some(skip_kickoff_tx) => { - let skip_kickoff_txid = skip_kickoff_tx.compute_txid(); - if !tx_on_chain(btc_client, &prekickoff_txid).await? { - broadcast_package(btc_client, &[prekickoff_tx, skip_kickoff_tx], true).await?; - if let Some(child_tx) = child_tx { - broadcast_tx(btc_client, &child_tx).await?; - } - } else if !tx_on_chain(btc_client, &skip_kickoff_txid).await? { - if let Some(child_tx) = child_tx { - broadcast_package(btc_client, &[skip_kickoff_tx, child_tx], true).await?; - } else { - broadcast_tx(btc_client, &skip_kickoff_tx).await?; - } - } - } - None => match child_tx { - Some(tx) => { - if !tx_on_chain(btc_client, &prekickoff_txid).await? { - broadcast_package(btc_client, &[prekickoff_tx, tx], true).await?; - } - } - None => { - if !tx_on_chain(btc_client, &prekickoff_txid).await? { - broadcast_tx(btc_client, &prekickoff_tx).await?; - } - } - }, - }; - Ok(()) -} - -pub async fn operator_kickoff(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result<()> { - let graph_nonce = graph.parameters.graph_nonce; - let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); - let operator_graph_keypair = operator_master_key.master_keypair(); - let mut prekickoff_tx = operator_sign_prekickoff_input_0(operator_graph_keypair, graph)?; - if prekickoff_tx.input.len() != 1 { - let operator_nonce_keypair = operator_master_key.keypair_for_nonce(graph_nonce - 1); - for i in 1..prekickoff_tx.input.len() { - let input_value = graph.cur_prekickoff.input_amounts[i]; - node_sign( - &mut prekickoff_tx, - i, - input_value, - bitcoin::EcdsaSighashType::All, - &operator_nonce_keypair, - )?; - } - } - let prekickoff_txid = prekickoff_tx.compute_txid(); - let anchor_vout = prekickoff_tx.output.len() as u64 - 1; - let prekickoff_tx_total_input_amount = - graph.cur_prekickoff.input_amounts.clone().into_iter().sum(); - let prekickoff_child_tx = - build_cpfp_txns(btc_client, &prekickoff_tx, anchor_vout, prekickoff_tx_total_input_amount) - .await?; - - let kickoff_tx = operator_sign_kickoff(operator_graph_keypair, graph)?; - let kickoff_txid = kickoff_tx.compute_txid(); - let anchor_vout = kickoff_tx.output.len() as u64 - 1; - let kickoff_tx_total_input_amount = graph.kickoff.prev_outs().iter().map(|o| o.value).sum(); - let kickoff_child_tx = - build_cpfp_txns(btc_client, &kickoff_tx, anchor_vout, kickoff_tx_total_input_amount) - .await?; - - // If a tx is already on-chain, skip rebroadcasting and move to the next one. - let mut kickoff_child_broadcasted = false; - if !tx_on_chain(btc_client, &prekickoff_txid).await? { - // Parent not on-chain yet: broadcast parent and kickoff together as a package. - broadcast_package(btc_client, &[prekickoff_tx, kickoff_tx], true).await?; - } else if !tx_on_chain(btc_client, &kickoff_txid).await? { - // Parent is on-chain, but kickoff isn't: try kickoff (and its CPFP child if present). - if let Some(child) = kickoff_child_tx.as_ref() { - broadcast_package(btc_client, &[kickoff_tx, child.clone()], true).await?; - kickoff_child_broadcasted = true; - } else { - broadcast_tx(btc_client, &kickoff_tx).await?; - } - } - - // Ensure both transactions are seen on-chain and then handle CPFP children. - if !tx_on_chain(btc_client, &prekickoff_txid).await? { - bail!("prekickoff tx not on chain after broadcasting"); - } - if let Some(prekickoff_child_tx) = prekickoff_child_tx - && let Err(e) = broadcast_tx(btc_client, &prekickoff_child_tx).await - { - tracing::warn!("failed to broadcast prekickoff child tx: {e}"); - } - if !tx_on_chain(btc_client, &kickoff_txid).await? { - bail!("kickoff tx not on chain after broadcasting"); - } - if !kickoff_child_broadcasted - && let Some(kickoff_child_tx) = kickoff_child_tx - && let Err(e) = broadcast_tx(btc_client, &kickoff_child_tx).await - { - tracing::warn!("failed to broadcast kickoff child tx: {e}"); - } - Ok(()) -} - -pub async fn send_challenge_tx(btc_client: &BTCClient, graph: &BitvmGcGraph) -> Result { - let (mut challenge_tx, _) = export_challenge_tx(graph)?; - let challenge_keypair = VerifierMasterKey::new(get_bitvm_key()?).master_keypair(); - let verifier_evm_address = get_node_goat_address() - .ok_or_else(|| anyhow::anyhow!("failed to get node goat address".to_string()))?; - challenge_tx.output.push(bitcoin::TxOut { - value: Amount::ZERO, - script_pubkey: generate_opreturn_script(verifier_evm_address.to_vec()), - }); - let connector_a_input = graph - .kickoff - .connector_a_input() - .map_err(|e| anyhow!("failed to get connector-a input: {e}"))?; - build_sign_and_broadcast_tx( - btc_client, - challenge_keypair, - challenge_tx.input, - connector_a_input.amount, - challenge_tx.output, - ) - .await -} - -pub async fn send_watchtower_challenge_tx( - btc_client: &BTCClient, - graph: &BitvmGcGraph, - watchtower_index: usize, - commitment_data: Vec, -) -> Result { - let watchtower_keypair = WatchtowerMasterKey::new(get_bitvm_key()?).master_keypair(); - let fee_rate = get_fee_rate(btc_client).await?; - let watchtower_challenge_tx_base_vbytes = - estimate_watchtower_challenge_vbytes(commitment_data.len()); - let node_address = node_p2wsh_address(get_network(), &watchtower_keypair.public_key().into()); - match get_proper_utxo_set( - btc_client, - watchtower_challenge_tx_base_vbytes as u64, - node_address.clone(), - Amount::ZERO, - fee_rate, - ) - .await? - { - Some((inputs, fee_amount, _)) => { - let mut watchtower_challenge_tx = build_watchtower_challenge_tx( - graph, - &watchtower_keypair, - watchtower_index, - &commitment_data, - inputs.clone(), - &node_address, - fee_amount, - ) - .unwrap(); - for (i, input) in inputs.iter().enumerate() { - node_sign( - &mut watchtower_challenge_tx, - i + 1, - input.amount, - EcdsaSighashType::All, - &watchtower_keypair, - )?; - } - broadcast_tx(btc_client, &watchtower_challenge_tx).await?; - Ok(watchtower_challenge_tx.compute_txid()) - } - None => { - let current_balance = btc_client - .get_address_utxo(node_address) - .await? - .iter() - .map(|u| u.value) - .sum::(); - bail!(SpecialError::InsufficientBalance(format!( - "Not enough balance to complete the transaction, current_balance: {current_balance}" - ))); - } - } -} - -pub async fn endorse_graph(goat_client: &GOATClient, graph: &BitvmGcGraph) -> Result { - let signer = PrivateKeySigner::from_str(&get_node_goat_private_key()?)?; - let graph_digest = get_graph_digest(goat_client, graph).await?; - let sig = signer.sign_hash(&graph_digest.into()).await?; - Ok(sig) -} - -pub async fn endorse_graph_params(graph: &BitvmGcGraph) -> Result { - let signer = PrivateKeySigner::from_str(&get_node_goat_private_key()?)?; - let params_hash = graph.parameters.canonical_graph_params_hash()?; - let sig = signer.sign_hash(¶ms_hash.into()).await?; - Ok(sig) -} - -pub async fn endorse_pegin( - goat_client: &GOATClient, - instance_id: Uuid, - pegin_txid: &Txid, -) -> Result { - let signer = PrivateKeySigner::from_str(&get_node_goat_private_key()?)?; - let pegin_digest = goat_client.gateway_get_post_pegin_digest(&instance_id, pegin_txid).await?; - let sig = signer.sign_hash(&pegin_digest.into()).await?; - Ok(sig) -} - -pub async fn verify_pegin_endorsement( - goat_client: &GOATClient, - instance_id: Uuid, - committee_pubkey: &PublicKey, - pegin_txid: &Txid, - signature: &[u8], -) -> Result { - let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; - let expected_evm_address = pegin_data - .committee_pubkeys - .iter() - .zip(pegin_data.committee_addresses.iter()) - .find_map(|(pubkey, evm_address)| { - PublicKey::from_slice(pubkey) - .ok() - .filter(|on_chain_pubkey| on_chain_pubkey == committee_pubkey) - .map(|_| *evm_address) - }) - .ok_or_else(|| anyhow!("committee pubkey {committee_pubkey} not found on-chain"))?; - let pegin_digest = goat_client.gateway_get_post_pegin_digest(&instance_id, pegin_txid).await?; - let sig = EvmSignature::try_from(signature)?; - sig.recover_address_from_prehash(&pegin_digest.into()) - .map(|addr| addr == expected_evm_address) - .map_err(|e| e.into()) -} - -pub async fn verify_graph_endorsement( - goat_client: &GOATClient, - evm_address: &EvmAddress, - graph: &BitvmGcGraph, - signature: &[u8], -) -> Result { - let graph_digest = get_graph_digest(goat_client, graph).await?; - let sig = EvmSignature::try_from(signature)?; - sig.recover_address_from_prehash(&graph_digest.into()) - .map(|addr| &addr == evm_address) - .map_err(|e| e.into()) -} - -pub fn verify_graph_params_endorsement( - evm_address: &EvmAddress, - graph: &BitvmGcGraph, - signature: &[u8], -) -> Result { - let params_hash = graph.parameters.canonical_graph_params_hash()?; - let sig = EvmSignature::try_from(signature)?; - sig.recover_address_from_prehash(¶ms_hash.into()) - .map(|addr| &addr == evm_address) - .map_err(|e| e.into()) -} - -pub fn sign_verifier_graph_params( - verifier_keypair: Keypair, - graph: &SimplifiedBitvmGcGraph, -) -> Result { - let params_hash = graph.canonical_graph_params_hash()?; - Ok(SECP256K1.sign_schnorr(&SecpMessage::from_digest(params_hash), &verifier_keypair)) -} - -pub fn verify_verifier_graph_params_endorsement( - verifier_pubkey: &PublicKey, - graph: &SimplifiedBitvmGcGraph, - signature: &SchnorrSignature, -) -> Result { - let params_hash = graph.canonical_graph_params_hash()?; - Ok(SECP256K1 - .verify_schnorr( - signature, - &SecpMessage::from_digest(params_hash), - &XOnlyPublicKey::from(*verifier_pubkey), - ) - .is_ok()) -} - -/// Validates whether the given kickoff transaction has been confirmed on Layer 1. -pub async fn tx_on_chain(client: &BTCClient, txid: &Txid) -> Result { - match client.get_tx(txid).await? { - Some(_) => Ok(true), - _ => Ok(false), - } -} - -pub async fn tx_confirmed(client: &BTCClient, txid: &Txid) -> Result { - Ok(client.get_tx_status(txid).await?.confirmed) -} - -pub async fn outpoint_available(client: &BTCClient, txid: &Txid, vout: u64) -> Result { - match client.get_output_status(txid, vout).await? { - Some(status) => Ok(!status.spent), - _ => Ok(false), - } -} - -pub async fn outpoint_spent_txid( - client: &BTCClient, - txid: &Txid, - vout: u64, -) -> Result> { - match client.get_output_status(txid, vout).await? { - Some(status) => Ok(status.txid), - _ => Ok(None), - } -} - -pub async fn outpoint_spent_txin( - client: &BTCClient, - txid: &Txid, - vout: u64, -) -> Result> { - match client.get_output_status(txid, vout).await? { - Some(status) => { - if let Some(spent_txid) = status.txid - && let Some(vin) = status.vin - && let Some(spent_tx) = client.get_tx(&spent_txid).await? - { - Ok(spent_tx.input.get(vin as usize).cloned().map(|txin| (spent_txid, vin, txin))) - } else { - Ok(None) - } - } - _ => Ok(None), - } -} - -pub async fn upsert_message( - storage_processor: &mut StorageProcessor<'_>, - is_update: bool, - from_peer: String, - actor: Actor, - message_content: GOATMessageContent, - weight: i64, - lock_time: i64, -) -> Result { - let key = LocalMessageKey::from_content(actor.clone(), &message_content)?; - let business_id = key.business_id(); - let message_id = key.message_id(); - let msg_type = message_content.kind(); - let message = GOATMessage::new(actor.clone(), message_content.clone()); - if is_update || storage_processor.find_messages_by_id(&message_id).await?.is_none() { - if let Some(cancel_msg_type) = match msg_type { - MessageKind::AssertSent => Some(MessageKind::WatchtowerChallengeInitSent), - _ => None, - } { - notify_to_cancel_proof_task(storage_processor, business_id, cancel_msg_type).await?; - } - - return storage_processor - .upsert_message(Message { - message_id, - business_id, - actor: actor.to_string(), - from_peer, - msg_type: msg_type.to_string(), - content: message.serialize_message().await?, - weight, - lock_time_until: current_time_secs() + lock_time, - state: MessageState::Pending.to_string(), - message_version: 0, - attempt_count: 0, - abandon_count: 0, - last_error: None, - created_at: 0, - }) - .await; - } else { - info!("{message_id} is already created for create action"); - } - - Ok(false) -} - -pub async fn notify_to_cancel_proof_task( - storage_processor: &mut StorageProcessor<'_>, - business_id: Uuid, - msg_type: MessageKind, -) -> Result<()> { - // AssertInitSent is removed; update related logic if needed; - if !matches!(msg_type, MessageKind::WatchtowerChallengeInitSent) { - warn!("notify_to_cancel_proof_task: input wrong message type:{msg_type}"); - return Ok(()); - } - if get_actor() != Actor::Watchtower { - return Ok(()); - } - - let host = match get_proof_build_rpc_host() { - Some(host) => Url::parse(&host)?, - None => { - warn!("notify_to_cancel_proof_task:failed to get proof_build_rpc_host"); - return Ok(()); - } - }; - - if let Some(message) = - storage_processor.find_message_by_business_id(&business_id, &msg_type.to_string()).await? - && let Some(graph) = storage_processor.find_graph(&business_id).await? - { - if !matches!(message.state.as_str(), "Pending" | "Processing") { - warn!( - "message {business_id}, msg_type: {msg_type} no need to cancel.as state is {}", - message.state - ); - return Ok(()); - } - - // It will only be called a few times under limited conditions, so we just create a new object - let http_client = HttpAsyncClient::new(None); - let notify_result = match msg_type { - MessageKind::WatchtowerChallengeInitSent => { - let url = host.join(PROOFS_WATCHTOWER_PROOF_TIMEOUT)?; - let payload = WatchtowerProofTimeoutUpdateRequest { - instance_id: graph.instance_id.to_string(), - graph_id: graph.graph_id.to_string(), - gateway_address: Some(env::get_goat_gateway_contract_from_env().to_string()), - public_key: get_node_pubkey()?.to_string(), - }; - let auth_keypair = get_bitvm_key()?; - let response = http_client - .post_response_json_with_dynamic_headers::( - url.as_str(), - &payload, - || { - proof_builder_auth_headers( - &auth_keypair, - ProofBuilderAuthRole::Watchtower, - PROOFS_WATCHTOWER_PROOF_TIMEOUT, - &payload, - ) - }, - ) - .await?; - info!("call {}, response:{:?}", PROOFS_WATCHTOWER_PROOF_TIMEOUT, response); - response.data.is_some() - } - _ => false, - }; - if notify_result { - // cancel unfinished p2p message; when notify success! - storage_processor - .cancel_messages_by_business_id(&business_id, Some(msg_type.to_string())) - .await?; - } - } else { - warn!("message {business_id}, msg_type: {msg_type} or graph {business_id} not fund in DB"); - } - - Ok(()) -} - -/// store new graph, graph_raw_data, and update instance_id -pub async fn get_bitvm_graph_from_db( - _local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result { - Err(anyhow!("graph:{graph_id} not found")) -} - -pub async fn get_graph_status( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let mut storage_process = local_db.acquire().await?; - let graph_op = storage_process.find_graph(&graph_id).await?; - if graph_op.is_none() { - return Ok(None); - }; - let graph = graph_op.unwrap(); - if graph.instance_id.ne(&instance_id) { - bail!( - "grap with graph_id:{graph_id} has instance_id:{} not match exp instance:{instance_id}", - graph.instance_id, - ); - } - Ok(Some( - GraphStatus::from_str(&graph.status) - .map_err(|_| anyhow!("unknown graph status: {}", graph.status))?, - )) -} - -/// Returns: -/// - `Ok(true)` tx confirmed, -/// - `Ok(false)` tx not confirmed, exceeds the maximum waiting time -pub async fn wait_tx_confirmation( - btc_client: &BTCClient, - txid: &Txid, - interval: u64, - max_wait_secs: u64, -) -> Result { - use std::{ - thread, - time::{Duration, Instant}, - }; - let start_time = Instant::now(); - loop { - if start_time.elapsed().as_secs() > max_wait_secs { - // println!("Timeout: Transaction not confirmed after {} seconds", max_wait_secs); - return Ok(false); - }; - // FIXME: should not use esplora directly - match btc_client.get_tx_status(txid).await { - Ok(status) => { - if let Some(_height) = status.block_height { - // println!("Transaction confirmed in block {}", height); - return Ok(true); - } else { - // println!("Transaction unconfirmed, polling again..."); - } - } - Err(e) => { - bail!("Failed to fetch transaction status: {e}"); - } - } - thread::sleep(Duration::from_secs(interval)); - } -} - -#[allow(dead_code)] -pub async fn wait_tx_appear( - btc_client: &BTCClient, - txid: &Txid, - interval: u64, - max_wait_secs: u64, -) -> Result { - use std::{ - thread, - time::{Duration, Instant}, - }; - let start_time = Instant::now(); - loop { - if start_time.elapsed().as_secs() > max_wait_secs { - // println!("Timeout: Transaction not appear after {} seconds", max_wait_secs); - return Ok(false); - }; - match btc_client.get_tx(txid).await { - Ok(tx) => { - if tx.is_some() { - return Ok(true); - } - } - Err(e) => { - bail!("Failed to fetch transaction status: {e}"); - } - } - thread::sleep(Duration::from_secs(interval)); - } -} - -pub mod defer { - pub struct Defer { - cleanup: Option, - } - impl Defer { - pub fn new(f: F) -> Self { - Self { cleanup: Some(f) } - } - pub fn dismiss(&mut self) { - self.cleanup = None; - } - } - impl Drop for Defer { - fn drop(&mut self) { - if let Some(cleanup) = self.cleanup.take() { - cleanup(); - } - } - } - #[macro_export] - macro_rules! defer { - ($name:ident, $cleanup:block) => { - let mut $name = $crate::utils::defer::Defer::new(|| $cleanup); - }; - } - #[macro_export] - macro_rules! dismiss_defer { - ($name:ident) => { - $name.dismiss(); - }; - } -} - -/// Upper bound for the operator-supplied node name, which `GET /v1/nodes` -/// returns verbatim. -const MAX_NODE_NAME_LEN: usize = 64; - -/// Upper bound for the advertised `host:port` string. -const MAX_SOCKET_ADDR_LEN: usize = 128; - -/// Every `NodeInfo` field is self-declared by the sender, yet the node registry -/// is keyed by `peer_id`. The one identity a peer cannot forge is the gossipsub -/// message source: the swarm runs with `MessageAuthenticity::Signed` and strict -/// validation, so `message.source` is bound to the sender's libp2p key. Key the -/// registry row on that identity and never on the `peer_id` inside the payload, -/// otherwise any peer can overwrite any other node's record. -pub fn node_info_matches_sender(node_info: &NodeInfo, from_peer_id: &PeerId) -> bool { - node_info.peer_id == from_peer_id.to_string() -} - -/// Reject payloads that cannot be persisted safely. These fields are served -/// verbatim by `GET /v1/nodes` and drive on-chain lookups and reward -/// attribution in `node_available_pbtc_update_monitor` and `add_node_reward`. -pub fn validate_node_info_payload(node_info: &NodeInfo) -> std::result::Result<(), String> { - if PeerId::from_str(&node_info.peer_id).is_err() { - return Err(format!("invalid peer_id {}", node_info.peer_id)); - } - if Actor::from_str(&node_info.actor).is_err() { - return Err(format!("invalid actor {}", node_info.actor)); - } - if !node_info.goat_addr.is_empty() && EvmAddress::from_str(&node_info.goat_addr).is_err() { - return Err(format!("invalid goat_addr {}", node_info.goat_addr)); - } - if !node_info.btc_pub_key.is_empty() && PublicKey::from_str(&node_info.btc_pub_key).is_err() { - return Err(format!("invalid btc_pub_key {}", node_info.btc_pub_key)); - } - if !node_info.available_peg_btc.is_empty() - && alloy::primitives::U256::from_str(&node_info.available_peg_btc).is_err() - { - return Err(format!("invalid available_peg_btc {}", node_info.available_peg_btc)); - } - if !node_info.service_fee_rate.is_finite() || !(0.0..=1.0).contains(&node_info.service_fee_rate) - { - return Err(format!("invalid service_fee_rate {}", node_info.service_fee_rate)); - } - if node_info.node_name.chars().count() > MAX_NODE_NAME_LEN - || node_info.node_name.chars().any(char::is_control) - { - return Err(format!("invalid node_name of {} chars", node_info.node_name.chars().count())); - } - validate_node_socket_addr(&node_info.socket_addr) -} - -/// The advertised socket address is only ever displayed and dialed by clients, -/// so require a plain `host:port` string that cannot smuggle control characters -/// into API responses. -fn validate_node_socket_addr(socket_addr: &str) -> std::result::Result<(), String> { - if socket_addr.is_empty() { - return Ok(()); - } - if socket_addr.chars().count() > MAX_SOCKET_ADDR_LEN - || socket_addr.chars().any(|c| c.is_control() || c.is_whitespace()) - { - return Err(format!("invalid socket_addr of {} chars", socket_addr.chars().count())); - } - let Some((host, port)) = socket_addr.rsplit_once(':') else { - return Err(format!("socket_addr {socket_addr} is missing a port")); - }; - if host.is_empty() || port.parse::().is_err() { - return Err(format!("socket_addr {socket_addr} is not a valid host:port")); - } - Ok(()) -} - -pub async fn save_node_info(local_db: &LocalDB, node_info: &NodeInfo) -> Result<()> { - // Reachable once per announcement from any peer: not louder than debug. - tracing::debug!("save_node_info for {}", node_info.peer_id); - let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; - let mut storage_process = local_db.acquire().await?; - storage_process - .upsert_node(&Node { - peer_id: node_info.peer_id.clone(), - actor: node_info.actor.clone(), - node_name: node_info.node_name.clone(), - goat_addr: node_info.goat_addr.clone(), - btc_pub_key: node_info.btc_pub_key.clone(), - socket_addr: node_info.socket_addr.clone(), - reward: "0".to_string(), - service_fee_rate: node_info.service_fee_rate, - available_peg_btc: node_info.available_peg_btc.clone(), - binding_sig: node_info.binding_sig.clone(), - binding_issued_at: node_info.binding_issued_at, - updated_at: current_time, - created_at: current_time, - }) - .await?; - Ok(()) -} - -pub async fn save_local_info(local_db: &LocalDB) { - let node = get_local_node_info(); - // Peers apply the same validation to the NodeInfo this node broadcasts, so a - // malformed local config would silently drop this node from every peer's - // registry. Surface it here instead, where the operator reads the logs. - if let Err(reason) = validate_node_info_payload(&node) { - tracing::error!( - "local node info will be rejected by peers: {reason}; fix the node configuration" - ); - } - match save_node_info(local_db, &node).await { - Ok(_) => {} - Err(err) => tracing::error!("save local node err: {err}"), - } -} - -pub async fn update_node_timestamp(local_db: &LocalDB, peer_id: &str) -> Result<()> { - tracing::debug!("update timestamp for {peer_id}"); - let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; - let mut storage_process = local_db.acquire().await?; - match storage_process.update_node_timestamp(peer_id, current_time).await { - Ok(_) => {} - Err(err) => warn!("{err}"), - }; - Ok(()) -} - -pub async fn detect_heart_beat(swarm: &mut Swarm) -> Result<()> { - tracing::info!("start detect_heart_beat"); - let message_content = GOATMessageContent::RequestNodeInfo(get_local_node_info()); - // send to actor - let actors = get_rpc_support_actors(); - for actor in actors { - match send_to_peer(swarm, GOATMessage::new(actor, message_content.clone())).await { - Ok(_) => {} - Err(err) => warn!("{err}"), - } - } - Ok(()) -} - -pub fn generate_random_bytes(len: usize) -> Vec { - let mut rng = rand::thread_rng(); - (0..len).map(|_| rng.gen_range(0..255)).collect() -} - -pub fn get_rand_btc_address_p2wpkh(network: Network) -> String { - let secp = Secp256k1::new(); - Address::p2wpkh( - &CompressedPublicKey::try_from(PrivateKey::generate(network).public_key(&secp)) - .expect("Could not compress public key"), - network, - ) - .to_string() -} - -pub fn get_rand_btc_address_p2pkh(network: Network) -> String { - let secp = Secp256k1::new(); - Address::p2pkh( - CompressedPublicKey::try_from(PrivateKey::generate(network).public_key(&secp)) - .expect("Could not compress public key"), - network, - ) - .to_string() -} - -pub fn get_rand_goat_address() -> String { - EvmAddress::from_slice(&generate_random_bytes(20)).to_string() -} - -pub fn strip_hex_prefix_owned(s: &str) -> String { - if s.starts_with("0x") || s.starts_with("0X") { s[2..].to_string() } else { s.to_string() } -} - -/// Retrieve the server's public IP via NAT protocol and combine it with -/// the configured RPC monitoring port`rpc_addr` to generate the external RPC service address. -pub async fn set_node_external_socket_addr_env(rpc_addr: &str) -> Result<()> { - if get_proof_server_url().is_some() || std::env::var(ENV_EXTERNAL_SOCKET_ADDR).is_ok() { - // not provide proof server - return Ok(()); - } - let addr = SocketAddr::from_str(rpc_addr)?; - let mut client = Client::new("0.0.0.0:0", None).await?; - let message_res = client.binding_request("stun.l.google.com:19302", None).await; - if message_res.is_err() { - warn!("fail to get message from stun.l.google.com:19302, err :{:?}", message_res.err()); - return Ok(()); - } - let message = message_res?; - if message.get_class() != Class::SuccessResponse { - warn!( - "fail to get message from stun.l.google.com:19302, return class :{:?}", - message.get_class() - ); - return Ok(()); - } - if let Some(socket_addr) = Attribute::get_xor_mapped_address(&message) { - unsafe { - std::env::set_var( - ENV_EXTERNAL_SOCKET_ADDR, - SocketAddr::new(socket_addr.ip(), addr.port()).to_string(), - ); - } - } - Ok(()) -} - -pub fn reflect_goat_address(addr_op: Option) -> (bool, Option) { - if let Some(addr) = addr_op - && let Ok(addr) = EvmAddress::from_str(&addr) - { - return (true, Some(addr.to_string())); - } - (false, None) -} - -/// Sweep the local queue and list the messages the dispatcher may attempt now. -/// -/// Returns `(candidates, quarantined)`. Nothing returned here is claimed yet: -/// the dispatcher claims each row immediately before dispatching it, so a -/// crash mid-dispatch is charged to that one row rather than to the whole -/// batch. Expired rows are retired and exhausted rows quarantined first. -pub async fn list_batch_local_msg( - local_db: &LocalDB, - max_abandons: i64, - limit: i64, -) -> Result<(Vec, u64)> { - let mut tx = local_db.start_transaction().await?; - let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; - let expired_before = current_time - MESSAGE_EXPIRE_TIME; - tx.set_messages_expired(expired_before).await?; - tx.delete_old_messages(expired_before).await?; - let quarantined = tx.quarantine_local_messages(current_time, max_abandons).await?; - let messages = - tx.list_claimable_local_messages(current_time, expired_before, limit, max_abandons).await?; - tx.commit().await?; - Ok((messages, quarantined)) -} - -pub async fn operator_scan_ready_proof( - _local_db: &LocalDB, - _remote_proof_server_socket: Option, - _uri: &str, -) -> Result<()> { - tracing::info!("start operator_scan_ready_proof"); - // todo - Ok(()) -} - -pub fn generate_local_key() -> libp2p::identity::Keypair { - libp2p::identity::Keypair::generate_ed25519() -} - -pub fn temp_sqlite_db_path() -> String { - let tmp_db = tempfile::NamedTempFile::new().unwrap(); - format!("sqlite:{}", tmp_db.path().as_os_str().to_str().unwrap()) -} - -// contract calls - -#[derive(Clone, Serialize, Deserialize, Debug, Default)] -pub struct InstanceProcessDataItem { - pub pub_nonce: Option, - pub agg_nonce_consensus_hash: Option<[u8; 32]>, - pub agg_nonce_consensus_signature: Option, - pub partial_sign: Option, - pub endorse_signature: Vec, -} -pub type InstanceProcessDataMap = IndexMap; - -#[derive(Clone, Serialize, Deserialize, Default)] -pub struct GraphProcessDataItem { - pub committee_pub_nonce: Option, - pub agg_nonce_consensus_hash: Option<[u8; 32]>, - pub agg_nonce_consensus_signature: Option, - pub partial_sigs: Option, - pub committee_evm_address: Option, - pub endorse_signature: Vec, - pub params_endorse_signature: Vec, - pub verifier_index: Option, - pub verifier_peer_id: Option, - pub verifier_params_signature: Option, -} -pub type GraphProcessDataMap = IndexMap; - -pub fn order_committee_values( - committee_pubkeys: &[PublicKey], - values: Vec<(PublicKey, T)>, - label: &str, -) -> Result> { - for (idx, (pubkey, _)) in values.iter().enumerate() { - if !committee_pubkeys.contains(pubkey) { - bail!("{label} contains non-committee pubkey {pubkey}"); - } - if values.iter().skip(idx + 1).any(|(other, _)| other == pubkey) { - bail!("{label} contains duplicate pubkey {pubkey}"); - } - } - - committee_pubkeys - .iter() - .map(|committee_pubkey| { - values - .iter() - .find(|(pubkey, _)| pubkey == committee_pubkey) - .map(|(_, value)| value.clone()) - .ok_or_else(|| anyhow!("{label} missing pubkey {committee_pubkey}")) - }) - .collect() -} - -// db operations -pub async fn get_current_prekickoff_tx( - local_db: &LocalDB, - operator_pubkey: &PublicKey, -) -> Result> { - // return (latest_graph.nonce + 1 , latest_graph.next_prekickoff_tx) - // return None if no graph yet - let mut storage_processor = local_db.acquire().await?; - let graphs = storage_processor - .get_operator_graphs( - GraphQuery::default() - .with_operator_pubkey(operator_pubkey.to_string()) - .with_order("kickoff_index DESC".to_string()) - .with_limit(1), - ) - .await?; - - if let Some(graph) = graphs.first() - && let Some(simplified_graph) = load_validated_graph_definition( - &mut storage_processor, - graph.instance_id, - graph.graph_id, - ) - .await? - { - Ok(Some(( - (graph.kickoff_index + 1) as u64, - BitvmGcGraph::from_simplified(&simplified_graph)?.next_prekickoff, - ))) - } else { - Ok(None) - } -} - -pub struct GenerateInstanceParams { - pub instance_id: Uuid, - pub user_info: UserInfo, - pub pegin_amount: Amount, - pub pegin_request_tx_hash: String, - pub pegin_request_height: i64, - pub pegin_timestamp: i64, -} -pub async fn generate_instance( - btc_client: &BTCClient, - params: GenerateInstanceParams, -) -> Result { - let from_addr = if !params.user_info.inputs.is_empty() - && let Some(tx) = btc_client.get_tx(¶ms.user_info.inputs[0].outpoint.txid).await? - { - let tx_scripts = - tx.output[params.user_info.inputs[0].outpoint.vout as usize].script_pubkey.clone(); - Address::from_script(&tx_scripts, env::get_network()) - .map(|addr| addr.to_string()) - .unwrap_or_default() - } else { - warn!( - "failed to decode instance {} from_address from pegin_request as input_utxos is empty or decode address failed", - params.instance_id - ); - "".to_string() - }; - let input_utxos = params - .user_info - .inputs - .iter() - .map(|input| ClientUtxo { - txid: input.outpoint.txid.to_byte_array(), - vout: input.outpoint.vout, - amount_sats: input.amount.to_sat(), - }) - .collect::>(); - let current_time = current_time_secs(); - - Ok(Instance { - instance_id: params.instance_id, - network: get_network().to_string(), - from_addr, - to_addr: EvmAddress::from(¶ms.user_info.depositor_evm_address).to_string(), - amount: params.pegin_amount.to_sat() as i64, - fees: UInt64Array3(params.user_info.txn_fees), - input_utxos: serde_json::to_string(&input_utxos)?, - status: InstanceBridgeInStatus::UserInited.to_string(), - goat_tx_hash: params.pegin_request_tx_hash, - goat_tx_height: params.pegin_request_height, - user_xonly_pubkey: ByteArray32(params.user_info.user_xonly_pubkey.clone().serialize()), - user_change_addr: params.user_info.user_change_address.clone().to_string(), - user_refund_addr: params.user_info.user_refund_address.clone().to_string(), - btc_txid: None, - pegin_confirm_txid: None, - pegin_cancel_txid: None, - committees_answers: IndexMap::new(), - pegin_data_tx_hash: "".to_string(), - btc_height: 0, - parameters: None, - post_pegin_txhash: None, - status_updated_at: params.pegin_timestamp, - created_at: current_time, - updated_at: current_time, - }) -} - -/// Store the pegin request against the local instance row. -/// -/// Returns: -/// - Ok(true) if the instance was created, or refreshed while still in a -/// pegin-request stage -/// - Ok(false) if the request was not applied because the instance has moved -/// past the pegin-request stage, or the id belongs to a bridge-out instance -pub async fn store_pegin_request( - btc_client: &BTCClient, - local_db: &LocalDB, - params: GenerateInstanceParams, -) -> Result { - // store instance info to local db - let instance_id = params.instance_id; - let mut storage_processor = local_db.acquire().await?; - let instance = generate_instance(btc_client, params).await?; - // A PeginRequest only initializes an instance. Guarding the write on the - // pre-pegin statuses keeps a replayed or forged request from rolling a live - // instance back to UserInited and clearing what later stages stored. - let stored = storage_processor - .upsert_pegin_request_instance( - &instance, - &[ - InstanceBridgeInStatus::UserIniting.to_string(), - InstanceBridgeInStatus::UserInited.to_string(), - ], - ) - .await?; - if !stored { - warn!( - "ignore PeginRequest for instance {instance_id}: it is not a bridge-in instance in a pegin-request stage" - ); - return Ok(false); - } - info!("stored PeginRequest for instance {instance_id}"); - Ok(true) -} - -pub async fn store_instance_parameters( - local_db: &LocalDB, - instance_params: &BitvmGcInstanceParameters, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let incoming_parameters_hash = instance_params.parameters_hash()?; - if let Some(existing_parameters) = - storage_processor.get_instance_parameters_by_id(&instance_params.instance_id).await? - { - let existing_instance_params: BitvmGcInstanceParameters = - serde_json::from_str(&existing_parameters)?; - let existing_parameters_hash = existing_instance_params.parameters_hash()?; - if existing_parameters_hash != incoming_parameters_hash { - bail!(SpecialError::InvalidPeginData(format!( - "instance parameters changed for instance_id {}: existing={}, incoming={}", - instance_params.instance_id, - hex::encode(existing_parameters_hash), - hex::encode(incoming_parameters_hash) - ))); - } - } - storage_processor - .update_instance_parameters( - &instance_params.instance_id, - &serde_json::to_string(&instance_params)?, - ) - .await?; - Ok(()) -} -pub async fn get_instance_parameters( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let Some(instance) = storage_processor.find_instance(&instance_id).await? else { - return Ok(None); - }; - - if let Some(parameters) = instance.parameters { - return Ok(Some(serde_json::from_str(¶meters)?)); - } - - Ok(Some( - gen_instance_parameters_local(&instance).with_context(|| { - format!("failed to reconstruct parameters for instance {instance_id}") - })?, - )) -} - -fn convert_graph( - bitvm_graph: &BitvmGcGraph, - current_time: i64, - initial_status: GraphStatus, - definition_hash: String, -) -> Graph { - Graph { - graph_id: bitvm_graph.parameters.graph_id, - instance_id: bitvm_graph.parameters.instance_parameters.instance_id, - kickoff_index: bitvm_graph.parameters.graph_nonce as i64, - from_addr: "".to_string(), - to_addr: "".to_string(), - amount: bitvm_graph.parameters.instance_parameters.pegin_amount.to_sat() as i64, - challenge_amount: bitvm_graph.parameters.challenge_amount.to_sat() as i64, - status: initial_status.to_string(), - sub_status: "".to_string(), - operator_pubkey: bitvm_graph.parameters.operator_pubkey.to_string(), - definition_hash, - cur_prekickoff_txid: Some(bitvm_graph.cur_prekickoff.finalize().compute_txid().into()), - next_prekickoff: Some(bitvm_graph.next_prekickoff.finalize().compute_txid().into()), - force_skip_kickoff_txid: Some( - bitvm_graph.force_skip_kickoff.finalize().compute_txid().into(), - ), - quick_challenge_txid: Some(bitvm_graph.quick_challenge.finalize().compute_txid().into()), - challenge_incomplete_kickoff_txid: Some( - bitvm_graph.challenge_incomplete_kickoff.finalize().compute_txid().into(), - ), - pegin_txid: Some(bitvm_graph.pegin.finalize().compute_txid().into()), - kickoff_txid: Some(bitvm_graph.kickoff.finalize().compute_txid().into()), - take1_txid: Some(bitvm_graph.take1.finalize().compute_txid().into()), - challenge_txid: None, - take2_txid: Some(bitvm_graph.take2.finalize().compute_txid().into()), - watchtower_challenge_init_txid: Some( - bitvm_graph.watchtower_challenge_init.finalize().compute_txid().into(), - ), - operator_assert_txid: Some(bitvm_graph.operator_assert.finalize().compute_txid().into()), - verifier_assert_txids: bitvm_graph - .verifier_asserts - .iter() - .map(|tx| tx.finalize().compute_txid().into()) - .collect(), - disprove_txids: bitvm_graph - .disproves - .iter() - .map(|tx| tx.finalize().compute_txid().into()) - .collect(), - watchtower_challenge_timeout_txids: bitvm_graph - .watchtower_challenge_timeouts - .iter() - .map(|tx| tx.finalize().compute_txid().into()) - .collect(), - operator_challenge_nack_txids: bitvm_graph - .operator_challenge_nacks - .iter() - .map(|tx| tx.finalize().compute_txid().into()) - .collect(), - operator_commit_timeout_txid: Some( - bitvm_graph.operator_commit_timeout.finalize().compute_txid().into(), - ), - init_withdraw_tx_hash: None, - bridge_out_start_at: 0, - status_updated_at: current_time, - proceed_withdraw_height: 0, - created_at: current_time, - updated_at: current_time, - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum FinalizedGraphStoreOutcome { - NewlyStored, - DefinitionUpgraded, - AlreadyFinalized, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum GraphDefinitionIngestKind { - OperatorPresigned, - Finalized, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum GraphDefinitionIngestOutcome { - Inserted, - FinalizedUpgrade, - Replay, - AlreadyFinalized, -} - -/// Store a verified operator-pre-signed graph definition. -/// -/// This path is intentionally unable to advance `GraphStatus`: receiving a -/// CreateGraph message is not evidence of committee finalization. -pub(crate) async fn store_operator_presigned_graph( - local_db: &LocalDB, - simple_graph: &SimplifiedBitvmGcGraph, -) -> Result<()> { - if !simple_graph.operator_pre_signed() || simple_graph.committee_pre_signed() { - bail!(SpecialError::InvalidGraph(format!( - "graph {} is not an operator-pre-signed proposal", - simple_graph.parameters.graph_id - ))); - } - - // The definition path reads existing state before updating it. Reserve the - // writer up front so another task cannot invalidate that read snapshot. - let mut tx = local_db.start_immediate_transaction().await?; - ingest_graph_definition(&mut tx, simple_graph, GraphDefinitionIngestKind::OperatorPresigned) - .await?; - tx.commit().await?; - Ok(()) -} - -/// Store a verified finalized graph once without replacing a graph whose -/// finalized form is already known. -pub(crate) async fn store_finalized_graph_if_needed( - local_db: &LocalDB, - simple_graph: &SimplifiedBitvmGcGraph, -) -> Result { - let graph_id = simple_graph.parameters.graph_id; - if !simple_graph.operator_pre_signed() || !simple_graph.committee_pre_signed() { - bail!(SpecialError::InvalidGraph(format!("graph {graph_id} is not fully pre-signed"))); - } - - // The definition path reads existing state before updating it. Reserve the - // writer up front so another task cannot invalidate that read snapshot. - let mut tx = local_db.start_immediate_transaction().await?; - let outcome = - ingest_graph_definition(&mut tx, simple_graph, GraphDefinitionIngestKind::Finalized) - .await?; - tx.commit().await?; - - Ok(match outcome { - GraphDefinitionIngestOutcome::Inserted => FinalizedGraphStoreOutcome::NewlyStored, - GraphDefinitionIngestOutcome::FinalizedUpgrade => { - FinalizedGraphStoreOutcome::DefinitionUpgraded - } - GraphDefinitionIngestOutcome::AlreadyFinalized => { - FinalizedGraphStoreOutcome::AlreadyFinalized - } - GraphDefinitionIngestOutcome::Replay => { - unreachable!("a finalized graph replay must be classified as already finalized") - } - }) -} - -async fn ingest_graph_definition( - tx: &mut StorageProcessor<'_>, - simple_graph: &SimplifiedBitvmGcGraph, - kind: GraphDefinitionIngestKind, -) -> Result { - let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(simple_graph)?; - let graph_id = simple_graph.parameters.graph_id; - verify_graph_operator_pre_signatures(&bitvm_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "graph {graph_id} has invalid operator pre-signatures: {error}" - )) - })?; - if kind == GraphDefinitionIngestKind::Finalized { - verify_graph_committee_pre_signatures(&bitvm_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "graph {graph_id} has invalid committee pre-signatures: {error}" - )) - })?; - } - let current_time = current_time_secs(); - let definition_hash = hex::encode(simple_graph.parameters_hash()?); - let existing_graph_row = tx.find_graph(&graph_id).await?; - let existing_raw_data = tx.find_graph_raw_data(&graph_id).await?; - let existing_graph = match (existing_graph_row.as_ref(), existing_raw_data) { - (None, None) => None, - (Some(_), None) => { - bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} has runtime data but no raw definition; explicit repair is required" - ))); - } - (None, Some(_)) => { - bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} has raw definition but no graph row; explicit repair is required" - ))); - } - (Some(existing_row), Some(existing_raw_data)) => { - if existing_row.definition_hash.is_empty() { - bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} is missing its stored definition hash; explicit repair is required" - ))); - } - if existing_row.definition_hash != definition_hash { - bail!(SpecialError::InvalidGraph(format!( - "graph parameters changed for graph_id {graph_id}: existing={}, incoming={definition_hash}", - existing_row.definition_hash - ))); - } - - let existing_graph = parse_graph_raw_data(existing_raw_data.raw_data, graph_id).await?; - let existing_raw_hash = hex::encode(existing_graph.parameters_hash()?); - if existing_raw_hash != existing_row.definition_hash { - bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} definition hash does not match its stored raw definition" - ))); - } - let existing_full_graph = BitvmGcGraph::from_simplified(&existing_graph)?; - verify_graph_operator_pre_signatures(&existing_full_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "stored graph {graph_id} has invalid operator pre-signatures: {error}" - )) - })?; - if existing_full_graph.committee_pre_signed() { - verify_graph_committee_pre_signatures(&existing_full_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "stored graph {graph_id} has invalid committee pre-signatures: {error}" - )) - })?; - } - Some(existing_graph) - } - }; - - let existing_is_finalized = - existing_graph.as_ref().is_some_and(SimplifiedBitvmGcGraph::committee_pre_signed); - if existing_is_finalized { - let existing_graph = existing_graph.as_ref().expect("checked above"); - if kind == GraphDefinitionIngestKind::Finalized && existing_graph != simple_graph { - bail!(SpecialError::InvalidGraph(format!( - "conflicting finalized graph for graph_id {graph_id}" - ))); - } - - // An old CreateGraph replay is harmless once a verified finalized - // graph is stored. Do not replace its raw signatures or runtime. - if kind == GraphDefinitionIngestKind::Finalized { - confirm_finalized_graph_definition( - tx, - bitvm_graph.parameters.instance_parameters.instance_id, - graph_id, - ) - .await?; - } - return Ok(GraphDefinitionIngestOutcome::AlreadyFinalized); - } - - if kind == GraphDefinitionIngestKind::OperatorPresigned - && let Some(existing_graph) = existing_graph.as_ref() - && existing_graph != simple_graph - { - bail!(SpecialError::InvalidGraph(format!( - "conflicting operator-pre-signed graph for graph_id {graph_id}" - ))); - } - if kind == GraphDefinitionIngestKind::Finalized - && let Some(existing_graph) = existing_graph.as_ref() - && existing_graph.operator_pre_sigs != simple_graph.operator_pre_sigs - { - bail!(SpecialError::InvalidGraph(format!( - "finalized graph {graph_id} changes the stored operator pre-signatures" - ))); - } - - let mut graph = convert_graph( - &bitvm_graph, - current_time, - // A graph row is always created at the operator-presigned baseline. - // Only a verified finalized definition may advance it through the - // dedicated Definition transition below. - GraphStatus::OperatorPresigned, - definition_hash.clone(), - ); - - if let Some(node_info) = - tx.get_node_by_btc_pub_key(&bitvm_graph.parameters.operator_pubkey.to_string()).await? - { - graph.from_addr = node_info.goat_addr.clone(); - graph.to_addr = - node_p2wsh_address(get_network(), &bitvm_graph.parameters.operator_pubkey).to_string(); - } - - tx.upsert_graph_definition(&graph).await?; - - let outcome = if existing_graph.is_none() { - let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; - tx.upsert_graph_raw_data( - bitvm_graph.parameters.instance_parameters.instance_id, - GraphRawData { graph_id, raw_data, created_at: current_time, updated_at: current_time }, - &definition_hash, - ) - .await?; - if kind == GraphDefinitionIngestKind::Finalized { - confirm_finalized_graph_definition( - tx, - bitvm_graph.parameters.instance_parameters.instance_id, - graph_id, - ) - .await?; - } - GraphDefinitionIngestOutcome::Inserted - } else if kind == GraphDefinitionIngestKind::Finalized { - let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; - tx.upsert_graph_raw_data( - bitvm_graph.parameters.instance_parameters.instance_id, - GraphRawData { graph_id, raw_data, created_at: current_time, updated_at: current_time }, - &definition_hash, - ) - .await?; - - // Only a verified finalized definition can advance an existing locally - // operator-pre-signed graph. A later chain-observed state is retained. - confirm_finalized_graph_definition( - tx, - bitvm_graph.parameters.instance_parameters.instance_id, - graph_id, - ) - .await?; - GraphDefinitionIngestOutcome::FinalizedUpgrade - } else { - GraphDefinitionIngestOutcome::Replay - }; - - Ok(outcome) -} - -/// Confirm that a fully verified graph definition has reached the committee -/// pre-signing stage without allowing it to overwrite a later runtime state. -async fn confirm_finalized_graph_definition( - tx: &mut StorageProcessor<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - match tx - .transition_graph_status( - instance_id, - graph_id, - GraphStatus::CommitteePresigned, - GraphStatusSource::Definition, - None, - ) - .await? - { - GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent => { - Ok(()) - } - GraphStatusTransitionOutcome::Rejected { current } => { - tracing::debug!( - "keep later graph status while storing finalized definition: graph={graph_id}, current={current}" - ); - Ok(()) - } - GraphStatusTransitionOutcome::NotFound => { - bail!("graph {graph_id} disappeared while confirming its finalized definition") - } - } -} - -/// Parse raw graph data JSON string to SimplifiedBitvmGcGraph using spawn_blocking -/// to handle large data and potential stack overflow issues -pub async fn parse_graph_raw_data( - raw_data: String, - graph_id: Uuid, -) -> Result { - let raw_data_len = raw_data.len(); - let raw_data_clone = raw_data.clone(); - let parse_result = tokio::task::spawn_blocking(move || { - serde_json::from_str::(&raw_data_clone) - }) - .await; - - match parse_result { - Ok(Ok(data)) => Ok(data), - Ok(Err(e)) => { - // Normal JSON parsing error - error!("Failed to parse graph data for graph_id {graph_id}: {e}"); - error!("Raw data length: {raw_data_len} bytes"); - Err(e.into()) - } - Err(join_err) => { - // spawn_blocking task failed (thread panic or task cancelled) - let msg = if join_err.is_panic() { - format!("Thread panic while parsing graph data for graph_id {graph_id}") - } else if join_err.is_cancelled() { - format!("Task cancelled while parsing graph data for graph_id {graph_id}") - } else { - format!( - "Task join error while parsing graph data for graph_id {graph_id}: {join_err}" - ) - }; - error!("{msg}"); - error!("Raw data length: {raw_data_len} bytes"); - Err(anyhow::anyhow!("{msg}")) - } - } -} - -/// Serialize SimplifiedBitvmGcGraph to JSON string using spawn_blocking -/// to handle large data and potential stack overflow issues -pub async fn serialize_graph_raw_data( - graph: &SimplifiedBitvmGcGraph, - graph_id: Uuid, -) -> Result { - let graph_clone = graph.clone(); - let serialize_result = - tokio::task::spawn_blocking(move || serde_json::to_string(&graph_clone)).await; - - match serialize_result { - Ok(Ok(data)) => Ok(data), - Ok(Err(e)) => { - // Normal JSON serialization error - error!("Failed to serialize graph data for graph_id {graph_id}: {e}"); - Err(e.into()) - } - Err(join_err) => { - // spawn_blocking task failed (thread panic or task cancelled) - let msg = if join_err.is_panic() { - format!("Thread panic while serializing graph data for graph_id {graph_id}") - } else if join_err.is_cancelled() { - format!("Task cancelled while serializing graph data for graph_id {graph_id}") - } else { - format!( - "Task join error while serializing graph data for graph_id {graph_id}: {join_err}", - ) - }; - error!("{msg}"); - Err(anyhow::anyhow!("{graph_id}")) - } - } -} - -/// Load a graph only after binding its runtime row, raw definition, canonical -/// parameters and pre-signatures together. Callers must use this instead of -/// parsing `graph_raw_data` directly. -pub(crate) async fn load_validated_graph_definition( - storage_processor: &mut StorageProcessor<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let Some(graph_row) = storage_processor.find_graph(&graph_id).await? else { - return Ok(None); - }; - if graph_row.instance_id != instance_id { - bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} belongs to instance {}, not {instance_id}", - graph_row.instance_id - ))); - } - if graph_row.definition_hash.is_empty() { - bail!(SpecialError::InvalidGraph(format!( - "graph {graph_id} is missing its definition hash; explicit repair is required" - ))); - } - - let graph_raw_data = storage_processor - .find_graph_raw_data(&graph_id) - .await? - .ok_or_else(|| { - SpecialError::InvalidGraph(format!( - "graph {graph_id} has a runtime row but no raw definition; explicit repair is required" - )) - })?; - let simplified_graph = parse_graph_raw_data(graph_raw_data.raw_data, graph_id).await?; - if simplified_graph.parameters.graph_id != graph_id - || simplified_graph.parameters.instance_parameters.instance_id != instance_id - { - bail!(SpecialError::InvalidGraph(format!( - "stored raw definition identifiers do not match graph {instance_id}:{graph_id}" - ))); - } - let definition_hash = hex::encode(simplified_graph.parameters_hash()?); - if definition_hash != graph_row.definition_hash { - bail!(SpecialError::InvalidGraph(format!( - "stored raw definition hash does not match graph row for {instance_id}:{graph_id}" - ))); - } - - let full_graph = BitvmGcGraph::from_simplified(&simplified_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "stored raw definition cannot rebuild graph for {instance_id}:{graph_id}: {error}" - )) - })?; - verify_graph_operator_pre_signatures(&full_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "stored raw definition has invalid operator pre-signatures for {instance_id}:{graph_id}: {error}" - )) - })?; - if full_graph.committee_pre_signed() { - verify_graph_committee_pre_signatures(&full_graph).map_err(|error| { - SpecialError::InvalidGraph(format!( - "stored raw definition has invalid committee pre-signatures for {instance_id}:{graph_id}: {error}" - )) - })?; - } - - Ok(Some(simplified_graph)) -} - -pub async fn get_graph( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - load_validated_graph_definition(&mut storage_processor, instance_id, graph_id).await -} - -pub async fn get_graph_by_instance_id_and_operator_pubkey( - local_db: &LocalDB, - instance_id: Uuid, - operator_pubkey: &PublicKey, -) -> Result> { - let mut storage_process = local_db.acquire().await?; - if let Some(graph_id) = storage_process - .get_graph_id_by_instance_id_and_operator_pubkey(&instance_id, &operator_pubkey.to_string()) - .await? - { - load_validated_graph_definition(&mut storage_process, instance_id, graph_id).await - } else { - Ok(None) - } -} - -pub async fn get_latest_pegout_finalized_graph( - local_db: &LocalDB, - operator_pubkey: &PublicKey, -) -> Result> { - // get latest pegout finalized graph nonce & id from local db - let statuses: Vec = - GraphStatus::get_closed_status().iter().map(|status| status.to_string()).collect(); - let mut storage_processor = local_db.acquire().await?; - let graphs = storage_processor - .get_operator_graphs( - GraphQuery::default() - .with_operator_pubkey(operator_pubkey.to_string()) - .with_statuses(statuses) - .with_order("kickoff_index DESC".to_string()) - .with_limit(1), - ) - .await?; - if graphs.is_empty() { - Ok(None) - } else { - Ok(Some((graphs[0].kickoff_index as u64, graphs[0].graph_id))) - } -} - -pub async fn get_graph_id_by_nonce( - local_db: &LocalDB, - graph_nonce: u64, - operator_pubkey: &PublicKey, -) -> Result> { - // get instance_id & graph_id by graph_nonce and operator_pubkey from local db - let mut storage_processor = local_db.acquire().await?; - let graphs = storage_processor - .get_operator_graphs( - GraphQuery::default() - .with_operator_pubkey(operator_pubkey.to_string()) - .with_kickoff_index(graph_nonce as i64) - .with_limit(1), - ) - .await?; - if graphs.is_empty() { Ok(None) } else { Ok(Some((graphs[0].instance_id, graphs[0].graph_id))) } -} - -pub async fn upsert_pegin_instance_process_data( - storage_processor: &mut StorageProcessor<'_>, - instance_id: Uuid, - process_data_map: &InstanceProcessDataMap, -) -> Result<()> { - let current_time = current_time_secs(); - storage_processor - .upsert_pegin_instance_process_data(&PeginInstanceProcessData { - instance_id, - process_data: serde_json::to_string(process_data_map)?, - updated_at: current_time, - created_at: current_time, - }) - .await?; - Ok(()) -} - -pub async fn find_pegin_instance_process_data( - storage_processor: &mut StorageProcessor<'_>, - instance_id: Uuid, -) -> Result { - if let Ok(Some(data)) = storage_processor.find_pegin_instance_process_data(&instance_id).await - && let Ok(process_data) = serde_json::from_str(data.process_data.as_str()) - { - Ok(process_data) - } else { - Ok(IndexMap::new()) - } -} - -pub async fn upsert_pegin_graph_process_data( - storage_processor: &mut StorageProcessor<'_>, - graph_id: Uuid, - instance_id: Uuid, - is_endorsed: bool, - process_data_map: &GraphProcessDataMap, -) -> Result<()> { - let current_time = current_time_secs(); - storage_processor - .upsert_pegin_graph_process_data(&PeginGraphProcessData { - graph_id, - instance_id, - is_endorsed, - process_data: serde_json::to_string(process_data_map)?, - updated_at: current_time, - created_at: current_time, - }) - .await?; - Ok(()) -} - -pub async fn find_pegin_graph_process_data( - storage_processor: &mut StorageProcessor<'_>, - graph_id: Uuid, -) -> Result<(bool, GraphProcessDataMap)> { - match storage_processor.find_pegin_graph_process_data(&graph_id).await? { - Some(data) => Ok((data.is_endorsed, serde_json::from_str(&data.process_data)?)), - None => Ok((false, IndexMap::new())), - } -} - -pub async fn store_committee_pub_nonces_for_graph( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - committee_pubkey: PublicKey, - pub_nonces: CommitteePubNonces, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (is_endorsed, mut process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - if let Some(existing_pub_nonces) = - process_data.get(&committee_pubkey).and_then(|v| v.committee_pub_nonce.as_ref()) - && existing_pub_nonces != &pub_nonces - { - bail!(SpecialError::InvalidGraph(format!( - "committee pub nonces changed for graph {graph_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|v| v.committee_pub_nonce = Some(pub_nonces.clone())) - .or_insert_with(|| GraphProcessDataItem { - committee_pub_nonce: Some(pub_nonces), - ..Default::default() - }); - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - is_endorsed, - &process_data, - ) - .await?; - Ok(()) -} -pub async fn get_committee_pub_nonces_for_graph( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| v.committee_pub_nonce.as_ref().map(|nonce| (*k, nonce.clone()))) - .collect::>()) -} -pub async fn store_committee_agg_nonce_consensus_for_graph( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - committee_pubkey: PublicKey, - consensus_hash: [u8; 32], - signature: SchnorrSignature, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (is_endorsed, mut process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - if let Some((existing_hash, existing_signature)) = process_data - .get(&committee_pubkey) - .and_then(|item| item.agg_nonce_consensus_hash.zip(item.agg_nonce_consensus_signature)) - && (existing_hash != consensus_hash || existing_signature != signature) - { - tracing::error!( - event = "committee_agg_nonce_consensus_conflict", - %instance_id, - %graph_id, - %committee_pubkey, - existing_consensus_hash = %hex::encode(existing_hash), - existing_signature = %existing_signature, - conflicting_consensus_hash = %hex::encode(consensus_hash), - conflicting_signature = %signature, - "detected conflicting committee agg nonce consensus signatures" - ); - bail!(SpecialError::InvalidGraph(format!( - "committee agg nonce consensus changed for graph {graph_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|item| { - item.agg_nonce_consensus_hash = Some(consensus_hash); - item.agg_nonce_consensus_signature = Some(signature); - }) - .or_insert_with(|| GraphProcessDataItem { - agg_nonce_consensus_hash: Some(consensus_hash), - agg_nonce_consensus_signature: Some(signature), - ..Default::default() - }); - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - is_endorsed, - &process_data, - ) - .await?; - Ok(()) -} -pub async fn get_committee_agg_nonce_consensus_for_graph( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data - .iter() - .filter_map(|(pubkey, item)| { - item.agg_nonce_consensus_hash - .zip(item.agg_nonce_consensus_signature) - .map(|(consensus_hash, signature)| (*pubkey, consensus_hash, signature)) - }) - .collect()) -} -pub async fn store_committee_partial_sigs_for_graph( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - committee_pubkey: PublicKey, - partial_sigs: CommitteePartialSignatures, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (is_endorsed, mut process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - if let Some(existing_partial_sigs) = - process_data.get(&committee_pubkey).and_then(|v| v.partial_sigs.as_ref()) - && existing_partial_sigs != &partial_sigs - { - bail!(SpecialError::InvalidGraph(format!( - "committee partial signatures changed for graph {graph_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|v| v.partial_sigs = Some(partial_sigs.clone())) - .or_insert_with(|| GraphProcessDataItem { - partial_sigs: Some(partial_sigs), - ..Default::default() - }); - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - is_endorsed, - &process_data, - ) - .await?; - Ok(()) -} -pub async fn get_committee_partial_sigs_for_graph( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| v.partial_sigs.as_ref().map(|nonce| (*k, nonce.clone()))) - .collect::>()) -} - -pub async fn get_committee_partial_sigs_for_graph_member( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, - committee_pubkey: &PublicKey, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data.get(committee_pubkey).and_then(|v| v.partial_sigs.clone())) -} -pub async fn store_committee_endorsement_for_graph( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - committee_pubkey: PublicKey, - committee_evm_address: EvmAddress, - endorse_signature: Vec, - params_endorse_signature: Vec, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (is_endorsed, mut process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - process_data - .entry(committee_pubkey) - .and_modify(|v| { - v.endorse_signature = endorse_signature.clone(); - v.params_endorse_signature = params_endorse_signature.clone(); - v.committee_evm_address = Some(committee_evm_address); - }) - .or_insert_with(|| GraphProcessDataItem { - committee_evm_address: Some(committee_evm_address), - endorse_signature, - params_endorse_signature, - ..Default::default() - }); - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - is_endorsed, - &process_data, - ) - .await?; - Ok(()) -} -pub async fn store_committee_endorsements_for_graph( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, - params_endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (is_endorsed, mut process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - - for (committee_pubkey, committee_evm_address, endorse_signature) in endorse_sigs { - process_data - .entry(committee_pubkey) - .and_modify(|v| { - v.endorse_signature = endorse_signature.clone(); - v.committee_evm_address = Some(committee_evm_address); - }) - .or_insert_with(|| GraphProcessDataItem { - committee_evm_address: Some(committee_evm_address), - endorse_signature, - ..Default::default() - }); - } - for (committee_pubkey, committee_evm_address, params_endorse_signature) in params_endorse_sigs { - process_data - .entry(committee_pubkey) - .and_modify(|v| { - v.params_endorse_signature = params_endorse_signature.clone(); - v.committee_evm_address = Some(committee_evm_address); - }) - .or_insert_with(|| GraphProcessDataItem { - committee_evm_address: Some(committee_evm_address), - params_endorse_signature, - ..Default::default() - }); - } - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - is_endorsed, - &process_data, - ) - .await?; - Ok(()) -} -pub async fn get_committee_endorsements_for_graph( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result)>> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| { - if !v.endorse_signature.is_empty() { - v.committee_evm_address - .as_ref() - .map(|evm_addr| (*k, *evm_addr, v.endorse_signature.clone())) - } else { - None - } - }) - .collect::)>>()) -} - -pub async fn get_committee_params_endorsements_for_graph( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result)>> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| { - if !v.params_endorse_signature.is_empty() { - v.committee_evm_address - .as_ref() - .map(|evm_addr| (*k, *evm_addr, v.params_endorse_signature.clone())) - } else { - None - } - }) - .collect::)>>()) -} - -pub async fn store_verifier_graph_params_endorsement( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - verifier_pubkey: PublicKey, - verifier_index: usize, - verifier_peer_id: &str, - signature: SchnorrSignature, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (is_endorsed, mut process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - for (key, item) in &process_data { - if let Some(peer) = &item.verifier_peer_id - && ((*key == verifier_pubkey && peer != verifier_peer_id) - || (*key != verifier_pubkey - && (peer == verifier_peer_id - || XOnlyPublicKey::from(*key) == XOnlyPublicKey::from(verifier_pubkey)))) - { - bail!(SpecialError::InvalidGraph("conflicting verifier peer/key binding".into())); - } - } - if let Some(existing_index) = process_data.get(&verifier_pubkey).and_then(|v| v.verifier_index) - && existing_index != verifier_index - { - bail!(SpecialError::InvalidGraph(format!( - "verifier params endorsement index changed for graph {graph_id} and verifier {verifier_pubkey}" - ))); - } - if let Some(existing_signature) = - process_data.get(&verifier_pubkey).and_then(|v| v.verifier_params_signature.as_ref()) - && existing_signature != &signature - { - bail!(SpecialError::InvalidGraph(format!( - "verifier params endorsement signature changed for graph {graph_id} and verifier {verifier_pubkey}" - ))); - } - process_data - .entry(verifier_pubkey) - .and_modify(|v| { - v.verifier_index = Some(verifier_index); - v.verifier_peer_id = Some(verifier_peer_id.to_owned()); - v.verifier_params_signature = Some(signature); - }) - .or_insert_with(|| GraphProcessDataItem { - verifier_index: Some(verifier_index), - verifier_peer_id: Some(verifier_peer_id.to_owned()), - verifier_params_signature: Some(signature), - ..Default::default() - }); - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - is_endorsed, - &process_data, - ) - .await?; - Ok(()) -} - -pub async fn get_verifier_graph_params_endorsements_for_graph( - local_db: &LocalDB, - _instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let (_is_endorsed, process_data) = - find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| { - v.verifier_index - .zip(v.verifier_params_signature.as_ref()) - .zip(v.verifier_peer_id.as_ref()) - .map(|((index, signature), peer)| (*k, index, peer.clone(), *signature)) - }) - .collect()) -} -pub async fn mark_graph_as_endorsed( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let (_, process_data) = find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; - upsert_pegin_graph_process_data( - &mut storage_processor, - graph_id, - instance_id, - true, - &process_data, - ) - .await -} -pub async fn get_endorsed_graph_count(local_db: &LocalDB, instance_id: Uuid) -> Result { - let mut storage_processor = local_db.acquire().await?; - Ok(storage_processor.get_pegin_graph_endorsed_len_by_instance_id(&instance_id, true).await? - as usize) -} - -pub async fn has_required_presigned_graphs(local_db: &LocalDB, instance_id: Uuid) -> Result { - Ok(get_endorsed_graph_count(local_db, instance_id).await? >= min_required_operator()) -} - -pub async fn try_transition_instance_to_presigned( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result { - if !has_required_presigned_graphs(local_db, instance_id).await? { - return Ok(false); - } - - let mut storage_processor = local_db.acquire().await?; - let transitioned = storage_processor - .update_instance_status_if_current( - &instance_id, - &InstanceBridgeInStatus::UserBroadcastPeginPrepare.to_string(), - &InstanceBridgeInStatus::Presigned.to_string(), - ) - .await?; - if transitioned { - info!("Instance {instance_id} reached the required finalized graph threshold"); - } - Ok(transitioned) -} -pub async fn store_committee_pub_nonce_for_instance( - local_db: &LocalDB, - instance_id: Uuid, - committee_pubkey: PublicKey, - pub_nonce: PubNonce, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let mut process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - if let Some(existing_pub_nonce) = - process_data.get(&committee_pubkey).and_then(|v| v.pub_nonce.as_ref()) - && existing_pub_nonce != &pub_nonce - { - bail!(SpecialError::InvalidPeginData(format!( - "committee pub nonce changed for instance {instance_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|v| v.pub_nonce = Some(pub_nonce.clone())) - .or_insert_with(|| InstanceProcessDataItem { - pub_nonce: Some(pub_nonce), - ..Default::default() - }); - upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; - Ok(()) -} -pub async fn get_committee_pub_nonce_for_instance( - local_db: &LocalDB, - instance_id: Uuid, - committee_pubkey: &PublicKey, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - Ok(process_data.get(committee_pubkey).and_then(|v| v.pub_nonce.clone())) -} -pub async fn get_committee_pub_nonces_for_instance( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| v.pub_nonce.as_ref().map(|pub_nonce| (*k, pub_nonce.clone()))) - .collect()) -} -pub async fn store_committee_agg_nonce_consensus_for_instance( - local_db: &LocalDB, - instance_id: Uuid, - committee_pubkey: PublicKey, - consensus_hash: [u8; 32], - signature: SchnorrSignature, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let mut process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - if let Some((existing_hash, existing_signature)) = process_data - .get(&committee_pubkey) - .and_then(|item| item.agg_nonce_consensus_hash.zip(item.agg_nonce_consensus_signature)) - && (existing_hash != consensus_hash || existing_signature != signature) - { - tracing::error!( - event = "committee_pegin_confirm_nonce_consensus_conflict", - %instance_id, - %committee_pubkey, - existing_consensus_hash = %hex::encode(existing_hash), - existing_signature = %existing_signature, - conflicting_consensus_hash = %hex::encode(consensus_hash), - conflicting_signature = %signature, - "detected conflicting committee PeginConfirm nonce consensus signatures" - ); - bail!(SpecialError::InvalidPeginData(format!( - "committee PeginConfirm nonce consensus changed for instance {instance_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|item| { - item.agg_nonce_consensus_hash = Some(consensus_hash); - item.agg_nonce_consensus_signature = Some(signature); - }) - .or_insert_with(|| InstanceProcessDataItem { - agg_nonce_consensus_hash: Some(consensus_hash), - agg_nonce_consensus_signature: Some(signature), - ..Default::default() - }); - upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; - Ok(()) -} - -pub async fn get_committee_agg_nonce_consensus_for_instance( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - Ok(process_data - .iter() - .filter_map(|(pubkey, item)| { - item.agg_nonce_consensus_hash - .zip(item.agg_nonce_consensus_signature) - .map(|(consensus_hash, signature)| (*pubkey, consensus_hash, signature)) - }) - .collect()) -} -pub async fn store_committee_partial_sig_for_instance( - local_db: &LocalDB, - instance_id: Uuid, - committee_pubkey: PublicKey, - partial_sigs: PartialSignature, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let mut process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - if let Some(existing_partial_sig) = - process_data.get(&committee_pubkey).and_then(|v| v.partial_sign.as_ref()) - && existing_partial_sig != &partial_sigs - { - bail!(SpecialError::InvalidPeginData(format!( - "committee partial signature changed for instance {instance_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|v| v.partial_sign = Some(partial_sigs)) - .or_insert_with(|| InstanceProcessDataItem { - partial_sign: Some(partial_sigs), - ..Default::default() - }); - upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; - Ok(()) -} - -pub async fn get_committee_partial_sigs_for_instance( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| v.partial_sign.as_ref().map(|partial_sign| (*k, *partial_sign))) - .collect()) -} - -pub async fn get_committee_partial_sig_for_instance( - local_db: &LocalDB, - instance_id: Uuid, - committee_pubkey: &PublicKey, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - Ok(process_data.get(committee_pubkey).and_then(|v| v.partial_sign)) -} - -pub async fn store_committee_endorse_sig_for_pegin( - local_db: &LocalDB, - instance_id: Uuid, - committee_pubkey: PublicKey, - endorse_sig: Vec, -) -> Result<()> { - let mut storage_processor = local_db.acquire().await?; - let mut process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - if let Some(existing_endorse_sig) = - process_data.get(&committee_pubkey).map(|v| v.endorse_signature.as_slice()) - && !existing_endorse_sig.is_empty() - && existing_endorse_sig != endorse_sig.as_slice() - { - bail!(SpecialError::InvalidPeginData(format!( - "committee endorse signature changed for instance {instance_id} and committee {committee_pubkey}" - ))); - } - process_data - .entry(committee_pubkey) - .and_modify(|v| v.endorse_signature = endorse_sig.clone()) - .or_insert_with(|| InstanceProcessDataItem { - endorse_signature: endorse_sig, - ..Default::default() - }); - upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; - Ok(()) -} -pub async fn get_committee_endorse_sigs_for_pegin( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result)>> { - let mut storage_processor = local_db.acquire().await?; - let process_data = - find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; - Ok(process_data - .iter() - .filter_map(|(k, v)| { - if !v.endorse_signature.is_empty() { - Some((*k, v.endorse_signature.clone())) - } else { - None - } - }) - .collect::)>>()) -} - -pub async fn graph_exists(local_db: &LocalDB, instance_id: Uuid, graph_id: Uuid) -> Result { - let mut storage_processor = local_db.acquire().await?; - if let Some(graph) = storage_processor.find_graph(&graph_id).await? - && graph.instance_id == instance_id - { - Ok(true) - } else { - Ok(false) - } -} - -#[allow(clippy::too_many_arguments)] -pub async fn try_update_graph_challenge_txid( - btc_client: &BTCClient, - local_db: &LocalDB, - graph_id: Uuid, - kickoff_txid: Txid, - connector_a_vout: u64, - take1_txid: Txid, -) -> Result<()> { - let graph = { - let mut storage_processor = local_db.acquire().await?; - match storage_processor.find_graph(&graph_id).await? { - Some(graph) => graph, - None => { - warn!("graph{graph_id} not in db"); - return Ok(()); - } - } - }; - - if graph.challenge_txid.is_none() - && let Some(spent_txid) = - outpoint_spent_txid(btc_client, &kickoff_txid, connector_a_vout).await? - && spent_txid != take1_txid - { - info!( - "try_update_graph_challenge_txid update challenge_txid: {spent_txid} for graph {graph_id}" - ); - let mut storage_processor = local_db.acquire().await?; - storage_processor - .update_graph_runtime( - &GraphRuntimeUpdate::new(graph.instance_id, graph_id) - .with_challenge_txid(spent_txid.into()), - ) - .await?; - } else { - info!("try_update_graph_challenge_txid no need to challenge_txid for graph {graph_id}"); - } - Ok(()) -} - -pub async fn update_graph_status( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - new_status: GraphStatus, - sub_status: Option, -) -> Result { - let mut storage_processor = local_db.acquire().await?; - storage_processor - .transition_graph_status( - instance_id, - graph_id, - new_status, - GraphStatusSource::ChainReconcile, - sub_status.map(|sub_status| serde_json::to_string(&sub_status)).transpose()?, - ) - .await -} - -/// Mark one graph obsolete and cancel its pending messages. -pub(crate) async fn obsolete_graph( - storage_processor: &mut StorageProcessor<'_>, - instance_id: Uuid, - graph_id: Uuid, -) -> Result { - let outcome = storage_processor - .transition_graph_status( - instance_id, - graph_id, - GraphStatus::Obsoleted, - GraphStatusSource::ChainReconcile, - None, - ) - .await?; - if !matches!( - outcome, - GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent - ) { - return Ok(false); - } - - // `None` means no message-type filter: cancel every durable pending message - // for this terminal graph so stale retries cannot consume queue capacity or - // trigger a later graph action. - storage_processor.cancel_messages_by_business_id(&graph_id, None).await?; - Ok(true) -} - -/// Mark every non-terminal graph for an already claimed instance as obsolete. -/// -/// `retained_graph_id` is the graph which completed the withdrawal. The caller -/// must only use this after a terminal Gateway withdraw event confirms that -/// this graph completed the withdrawal. -pub(crate) async fn obsolete_instance_graphs_except( - storage_processor: &mut StorageProcessor<'_>, - instance_id: Uuid, - retained_graph_id: Option, -) -> Result> { - let graphs = storage_processor.get_graphs_by_instance_id(&instance_id).await?; - let mut obsoleted_graph_ids = Vec::new(); - - for graph in graphs { - if Some(graph.graph_id) != retained_graph_id - && obsolete_graph(storage_processor, instance_id, graph.graph_id).await? - { - obsoleted_graph_ids.push(graph.graph_id); - } - } - - Ok(obsoleted_graph_ids) -} - -pub async fn get_graph_ids_for_instance( - local_db: &LocalDB, - instance_id: Uuid, -) -> Result> { - let mut storage_processor = local_db.acquire().await?; - let graphs = storage_processor.get_graphs_by_instance_id(&instance_id).await?; - Ok(graphs.into_iter().map(|v| v.graph_id).collect()) -} - -pub fn gen_instance_parameters_local( - instance: &Instance, -) -> anyhow::Result { - let network = Network::from_str(&instance.network)?; - let committee_pubkeys: Vec = instance - .committees_answers - .iter() - .map(|(committee, pubkey)| { - PublicKey::from_slice(pubkey).with_context(|| { - format!( - "invalid committee public key for committee {committee} in instance {}", - instance.instance_id - ) - }) - }) - .collect::>()?; - - let committee_agg_pubkey = generate_n_of_n_public_key(&committee_pubkeys).0; - let utxos: Vec = serde_json::from_str(&instance.input_utxos)?; - Ok(BitvmGcInstanceParameters { - network, - instance_id: instance.instance_id, - user_info: gen_user_info( - network, - &instance.to_addr, - &instance.user_change_addr.clone(), - &instance.user_refund_addr.clone(), - utxos, - instance.fees.0, - &instance.user_xonly_pubkey.0, - )?, - pegin_amount: Amount::from_sat(instance.amount as u64), - committee_pubkeys, - committee_agg_pubkey, - }) -} - -fn gen_user_info( - network: Network, - depositor_evm_address: &str, - user_change_addr: &str, - user_refund_addr: &str, - utxos: Vec, - txn_fees: [u64; 3], - user_xonly_pubkey: &[u8; 32], -) -> anyhow::Result { - let user_change_address: Address = Address::from_str(user_change_addr)?; - let user_refund_addr: Address = Address::from_str(user_refund_addr)?; - let inputs = utxos - .into_iter() - .map(|utxo| Input { - outpoint: OutPoint { txid: Txid::from_slice(&utxo.txid).unwrap(), vout: utxo.vout }, - amount: Amount::from_sat(utxo.amount_sats), - }) - .collect(); - Ok(UserInfo { - depositor_evm_address: EvmAddress::from_str(depositor_evm_address)?.into_array(), - txn_fees, - inputs, - user_xonly_pubkey: XOnlyPublicKey::from_slice(user_xonly_pubkey)?, - user_change_address: user_change_address.require_network(network)?, - user_refund_address: user_refund_addr.require_network(network)?, - }) -} - -pub async fn check_bridge_in_uxto_available_or_self_spent( - btc_client: &BTCClient, - target_txid: Option, - utxos: &[client::Utxo], -) -> Result { - for utxo in utxos { - if let Ok(txid) = Txid::from_slice(&utxo.txid) - && let Ok(Some(status)) = btc_client.get_output_status(&txid, utxo.vout as u64).await - && status.spent - { - if let Some(target_txid) = target_txid - && let Some(txid) = status.txid - && txid.to_string() == target_txid - { - return Ok(true); - } - return Ok(false); - } - } - Ok(true) -} - -/// Computes the graph constant committed by the Operator guest and Connector D. -pub fn get_guest_constant_value( - graph_id: Uuid, - challenge_init_txid: [u8; 32], - watchtower_pubkeys: &[XOnlyPublicKey], -) -> Result<[u8; 32]> { - let key_bytes = watchtower_pubkeys.iter().map(XOnlyPublicKey::serialize).collect::>(); - Ok(hash_operator_constant( - graph_id.into_bytes(), - get_genesis_sequencer_commit_id(), - challenge_init_txid, - &key_bytes, - )) -} -pub(crate) async fn get_bridge_out_global_stats<'a>( - storage_processor: &mut StorageProcessor<'a>, -) -> Result { - match storage_processor.find_bridge_out_global_stats_by_id(BRIDGE_OUT_GLOBAL_STATS_ID).await? { - Some(global_stats) => Ok(global_stats), - None => Ok(BridgeOutGlobalStats { - id: BRIDGE_OUT_GLOBAL_STATS_ID, - initial_txn: 0, - initial_amount: "0".to_string(), - claim_txn: 0, - claim_amount: "0".to_string(), - refund_txn: 0, - refund_amount: "0".to_string(), - created_at: 0, - updated_at: 0, - }), - } -} - -pub async fn get_largest_watchtower_challenge_block( - graph: &BitvmGcGraph, - btc_client: &BTCClient, -) -> anyhow::Result { - let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); - let mut largest_watchtower_challenge_block_height = 0u32; - let mut largest_watchtower_challenge_block_hash: BlockHash = - BlockHash::from_slice(&[0u8; 32]).unwrap(); - for watchtower_index in 0..graph.parameters.watchtower_pubkeys.len() { - let watchtower_challenge_vout = - output_topology::watchtower_challenge_init::watchtower_connector(watchtower_index) - as u64; - match outpoint_spent_txid( - btc_client, - &watchtower_challenge_init_txid, - watchtower_challenge_vout, - ) - .await - { - Ok(Some(txid)) => { - let tx_status = btc_client.get_tx_status(&txid).await?; - if let Some(block_height) = tx_status.block_height { - if block_height > largest_watchtower_challenge_block_height { - largest_watchtower_challenge_block_height = block_height; - largest_watchtower_challenge_block_hash = tx_status.block_hash.ok_or_else(|| { - anyhow!( - "Watchtower challenge tx {txid} for graph {}, index: {watchtower_index} is confirmed at height {block_height} without a block hash", - graph.parameters.graph_id - ) - })?; - } - } else { - anyhow::bail!( - "Watchtower challenge tx {txid} for graph {}, index: {watchtower_index} not confirmed yet", - graph.parameters.graph_id - ); - } - } - Ok(None) => { - anyhow::bail!( - "Watchtower challenge connector {watchtower_index} for graph {}, index: {watchtower_index} not spent yet", - graph.parameters.graph_id - ); - } - Err(e) => { - anyhow::bail!( - "Error checking watchtower challenge connector graph: {}, index: {watchtower_index} spent status: {e}", - graph.parameters.graph_id - ); - } - } - } - Ok(largest_watchtower_challenge_block_hash) -} - -pub(crate) fn babe_setup_state_root(local_db: &LocalDB) -> PathBuf { - std::env::var_os(ENV_BABE_SETUP_STATE_DIR) - .filter(|path| !path.as_os_str().is_empty()) - .map(PathBuf::from) - .unwrap_or_else(|| { - let db_path = local_db - .path - .strip_prefix("sqlite://") - .or_else(|| local_db.path.strip_prefix("sqlite:")) - .unwrap_or(&local_db.path); - PathBuf::from(db_path) - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from(".")) - .join(".bitvm-babe-state") - }) -} - -fn babe_setup_state_path(local_db: &LocalDB, instance_id: Uuid, graph_id: Uuid) -> PathBuf { - babe_setup_state_root(local_db).join(instance_id.to_string()).join(format!("{graph_id}.bin")) -} - -pub(crate) fn soldering_payload_hash(payload: &[u8]) -> [u8; 32] { - Sha256::digest(payload).into() -} - -pub(crate) fn soldering_payload_hash_hex(payload_hash: &[u8; 32]) -> String { - payload_hash.iter().map(|byte| format!("{byte:02x}")).collect() -} - -pub(crate) async fn pending_graph_belongs_to_operator( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - operator_pubkey: &PublicKey, -) -> Result { - let mut storage = local_db.acquire().await?; - Ok(storage.find_pending_graph_init_by_graph_id(&graph_id).await?.is_some_and(|pending| { - pending.instance_id == instance_id && pending.operator_pubkey == operator_pubkey.to_string() - })) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) async fn save_soldering_proof_payload( - instance_id: Uuid, - graph_id: Uuid, - candidate_index: usize, - opened: &[(usize, Seed)], - finalized: &[FinalizedInstanceData], - soldering: &SolderingData, -) -> Result { - let compact_payload = compact_soldering_proof_payload(opened, finalized, soldering)?; - let payload = bincode::serialize(&compact_payload) - .context("serialize compact soldering proof payload")?; - let payload_hash = soldering_payload_hash(&payload); - let total_len = payload.len(); - let store_base_path = get_soldering_proof_payload_store_path()?; - let payload_path = soldering_proof_payload_store_path( - &store_base_path, - instance_id, - graph_id, - candidate_index, - &payload_hash, - )?; - let store_mode = if is_soldering_proof_s3_path(&store_base_path) { "s3" } else { "local" }; - if let Err(err) = - write_soldering_proof_store_payload(&payload_path, &payload).await.with_context(|| { - format!("write soldering proof payload to configured store path {payload_path}") - }) - { - tracing::error!( - store_mode, - total_len, - payload_hash = %soldering_payload_hash_hex(&payload_hash), - payload_path = %payload_path, - error = %err, - "failed to write soldering proof payload to store" - ); - return Err(err); - } - tracing::info!( - store_mode, - total_len, - payload_hash = %soldering_payload_hash_hex(&payload_hash), - payload_path = %payload_path, - "saved compact soldering proof to payload store" - ); - Ok(SolderingProofReady { instance_id, graph_id, candidate_index, payload_hash, total_len }) -} - -fn load_babe_setup_state_from_path(path: &Path) -> Result> { - match std::fs::read(path) { - Ok(bytes) => Ok(Some( - bincode::deserialize(&bytes) - .with_context(|| format!("decode BABE setup state {}", path.display()))?, - )), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err).with_context(|| format!("read BABE setup state {}", path.display())), - } -} - -fn save_babe_setup_state_to_path(path: &Path, state: &BabeSetupState) -> Result<()> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("create BABE setup state dir {}", parent.display()))?; - } - let bytes = bincode::serialize(state)?; - let mut temp_path = path.as_os_str().to_os_string(); - temp_path.push(".tmp"); - let temp_path = PathBuf::from(temp_path); - std::fs::write(&temp_path, bytes) - .with_context(|| format!("write BABE setup state {}", temp_path.display()))?; - std::fs::rename(&temp_path, path).with_context(|| { - format!("move BABE setup state {} to {}", temp_path.display(), path.display()) - }) -} - -pub(crate) fn load_babe_setup_state( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, -) -> Result> { - load_babe_setup_state_from_path(&babe_setup_state_path(local_db, instance_id, graph_id)) -} - -pub(crate) fn save_babe_setup_state( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - state: &BabeSetupState, -) -> Result<()> { - save_babe_setup_state_to_path(&babe_setup_state_path(local_db, instance_id, graph_id), state) -} - -pub(crate) fn update_babe_setup_state( - local_db: &LocalDB, - instance_id: Uuid, - graph_id: Uuid, - update: impl FnOnce(&mut BabeSetupState), -) -> Result { - let mut state = load_babe_setup_state(local_db, instance_id, graph_id)?.unwrap_or_default(); - update(&mut state); - save_babe_setup_state(local_db, instance_id, graph_id, &state)?; - Ok(state) -} - -fn babe_setup_state_is_stale(path: &Path, now: SystemTime) -> Result { - let modified = std::fs::metadata(path) - .with_context(|| format!("read BABE setup state metadata {}", path.display()))? - .modified() - .with_context(|| format!("read BABE setup state modified time {}", path.display()))?; - Ok(now.duration_since(modified).unwrap_or_default() >= BABE_SETUP_STATE_ORPHAN_RETENTION) -} - -fn should_cleanup_babe_setup_state(status: Option, is_stale: bool) -> bool { - match status { - Some(status) if status.is_closed() => true, - Some(status) if status.is_obsoleted() => is_stale, - None => is_stale, - Some(_) => false, - } -} - -pub(crate) async fn cleanup_babe_setup_states( - local_db: &LocalDB, - payload_store: &str, -) -> Result { - let root = babe_setup_state_root(local_db); - let instance_dirs = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(err) => { - return Err(err) - .with_context(|| format!("read BABE setup state root {}", root.display())); - } - }; - let mut storage = local_db.acquire().await?; - let now = SystemTime::now(); - let mut deleted = 0; - for instance_dir in instance_dirs { - let instance_dir = instance_dir?; - if !instance_dir.file_type()?.is_dir() { - continue; - } - let Ok(instance_id) = Uuid::parse_str(&instance_dir.file_name().to_string_lossy()) else { - continue; - }; - for state_file in std::fs::read_dir(instance_dir.path())? { - let state_file = state_file?; - let path = state_file.path(); - if path.extension().and_then(|extension| extension.to_str()) != Some("bin") { - continue; - } - let Some(graph_id) = path - .file_stem() - .and_then(|stem| stem.to_str()) - .and_then(|stem| Uuid::parse_str(stem).ok()) - else { - continue; - }; - let is_stale = babe_setup_state_is_stale(&path, now)?; - let status = match storage.find_graph(&graph_id).await? { - Some(graph) if graph.instance_id == instance_id => { - let Ok(status) = GraphStatus::from_str(&graph.status) else { - warn!( - "Skip BABE setup cleanup for {graph_id}: invalid graph status {}", - graph.status - ); - continue; - }; - Some(status) - } - Some(graph) => { - warn!( - "BABE setup state {graph_id} belongs to {instance_id}, but the stored graph belongs to {}", - graph.instance_id - ); - None - } - None => None, - }; - if !should_cleanup_babe_setup_state(status, is_stale) { - continue; - } - if status.is_none() { - warn!("Cleaning stale orphan BABE setup state for {instance_id}:{graph_id}"); - } - let Some(state) = load_babe_setup_state_from_path(&path)? else { - continue; - }; - let mut payloads = Vec::new(); - if let Some(ready) = state.verifier.and_then(|verifier| verifier.soldering_proof_ready) - { - payloads.push(ready); - } - if let Some(operator) = state.operator { - payloads.extend( - operator - .candidates - .into_iter() - .filter_map(|candidate| candidate.soldering_proof_ready), - ); - } - for ready in payloads { - if ready.instance_id != instance_id || ready.graph_id != graph_id { - bail!("BABE setup payload reference does not match state path"); - } - let payload_path = soldering_proof_payload_store_path( - payload_store, - instance_id, - graph_id, - ready.candidate_index, - &ready.payload_hash, - )?; - delete_soldering_proof_store_payload(&payload_path).await?; - } - std::fs::remove_file(&path) - .with_context(|| format!("delete BABE setup state {}", path.display()))?; - deleted += 1; - } - } - Ok(deleted) -} - -#[derive(Clone, Default, Serialize, Deserialize)] -pub struct BabeSetupState { - pub verifier: Option, - pub operator: Option, -} - -#[derive(Clone, Serialize, Deserialize)] -pub struct VerifierBabeSetupState { - pub verifier_pubkey: PublicKey, - /// Authenticated operator identity that initiated this setup. - pub operator_pubkey: PublicKey, - /// Signed operator P2P identity used for directed setup replies and ACKs. - pub operator_peer_id: Vec, - pub setup_package: CACSetupPackage, - pub private_state: BabeVerifierPrivateState, - pub finalized_indices: Vec, - pub soldering_proof_ready: Option, -} - -#[derive(Clone, Serialize, Deserialize)] -pub struct OperatorVerifierCandidate { - pub verifier_peer_id: Vec, - pub verifier_pubkey: PublicKey, - pub setup_package: CACSetupPackage, - /// Immutable slot within the candidate pool and proof payload store. - pub candidate_index: Option, - pub selected_circuit_indexes: Vec, - pub gc_data: Option, - pub soldering_proof_ready: Option, -} - -#[derive(Clone, Serialize, Deserialize)] -pub struct OperatorBabeSetupState { - /// Candidate pool closed before CutCircuits. This is not the final graph - /// verifier set; `selected_verifier_pubkeys` is set after proof validation. - pub candidate_verifier_pubkeys: Option>, - pub candidates: Vec, - #[serde(default)] - pub candidate_collection_started_at: Option, - #[serde(default)] - pub proof_collection_started_at: Option, - #[serde(default)] - pub selected_verifier_pubkeys: Option>, - #[serde(default)] - pub asserted_operator_proof: Option>, -} - -#[cfg(test)] -mod commit_pubin_tests { - use super::*; - use ark_serialize::CanonicalSerialize; - use bitcoin::BlockHash; - use client::btc_chain::BTCClient; - use esplora_client::{Tx, TxStatus, Vin}; - use store::SerializableTxid; - - fn make_txid(byte: u8) -> Txid { - Txid::from_slice(&[byte; 32]).unwrap() - } - - fn make_block_hash(byte: u8) -> BlockHash { - BlockHash::from_slice(&[byte; 32]).unwrap() - } - - fn create_confirmed_tx( - txid: Txid, - spends: &[(Txid, u32)], - height: u32, - block_hash: BlockHash, - ) -> Tx { - Tx { - txid, - version: 2, - locktime: 0, - vin: spends - .iter() - .map(|(prev_txid, vout)| Vin { - txid: *prev_txid, - vout: *vout, - prevout: None, - scriptsig: ScriptBuf::default(), - witness: vec![], - sequence: 1, - is_coinbase: false, - }) - .collect(), - vout: vec![], - size: 1, - weight: 1, - status: TxStatus { - confirmed: true, - block_height: Some(height), - block_hash: Some(block_hash), - block_time: Some(1_000_000), - }, - fee: 0, - } - } - - /// The operator guest commits three 32-byte values, so its public values are the 96-byte pubin - /// itself. The wrap circuit's dynamic input must be BLAKE3 of those bytes (imm-wrap-vk mode, - /// set by `.cargo/config.toml`) with the top three bits cleared. - #[test] - fn assert_ready_extracts_blake3_xd_from_operator_public_values() { - let mut pubin = [0u8; 96]; - pubin[..32].copy_from_slice(&[0x11; 32]); - pubin[32..64].copy_from_slice(&[0x22; 32]); - pubin[64..].copy_from_slice(&[0x33; 32]); - let outputs = decode_operator_public_outputs(&pubin).unwrap(); - assert_eq!(outputs.btc_best_block_hash, [0x11; 32]); - assert_eq!(outputs.constant, [0x22; 32]); - assert_eq!(outputs.included_watchtowers, [0x33; 32]); - - let public_inputs: PublicInputs = zkm_verifier::load_ark_public_inputs_from_bytes( - &[0u8; 32], - &zkm_verifier::hash_public_inputs(&pubin), - &VK_ROOT_BYTES, - ) - .into(); - let dynamic_input = operator_assert_dynamic_input(&public_inputs).unwrap(); - assert_eq!( - operator_static_inputs(&public_inputs).unwrap(), - ziren_static_inputs(&[0u8; 32], &VK_ROOT_BYTES) - ); - - let mut expected_xd = *blake3::hash(&pubin).as_bytes(); - expected_xd[0] &= 0x1f; - let mut actual_xd = Vec::new(); - dynamic_input.serialize_uncompressed(&mut actual_xd).unwrap(); - // `pi1_xd_to_wots96_msg` serializes the field element most-significant byte first, - // whereas the public-input digest is defined in little-endian order. - actual_xd.reverse(); - assert_eq!(actual_xd, expected_xd); - } - - #[tokio::test] - async fn test_get_watchtower_challenge_info_partial_inclusion() { - let (btc_client, mock_adaptor) = BTCClient::new_mock_client(); - - let watchtower_init_txid = make_txid(0x00); - let challenge_txid_wt0 = make_txid(0x01); - let block_hash = make_block_hash(0xAA); - - let challenge_vout_0 = - output_topology::watchtower_challenge_init::watchtower_connector(0) as u32; - let ack_vout_0 = output_topology::watchtower_challenge_init::ack_connector(0) as u32; - - // watchtower 0 spent its challenge connector and ACK connector; watchtower 1 is unresolved. - mock_adaptor.set_tx( - challenge_txid_wt0, - create_confirmed_tx( - challenge_txid_wt0, - &[(watchtower_init_txid, challenge_vout_0)], - 100, - block_hash, - ), - ); - let ack_txid_wt0 = make_txid(0x02); - mock_adaptor.set_tx( - ack_txid_wt0, - create_confirmed_tx( - ack_txid_wt0, - &[(watchtower_init_txid, ack_vout_0)], - 101, - block_hash, - ), - ); - - let info = get_watchtower_challenge_info( - &btc_client, - &SerializableTxid::from(watchtower_init_txid), - &[], - 2, // 2 watchtowers - ) - .await - .unwrap(); - - assert_eq!(info.challenge_txids, vec![Some(challenge_txid_wt0.to_string()), None]); - assert_eq!(info.included_watchtowers, vec![true, false]); - assert_eq!(info.resolved_branch_txids, vec![challenge_txid_wt0]); - } - - #[tokio::test] - async fn test_compute_operator_pubin_blockhash_and_bitmap() { - let (btc_client, mock_adaptor) = BTCClient::new_mock_client(); - - let init_txid = make_txid(0x00); - let challenge_txid_wt0 = make_txid(0x01); - let timeout_txid_wt1 = make_txid(0x03); - let challenge_txid_wt2 = make_txid(0x02); - let block_hash_low = make_block_hash(0x10); - let block_hash_middle = make_block_hash(0x15); - let block_hash_high = make_block_hash(0x20); - - let challenge_vout_0 = - output_topology::watchtower_challenge_init::watchtower_connector(0) as u32; - let challenge_vout_2 = - output_topology::watchtower_challenge_init::watchtower_connector(2) as u32; - - // All three branches are finalized; wt2's height 200 block is the anchor. - let required_burial_depth = get_btc_block_confirms(btc_client.network()); - mock_adaptor.set_height(200 + required_burial_depth + 1); - mock_adaptor.set_tx( - challenge_txid_wt0, - create_confirmed_tx( - challenge_txid_wt0, - &[(init_txid, challenge_vout_0)], - 100, - block_hash_low, - ), - ); - mock_adaptor.set_tx( - timeout_txid_wt1, - create_confirmed_tx(timeout_txid_wt1, &[(init_txid, 1)], 150, block_hash_middle), - ); - mock_adaptor.set_tx( - challenge_txid_wt2, - create_confirmed_tx( - challenge_txid_wt2, - &[(init_txid, challenge_vout_2)], - 200, - block_hash_high, - ), - ); - - let resolved_branch_txids = vec![challenge_txid_wt0, timeout_txid_wt1, challenge_txid_wt2]; - let bits = vec![true, false, true]; - - let (best_hash, bitmap) = - compute_operator_pubin_blockhash_and_bitmap(&btc_client, &resolved_branch_txids, &bits) - .await - .unwrap(); - - // block at height 200 wins - assert_eq!(best_hash, block_hash_high.to_byte_array()); - - // bits 0 and 2 set → byte 0 = 0b0000_0101 - assert_eq!(bitmap[0], 0b0000_0101u8); - assert_eq!(&bitmap[1..], &[0u8; 31]); - } -} - -#[cfg(test)] -mod node_info_tests { - use super::*; - - /// One field mutation applied to an otherwise valid payload. - type NodeInfoMutation = fn(&mut NodeInfo); - - fn valid_node_info(peer_id: &str) -> NodeInfo { - NodeInfo { - peer_id: peer_id.to_string(), - actor: Actor::Operator.to_string(), - goat_addr: "0x1234567890abcdef1234567890abcdef12345678".to_string(), - btc_pub_key: "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798" - .to_string(), - socket_addr: "127.0.0.1:8080".to_string(), - node_name: "zkm".to_string(), - service_fee_rate: 0.001, - available_peg_btc: "1000".to_string(), - ..Default::default() - } - } - - #[test] - fn node_info_is_accepted_only_for_its_own_sender() { - let sender = PeerId::random(); - let other = PeerId::random(); - let node_info = valid_node_info(&sender.to_string()); - - assert!(node_info_matches_sender(&node_info, &sender)); - assert!(!node_info_matches_sender(&node_info, &other)); - } - - #[test] - fn valid_node_info_payload_is_accepted() { - let node_info = valid_node_info(&PeerId::random().to_string()); - assert!(validate_node_info_payload(&node_info).is_ok()); - } - - #[test] - fn optional_node_info_fields_may_be_empty() { - let mut node_info = valid_node_info(&PeerId::random().to_string()); - node_info.goat_addr = String::new(); - node_info.btc_pub_key = String::new(); - node_info.socket_addr = String::new(); - node_info.available_peg_btc = String::new(); - node_info.node_name = String::new(); - assert!(validate_node_info_payload(&node_info).is_ok()); - } - - #[test] - fn malformed_node_info_fields_are_rejected() { - let peer_id = PeerId::random().to_string(); - let cases: Vec<(&str, NodeInfoMutation)> = vec![ - ("peer_id", |n| n.peer_id = "not-a-peer-id".to_string()), - ("actor", |n| n.actor = "Auditor".to_string()), - ("goat_addr", |n| n.goat_addr = "0xdeadbeef".to_string()), - ("btc_pub_key", |n| n.btc_pub_key = "02zz".to_string()), - ("available_peg_btc", |n| n.available_peg_btc = "lots".to_string()), - ("service_fee_rate NaN", |n| n.service_fee_rate = f64::NAN), - ("service_fee_rate range", |n| n.service_fee_rate = 42.0), - ("node_name length", |n| n.node_name = "a".repeat(MAX_NODE_NAME_LEN + 1)), - ("node_name control", |n| n.node_name = "a\nb".to_string()), - ("socket_addr length", |n| { - n.socket_addr = format!("{}:8080", "a".repeat(MAX_SOCKET_ADDR_LEN)) - }), - ("socket_addr port", |n| n.socket_addr = "host".to_string()), - ("socket_addr host", |n| n.socket_addr = ":8080".to_string()), - ("socket_addr control", |n| n.socket_addr = "127.0.0.1 :8080".to_string()), - ]; - - for (label, mutate) in cases { - let mut node_info = valid_node_info(&peer_id); - mutate(&mut node_info); - assert!( - validate_node_info_payload(&node_info).is_err(), - "expected {label} to be rejected" - ); - } - } -} diff --git a/node/src/utils/btc.rs b/node/src/utils/btc.rs new file mode 100644 index 000000000..d1916a4e5 --- /dev/null +++ b/node/src/utils/btc.rs @@ -0,0 +1,565 @@ +//! Bitcoin transactions: fees, broadcasts, CPFP, UTXO selection, outpoint and +//! confirmation checks. + +use crate::env::*; +use crate::error::SpecialError; +use anyhow::{Result, anyhow, bail}; +use bitcoin::hashes::Hash; +use bitcoin::key::Keypair; +use bitcoin::{ + Address, Amount, EcdsaSighashType, Network, OutPoint, PublicKey, ScriptBuf, Sequence, + Transaction, TxIn, TxOut, Txid, Witness, +}; +use bitvm::treepp::*; +use bitvm_lib::operator::*; +use bitvm_lib::timelocks::connector_f_timelock_blocks; +use client::btc_chain::BTCClient; +use esplora_client::Utxo; +use goat::constants::TimelockConfig; +use goat::scripts::{generate_opreturn_script, p2a_output}; +use goat::transactions::base::Input; +use goat::transactions::signing::populate_p2wsh_witness; +use std::time::Instant; + +pub async fn is_take1_timelock_expired( + client: &BTCClient, + kickoff_height: u32, + timelock_config: &TimelockConfig, +) -> Result { + let lock_blocks = take1_timelock_with_config(get_network(), timelock_config); + let current_height = client.get_height().await?; + Ok(current_height >= kickoff_height + lock_blocks) +} + +pub async fn is_take2_timelock_expired( + client: &BTCClient, + operator_assert_height: u32, + watchtower_challenge_init_height: u32, + timelock_config: &TimelockConfig, +) -> Result { + let network = get_network(); + let connector_d_lock_blocks = take2_timelock_with_config(network, timelock_config); + let connector_f_lock_blocks = connector_f_timelock_blocks(network, timelock_config); + let current_height = client.get_height().await?; + Ok(current_height >= operator_assert_height + connector_d_lock_blocks + && current_height >= watchtower_challenge_init_height + connector_f_lock_blocks) +} + +pub async fn get_fee_rate(client: &BTCClient) -> Result { + match client.network() { + //TODO mempool api /fee-estimates failed, fix it latter + Network::Testnet | Network::Testnet4 | Network::Regtest => Ok(5.0), + _ => { + let res = client.get_fee_estimates().await?; + Ok(*res.get(&DEFAULT_CONFIRMATION_TARGET).ok_or(anyhow!( + "fee for {DEFAULT_CONFIRMATION_TARGET} confirmation target not found" + ))?) + } + } +} + +/// Broadcasts a raw transaction to the Bitcoin network using the mempool API. +/// +/// Requirements: +/// - The mempool API URL must be configured. +/// - The transaction should already be fully signed. +pub async fn broadcast_tx(client: &BTCClient, tx: &Transaction) -> Result<()> { + let txid = tx.compute_txid(); + let started_at = Instant::now(); + match client.broadcast(tx).await { + Ok(()) => { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_btc_tx_broadcast(true); + } + tracing::info!( + event = "btc_tx_broadcast", + outcome = "broadcasted", + txid = %txid, + input_count = tx.input.len(), + output_count = tx.output.len(), + elapsed_ms = started_at.elapsed().as_millis() as u64, + "bitcoin transaction broadcast accepted by client" + ); + Ok(()) + } + Err(err) => { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_btc_tx_broadcast(false); + } + tracing::warn!( + event = "btc_tx_broadcast", + outcome = "failed", + txid = %txid, + input_count = tx.input.len(), + output_count = tx.output.len(), + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %err, + "bitcoin transaction broadcast failed" + ); + Err(err) + } + } +} + +pub async fn broadcast_package( + client: &BTCClient, + txns: &[Transaction], + fallback_on_failure: bool, +) -> Result<()> { + let txids: Vec = txns.iter().map(Transaction::compute_txid).collect(); + let started_at = Instant::now(); + match client.broadcast_package(txns).await { + Ok(_) => { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_btc_tx_broadcast(true); + } + tracing::info!( + event = "btc_tx_package_broadcast", + outcome = "broadcasted", + transaction_count = txids.len(), + txids = ?txids, + elapsed_ms = started_at.elapsed().as_millis() as u64, + "bitcoin transaction package broadcast accepted by client" + ); + } + Err(e) => { + if fallback_on_failure { + tracing::warn!( + event = "btc_tx_package_broadcast", + outcome = "fallback_to_individual_broadcast", + transaction_count = txids.len(), + txids = ?txids, + elapsed_ms = started_at.elapsed().as_millis() as u64, + error = %e, + "bitcoin transaction package broadcast failed; falling back to individual broadcasts" + ); + for tx in txns { + broadcast_tx(client, tx).await?; + } + } else { + if let Some(metrics_state) = crate::metrics_service::node_metrics_state() { + metrics_state.record_btc_tx_broadcast(false); + } + // Surface the original error when fallback is disabled + return Err(e); + } + } + } + Ok(()) +} + +pub async fn fund_address( + client: &BTCClient, + node_keypair: Keypair, + address: Address, + amount: Amount, +) -> Result { + let txins = Vec::new(); + let txouts = vec![TxOut { script_pubkey: address.script_pubkey(), value: amount }]; + let txid = + build_sign_and_broadcast_tx(client, node_keypair, txins, Amount::ZERO, txouts).await?; + Ok(OutPoint { txid, vout: 0 }) +} + +pub async fn build_sign_and_broadcast_tx( + client: &BTCClient, + node_keypair: Keypair, + txins: Vec, + total_input_amount: Amount, + txouts: Vec, +) -> Result { + let txouts = if txouts.is_empty() { + // bitcoin network does not allow a transaction without outputs + vec![TxOut { value: Amount::ZERO, script_pubkey: generate_opreturn_script(vec![]) }] + } else { + txouts + }; + let mut tx = Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: txins, + output: txouts, + }; + let fixed_inputs_num = tx.input.len(); + let total_output_amount: Amount = tx.output.iter().map(|o| o.value).sum(); + let fee_rate = get_fee_rate(client).await?; + let node_address = node_p2wsh_address(get_network(), &node_keypair.public_key().into()); + let shortfall = + Amount::from_sat(total_output_amount.to_sat().saturating_sub(total_input_amount.to_sat())); + match get_proper_utxo_set( + client, + tx.weight().to_vbytes_ceil(), + node_address.clone(), + shortfall, + fee_rate, + ) + .await? + { + Some((inputs, _, change_amount)) => { + for input in &inputs { + tx.input.push(TxIn { + previous_output: input.outpoint, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::default(), + }); + } + if change_amount > Amount::from_sat(DUST_AMOUNT) { + tx.output.push(TxOut { + script_pubkey: node_address.script_pubkey(), + value: change_amount, + }); + } + for (i, input) in inputs.iter().enumerate() { + node_sign( + &mut tx, + i + fixed_inputs_num, + input.amount, + EcdsaSighashType::All, + &node_keypair, + )?; + } + broadcast_tx(client, &tx).await?; + Ok(tx.compute_txid()) + } + None => { + let current_balance = client + .get_address_utxo(node_address) + .await? + .iter() + .map(|u| u.value) + .sum::(); + bail!(SpecialError::InsufficientBalance(format!( + "Not enough balance to complete the transaction, current_balance: {current_balance} < shortfall: {shortfall}" + ))); + } + } +} + +pub async fn build_cpfp_txns( + btc_client: &BTCClient, + parent_tx: &Transaction, + anchor_vout: u64, + parent_tx_total_input_amount: Amount, +) -> Result> { + let network = get_network(); + if network == Network::Regtest || network == Network::Testnet || network == Network::Testnet4 { + // skip cpfp in test network for testing convenience + return Ok(None); + } + let node_master_keypair = get_bitvm_key()?; + let node_address = node_p2wsh_address(network, &node_master_keypair.public_key().into()); + let total_output_amount: Amount = parent_tx.output.iter().map(|o| o.value).sum(); + let fee_rate = get_fee_rate(btc_client).await?; + let fee_amount = + Amount::from_sat((parent_tx.weight().to_vbytes_ceil() as f64 * fee_rate).ceil() as u64); + if total_output_amount + fee_amount <= parent_tx_total_input_amount { + return Ok(None); + }; + let shortfall = total_output_amount + fee_amount - parent_tx_total_input_amount; + match get_proper_utxo_set( + btc_client, + ANCHOR_CHILD_BASE_VBYTES, + node_address.clone(), + shortfall, + fee_rate, + ) + .await? + { + Some((inputs, _, change_amount)) => { + let mut child_tx = Transaction { + version: bitcoin::transaction::Version(2), + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![], + output: vec![], + }; + if change_amount > Amount::from_sat(DUST_AMOUNT) { + child_tx.output.push(TxOut { + script_pubkey: node_address.script_pubkey(), + value: change_amount, + }); + } else { + // add an op_return output to avoid no-output transaction + child_tx.output.push(TxOut { + script_pubkey: generate_opreturn_script(vec![]), + value: Amount::ZERO, + }); + } + for input in &inputs { + child_tx.input.push(TxIn { + previous_output: input.outpoint, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::default(), + }); + } + child_tx.input.push(TxIn { + previous_output: OutPoint { + txid: parent_tx.compute_txid(), + vout: anchor_vout as u32, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::default(), + }); + for (i, input) in inputs.iter().enumerate() { + node_sign( + &mut child_tx, + i, + input.amount, + EcdsaSighashType::All, + &node_master_keypair, + )?; + } + Ok(Some(child_tx)) + } + None => { + let current_balance = btc_client + .get_address_utxo(node_address) + .await? + .iter() + .map(|u| u.value) + .sum::(); + bail!(SpecialError::InsufficientBalance(format!( + "Not enough balance to complete the transaction, current_balance: {current_balance}" + ))) + } + } +} + +pub async fn broadcast_tx_with_cpfp( + btc_client: &BTCClient, + parent_tx: Transaction, + parent_tx_total_input_amount: Amount, +) -> Result<()> { + let anchor_output = p2a_output(); + let anchor_vout = parent_tx + .output + .iter() + .position(|output| output == &anchor_output) + .ok_or_else(|| anyhow!("cannot CPFP transaction without a P2A anchor output"))?; + if let Some(duplicate_vout) = parent_tx + .output + .iter() + .enumerate() + .skip(anchor_vout + 1) + .find_map(|(vout, output)| if output == &anchor_output { Some(vout) } else { None }) + { + bail!("transaction has multiple P2A anchor outputs at {anchor_vout} and {duplicate_vout}"); + }; + let child_tx = + build_cpfp_txns(btc_client, &parent_tx, anchor_vout as u64, parent_tx_total_input_amount) + .await?; + match child_tx { + Some(tx) => broadcast_package(btc_client, &[parent_tx, tx], true).await?, + None => broadcast_tx(btc_client, &parent_tx).await?, + }; + Ok(()) +} + +/// Returns: +/// - `Ok(None)` if given address does not have enough btc, +/// - `Ok(Some((utxos, fee_amount, change_amount)))` +pub async fn get_proper_utxo_set( + client: &BTCClient, + base_vbytes: u64, + address: Address, + target_amount: Amount, + fee_rate: f64, +) -> Result, Amount, Amount)>> { + fn estimate_tx_vbytes(base_vbytes: u64, extra_inputs: usize, extra_outputs: usize) -> u64 { + // p2wsh inputs/outputs + base_vbytes + + (extra_inputs as u64 * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE) + + (extra_outputs as u64 * P2WSH_OUTPUT_VBYTES) + } + fn to_input(utxos: Vec) -> Vec { + utxos + .into_iter() + .map(|utxo| Input { + outpoint: OutPoint { txid: utxo.txid, vout: utxo.vout }, + amount: utxo.value, + }) + .collect() + } + tracing::debug!("get utxos from: {address}"); + + let utxos = client.get_address_utxo(address).await?; + let mut sorted_utxos = utxos; + sorted_utxos.sort_by(|a, b| b.value.cmp(&a.value)); + + let mut selected = Vec::new(); + let mut total_value = Amount::ZERO; + + for utxo in sorted_utxos.into_iter().take(MAX_CUSTOM_INPUTS) { + selected.push(utxo.clone()); + total_value += utxo.value; + + let num_inputs = selected.len(); + let num_outputs = 1; // change + let tx_vbytes = estimate_tx_vbytes(base_vbytes, num_inputs, num_outputs); + let fee = Amount::from_sat((tx_vbytes as f64 * fee_rate).ceil() as u64); + + if total_value >= target_amount + fee { + let change = total_value - target_amount - fee; + return Ok(Some((to_input(selected), fee, change))); + } + } + + Ok(None) +} + +pub fn node_p2wsh_script(pubkey: &PublicKey) -> ScriptBuf { + script! { + { *pubkey } + OP_CHECKSIG + } + .compile() +} + +pub fn node_p2wsh_address(network: Network, pubkey: &PublicKey) -> Address { + Address::p2wsh(&node_p2wsh_script(pubkey), network) +} + +pub fn node_sign( + tx: &mut Transaction, + input_index: usize, + input_value: Amount, + sighash_type: EcdsaSighashType, + node_keypair: &Keypair, +) -> Result<()> { + let node_pubkey = node_keypair.public_key(); + populate_p2wsh_witness( + tx, + input_index, + sighash_type, + &node_p2wsh_script(&node_pubkey.into()), + input_value, + &vec![node_keypair], + ); + Ok(()) +} + +/// Whether the backend knows the transaction: confirmed, or still in its mempool. +pub async fn tx_visible(client: &BTCClient, txid: &Txid) -> Result { + match client.get_tx(txid).await? { + Some(_) => Ok(true), + _ => Ok(false), + } +} + +pub async fn tx_confirmed(client: &BTCClient, txid: &Txid) -> Result { + Ok(client.get_tx_status(txid).await?.confirmed) +} + +/// The confirmed spender of an outpoint. A spend that is only in the mempool can be replaced, +/// so nothing irreversible is decided on it. +pub async fn outpoint_spent_confirmed( + client: &BTCClient, + txid: &Txid, + vout: u64, +) -> Result> { + match outpoint_spent_txid(client, txid, vout).await? { + Some(spent_txid) if client.get_tx_status(&spent_txid).await?.confirmed => { + Ok(Some(spent_txid)) + } + _ => Ok(None), + } +} + +pub async fn outpoint_available(client: &BTCClient, txid: &Txid, vout: u64) -> Result { + match client.get_output_status(txid, vout).await? { + Some(status) => Ok(!status.spent), + _ => Ok(false), + } +} + +pub async fn outpoint_spent_txid( + client: &BTCClient, + txid: &Txid, + vout: u64, +) -> Result> { + match client.get_output_status(txid, vout).await? { + Some(status) => Ok(status.txid), + _ => Ok(None), + } +} + +pub async fn outpoint_spent_txin( + client: &BTCClient, + txid: &Txid, + vout: u64, +) -> Result> { + match client.get_output_status(txid, vout).await? { + Some(status) => { + if let Some(spent_txid) = status.txid + && let Some(vin) = status.vin + && let Some(spent_tx) = client.get_tx(&spent_txid).await? + { + Ok(spent_tx.input.get(vin as usize).cloned().map(|txin| (spent_txid, vin, txin))) + } else { + Ok(None) + } + } + _ => Ok(None), + } +} + +/// Returns: +/// - `Ok(true)` tx confirmed, +/// - `Ok(false)` tx not confirmed, exceeds the maximum waiting time +pub async fn wait_tx_confirmation( + btc_client: &BTCClient, + txid: &Txid, + interval: u64, + max_wait_secs: u64, +) -> Result { + use std::{ + thread, + time::{Duration, Instant}, + }; + let start_time = Instant::now(); + loop { + if start_time.elapsed().as_secs() > max_wait_secs { + // println!("Timeout: Transaction not confirmed after {} seconds", max_wait_secs); + return Ok(false); + }; + // FIXME: should not use esplora directly + match btc_client.get_tx_status(txid).await { + Ok(status) => { + if let Some(_height) = status.block_height { + // println!("Transaction confirmed in block {}", height); + return Ok(true); + } else { + // println!("Transaction unconfirmed, polling again..."); + } + } + Err(e) => { + bail!("Failed to fetch transaction status: {e}"); + } + } + thread::sleep(Duration::from_secs(interval)); + } +} + +pub async fn check_bridge_in_uxto_available_or_self_spent( + btc_client: &BTCClient, + target_txid: Option, + utxos: &[client::Utxo], +) -> Result { + for utxo in utxos { + if let Ok(txid) = Txid::from_slice(&utxo.txid) + && let Ok(Some(status)) = btc_client.get_output_status(&txid, utxo.vout as u64).await + && status.spent + { + if let Some(target_txid) = target_txid + && let Some(txid) = status.txid + && txid.to_string() == target_txid + { + return Ok(true); + } + return Ok(false); + } + } + Ok(true) +} diff --git a/node/src/utils/committee_keys.rs b/node/src/utils/committee_keys.rs new file mode 100644 index 000000000..1d1d646ea --- /dev/null +++ b/node/src/utils/committee_keys.rs @@ -0,0 +1,77 @@ +//! The committee instance keys on disk and this node's membership of an instance. + +use crate::env::*; +use anyhow::{Context, Result}; +use bitcoin::PublicKey; +use bitcoin::key::Keypair; +use bitvm_lib::keys::CommitteeMasterKey; +use client::goat_chain::GOATClient; +use std::path::PathBuf; + +use uuid::Uuid; + +/// Path of the committee instance key envelope for `instance_id`. +pub fn committee_instance_keys_envelope_path(instance_id: Uuid) -> PathBuf { + let mut envelope_path = PathBuf::from(COMMITTEE_INSTANCE_KEYS_DIR); + envelope_path.push(format!("{instance_id}.json")); + envelope_path +} + +/// Whether this node signs for an instance: it holds the instance key +/// envelope and that key is in the instance's committee on Goat. Nodes that +/// did not answer the peg-in request hold no envelope and take no part in +/// the instance's signing rounds. +pub enum CommitteeMembership { + Member(bitcoin::key::Keypair), + NotMember(String), +} + +pub async fn local_committee_membership( + goat_client: &GOATClient, + instance_id: Uuid, +) -> Result { + let envelope_path = committee_instance_keys_envelope_path(instance_id); + if !envelope_path.exists() { + return Ok(CommitteeMembership::NotMember("no instance key envelope".to_string())); + } + let committee_master_key = CommitteeMasterKey::new(get_bitvm_key()?); + let keypair = match load_committee_instance_keypair(&committee_master_key, instance_id) { + Ok(keypair) => keypair, + Err(error) if is_io_not_found_error(&error) => { + return Ok(CommitteeMembership::NotMember( + "instance key envelope vanished".to_string(), + )); + } + Err(error) => return Err(error), + }; + let pubkey: PublicKey = keypair.public_key().into(); + let committee = goat_client.gateway_get_committee_pubkeys(&instance_id).await?; + if !committee.contains(&pubkey) { + return Ok(CommitteeMembership::NotMember(format!( + "instance key {pubkey} is not in the committee on Goat" + ))); + } + Ok(CommitteeMembership::Member(keypair)) +} + +pub fn is_io_not_found_error(err: &anyhow::Error) -> bool { + err.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io_err| io_err.kind() == std::io::ErrorKind::NotFound) + }) +} + +pub(crate) fn load_committee_instance_keypair( + committee_master_key: &CommitteeMasterKey, + instance_id: Uuid, +) -> Result { + let envelope_path = committee_instance_keys_envelope_path(instance_id); + committee_master_key.load_instance_keypair(instance_id, &envelope_path).with_context(|| { + format!( + "load committee instance keypair failed for {} at {}", + instance_id, + envelope_path.display() + ) + }) +} diff --git a/node/src/utils/endorse.rs b/node/src/utils/endorse.rs new file mode 100644 index 000000000..b5c40d40d --- /dev/null +++ b/node/src/utils/endorse.rs @@ -0,0 +1,133 @@ +//! Endorsement signatures over graphs, graph parameters and peg-ins. + +use crate::env::*; +use alloy::primitives::{Address as EvmAddress, Signature as EvmSignature}; +use alloy::signers::Signer; +use alloy::signers::local::PrivateKeySigner; +use anyhow::{Result, anyhow}; +use bitcoin::key::Keypair; +use bitcoin::{PublicKey, Txid, XOnlyPublicKey}; +use bitvm_lib::types::{BitvmGcGraph, SimplifiedBitvmGcGraph}; +use client::goat_chain::GOATClient; +use secp256k1::schnorr::Signature as SchnorrSignature; +use secp256k1::{Message as SecpMessage, SECP256K1}; +use std::str::FromStr; + +use uuid::Uuid; + +use super::goat_chain::build_graph_data; + +pub async fn get_graph_digest(goat_client: &GOATClient, graph: &BitvmGcGraph) -> Result<[u8; 32]> { + let instance_id = graph.parameters.instance_parameters.instance_id; + let graph_id = graph.parameters.graph_id; + let graph_data = build_graph_data(graph)?; + goat_client.gateway_get_post_graph_digest(&instance_id, &graph_id, graph_data).await +} + +pub async fn endorse_graph(goat_client: &GOATClient, graph: &BitvmGcGraph) -> Result { + let signer = PrivateKeySigner::from_str(&get_node_goat_private_key()?)?; + let graph_digest = get_graph_digest(goat_client, graph).await?; + let sig = signer.sign_hash(&graph_digest.into()).await?; + Ok(sig) +} + +pub async fn endorse_graph_params(graph: &BitvmGcGraph) -> Result { + let signer = PrivateKeySigner::from_str(&get_node_goat_private_key()?)?; + let params_hash = graph.parameters.canonical_graph_params_hash()?; + let sig = signer.sign_hash(¶ms_hash.into()).await?; + Ok(sig) +} + +pub async fn endorse_pegin( + goat_client: &GOATClient, + instance_id: Uuid, + pegin_txid: &Txid, +) -> Result { + let signer = PrivateKeySigner::from_str(&get_node_goat_private_key()?)?; + let pegin_digest = goat_client.gateway_get_post_pegin_digest(&instance_id, pegin_txid).await?; + let sig = signer.sign_hash(&pegin_digest.into()).await?; + Ok(sig) +} + +pub async fn verify_pegin_endorsement( + goat_client: &GOATClient, + instance_id: Uuid, + committee_pubkey: &PublicKey, + pegin_txid: &Txid, + signature: &[u8], +) -> Result { + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + let expected_evm_address = pegin_data + .committee_pubkeys + .iter() + .zip(pegin_data.committee_addresses.iter()) + .find_map(|(pubkey, evm_address)| { + PublicKey::from_slice(pubkey) + .ok() + .filter(|on_chain_pubkey| on_chain_pubkey == committee_pubkey) + .map(|_| *evm_address) + }) + .ok_or_else(|| anyhow!("committee pubkey {committee_pubkey} not found on-chain"))?; + let pegin_digest = goat_client.gateway_get_post_pegin_digest(&instance_id, pegin_txid).await?; + let sig = EvmSignature::try_from(signature)?; + sig.recover_address_from_prehash(&pegin_digest.into()) + .map(|addr| addr == expected_evm_address) + .map_err(|e| e.into()) +} + +pub async fn verify_graph_endorsement( + goat_client: &GOATClient, + evm_address: &EvmAddress, + graph: &BitvmGcGraph, + signature: &[u8], +) -> Result { + let graph_digest = get_graph_digest(goat_client, graph).await?; + verify_graph_endorsement_digest(evm_address, &graph_digest, signature) +} + +/// Whether `signature` over the graph digest recovers to `evm_address`. +pub fn verify_graph_endorsement_digest( + evm_address: &EvmAddress, + graph_digest: &[u8; 32], + signature: &[u8], +) -> Result { + let sig = EvmSignature::try_from(signature)?; + sig.recover_address_from_prehash(&(*graph_digest).into()) + .map(|addr| &addr == evm_address) + .map_err(|e| e.into()) +} + +pub fn verify_graph_params_endorsement( + evm_address: &EvmAddress, + graph: &BitvmGcGraph, + signature: &[u8], +) -> Result { + let params_hash = graph.parameters.canonical_graph_params_hash()?; + let sig = EvmSignature::try_from(signature)?; + sig.recover_address_from_prehash(¶ms_hash.into()) + .map(|addr| &addr == evm_address) + .map_err(|e| e.into()) +} + +pub fn sign_verifier_graph_params( + verifier_keypair: Keypair, + graph: &SimplifiedBitvmGcGraph, +) -> Result { + let params_hash = graph.canonical_graph_params_hash()?; + Ok(SECP256K1.sign_schnorr(&SecpMessage::from_digest(params_hash), &verifier_keypair)) +} + +pub fn verify_verifier_graph_params_endorsement( + verifier_pubkey: &PublicKey, + graph: &SimplifiedBitvmGcGraph, + signature: &SchnorrSignature, +) -> Result { + let params_hash = graph.canonical_graph_params_hash()?; + Ok(SECP256K1 + .verify_schnorr( + signature, + &SecpMessage::from_digest(params_hash), + &XOnlyPublicKey::from(*verifier_pubkey), + ) + .is_ok()) +} diff --git a/node/src/utils/evm_swap_utils.rs b/node/src/utils/evm_swap_utils.rs new file mode 100644 index 000000000..e56ce68a3 --- /dev/null +++ b/node/src/utils/evm_swap_utils.rs @@ -0,0 +1,405 @@ +use alloy::primitives::Address as EvmAddress; +use bitcoin::consensus::encode::deserialize; +use bitcoin::{Transaction, Txid}; +use client::goat_chain::GOATClient; +use serde::{Deserialize, Serialize}; + +use crate::utils::evm_swap_utils::IEscrowManager::{EscrowData, IEscrowManagerCalls}; +use alloy::primitives::{B256, keccak256}; +use alloy::rpc::types::trace::geth::{CallConfig, CallFrame, GethDebugTracingOptions}; +use alloy::sol; +use alloy::sol_types::SolInterface; +use alloy::sol_types::SolValue; +#[cfg(test)] +use bitcoin::hashes::Hash; +#[cfg(test)] +use std::str::FromStr; +use tracing::warn; + +sol! { + interface IEscrowManager { + event Initialize(address indexed offerer, address indexed claimer, bytes32 indexed escrowHash, address claimHandler, address refundHandler); + event Claim(address indexed offerer, address indexed claimer, bytes32 indexed escrowHash, address claimHandler, bytes witnessResult); // for BitcoinNoncedOutputClaimHandler, Claim.witnessResult = payout_btc_txid + event Refund(address indexed offerer, address indexed claimer, bytes32 indexed escrowHash, address refundHandler, bytes witnessResult); + event ExecutionError(bytes32 indexed escrowHash, bytes error); + #[derive(Debug)] + struct EscrowData { + //Account funding the escrow + address offerer; + //Account entitled to claim the funds from the escrow + address claimer; + + //Amount of tokens in the escrow + uint256 amount; + //Token of the escrow + address token; + + //Misc escrow data flags, currently defined: payIn, payOut, reputation. + //It is recommended to randomize the other unused bits in the flags to act as a salt, + // such that no 2 escrow data are the same, even if all the other data in them match. + uint256 flags; + + //Address of the IClaimHandler deciding if this escrow is claimable + // use BitcoinNoncedOutputClaimHandler for Goat -> Bitcoin swaps + address claimHandler; + //Data provided to the claim handler along with the witness to check claimability + // for BitcoinNoncedOutputClaimHandler, this is the hash commitment of the claim data, see hash_claim_commitment + bytes32 claimData; + + //Address of the IRefundHandler deciding if this escrow is refundable + // use TimelockRefundHandler for Goat -> Bitcoin swaps + address refundHandler; + //Data provided to the refund handler along with the witness to check for refundability + // for TimelockRefundHandler, this is the timestamp after which refund is possible + bytes32 refundData; + + //Security deposit taken by the offerer if swap expires without claimer claiming (i.e. options premium) + uint256 securityDeposit; + //Claimer bounty that can be claimed by a 3rd party claimer if he were to claim this swap on behalf of claimer + uint256 claimerBounty; + //Deposit token of the swap used for securityDeposit and claimerBounty + address depositToken; + + //ExecutionAction hash commitment to be executed on claim, left 0x0 if no execution should happen on claim + bytes32 successActionCommitment; + } + function initialize(EscrowData calldata escrow, bytes calldata signature, uint256 timeout, bytes memory _extraData) external payable {} + function claim(EscrowData calldata escrow, bytes calldata witness) external {} + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct ClaimData { + pub txid: Txid, + pub nonce: u64, + pub output_amount: u64, + pub output_script: Vec, + pub confirmations: u32, + pub btc_relay_contract: EvmAddress, + pub witness: String, +} + +pub fn hash_escrow_data(escrow: &EscrowData) -> B256 { + let encoded = escrow.abi_encode(); + keccak256(&encoded) +} + +pub fn hash_claim_commitment(claim_data: &ClaimData) -> B256 { + // txoHash = keccak256(uint64 nonce || uint64 outputAmount || keccak256(bytes outputScript)) + // Commitment: C = abi.encodePacked(bytes32 txoHash, uint32 confirmations, address btcRelayContract) + // Witness: W = C || StoredBlockHeader blockheader || uint32 vout || bytes transaction || uint32 position || bytes32[] merkleProof + sol! { + interface IClaimHandlerHelper { + struct Txo { + uint64 nonce; + uint64 outputAmount; + bytes32 outputScriptHash; + } + struct ClaimCommitment { + bytes32 TxoHash; + uint32 confirmations; + address btcRelayContract; + } + } + } + let output_script_hash = keccak256(&claim_data.output_script); + let txo = IClaimHandlerHelper::Txo { + nonce: claim_data.nonce, + outputAmount: claim_data.output_amount, + outputScriptHash: output_script_hash, + }; + let txo_hash = keccak256(txo.abi_encode_packed()); + let claim_commitment = IClaimHandlerHelper::ClaimCommitment { + TxoHash: txo_hash, + confirmations: claim_data.confirmations, + btcRelayContract: claim_data.btc_relay_contract, + }; + keccak256(claim_commitment.abi_encode_packed()) +} + +fn find_escrow_data( + call: &CallFrame, + swap_contract_address: &EvmAddress, + escrow_hash: &[u8; 32], +) -> anyhow::Result> { + if call.to == Some(*swap_contract_address) + && let Ok(calldata) = IEscrowManagerCalls::abi_decode(&call.input) + && let IEscrowManagerCalls::initialize(args) = calldata + { + let computed_hash = hash_escrow_data(&args.escrow); + if &computed_hash.0 == escrow_hash { + return Ok(Some(args.escrow)); + } + } + + for sub_call in &call.calls { + if let Some(escrow_data) = find_escrow_data(sub_call, swap_contract_address, escrow_hash)? { + return Ok(Some(escrow_data)); + } + } + Ok(None) +} + +pub async fn extract_escrow_data_from_tx( + goat_client: &GOATClient, + tx_hash: &str, + swap_contract_address: &EvmAddress, + escrow_hash: &[u8; 32], +) -> anyhow::Result> { + let trace_opts = GethDebugTracingOptions::call_tracer(CallConfig::default()); + let trace_raw = goat_client.debug_trace_tx(tx_hash, Some(trace_opts)).await?; + let call_trace = trace_raw.try_into_call_frame()?; + if let Some(escrow_data) = find_escrow_data(&call_trace, swap_contract_address, escrow_hash)? { + Ok(Some(escrow_data)) + } else { + Ok(None) + } +} + +fn claim_data_from_witness(witness: &[u8]) -> anyhow::Result { + // txoHash = keccak256(uint64 nonce || uint64 outputAmount || keccak256(bytes outputScript)) + // Witness: W = bytes32 txoHash + // || uint32 confirmations + // || address btcRelayContract + // || StoredBlockHeader(160-bytes) blockheader + // || uint32 vout + // || bytes transaction (32-byte length prefix + data) + // || uint32 position || bytes32[] merkleProof + // claimData.nonce = or(shl(24, locktimeSub500M), and(firstNSequence, 0x00FFFFFF)) + // claimData.output_amount = W.transaction.outputs[vout].value + // claimData.output_script = W.transaction.outputs[vout].scriptPubKey + + let mut offset = 0; + + // 1. txoHash (32 bytes) + if witness.len() < 32 { + anyhow::bail!("witness too short for txoHash"); + } + offset += 32; + + // 2. confirmations (4 bytes) + if witness.len() < offset + 4 { + anyhow::bail!("witness too short for confirmations"); + } + let confirmations = u32::from_be_bytes(witness[offset..offset + 4].try_into()?); + offset += 4; + + // 3. btcRelayContract (20 bytes) + if witness.len() < offset + 20 { + anyhow::bail!("witness too short for btcRelayContract"); + } + let btc_relay_contract = EvmAddress::from_slice(&witness[offset..offset + 20]); + offset += 20; + + // 4. StoredBlockHeader (160 bytes) + if witness.len() < offset + 160 { + anyhow::bail!("witness too short for blockheader"); + } + offset += 160; + + // 5. vout (4 bytes) + if witness.len() < offset + 4 { + anyhow::bail!("witness too short for vout"); + } + let vout = u32::from_be_bytes(witness[offset..offset + 4].try_into()?); + offset += 4; + + // 6. transaction (32-byte length prefix + data) + if witness.len() < offset + 32 { + anyhow::bail!("witness too short for transaction length"); + } + let tx_len = alloy::primitives::U256::from_be_slice(&witness[offset..offset + 32]); + offset += 32; + + let tx_len_usize: usize = + tx_len.try_into().map_err(|_| anyhow::anyhow!("tx length too large"))?; + + if witness.len() < offset + tx_len_usize { + anyhow::bail!("witness too short for transaction data"); + } + let tx_bytes = &witness[offset..offset + tx_len_usize]; + + let tx: Transaction = deserialize(tx_bytes)?; + + if vout as usize >= tx.output.len() { + anyhow::bail!("vout index out of bounds"); + } + let output = &tx.output[vout as usize]; + + if tx.input.is_empty() { + anyhow::bail!("transaction has no inputs"); + } + let first_input_sequence = tx.input[0].sequence.to_consensus_u32(); + + let lock_time = tx.lock_time.to_consensus_u32(); + let lock_time_val = if lock_time >= 500_000_000 { lock_time - 500_000_000 } else { lock_time }; + + let nonce = ((lock_time_val as u64) << 24) | ((first_input_sequence as u64) & 0x00ffffff); + + Ok(ClaimData { + txid: tx.compute_txid(), + nonce, + output_amount: output.value.to_sat(), + output_script: output.script_pubkey.to_bytes(), + confirmations, + btc_relay_contract, + witness: hex::encode(witness), + }) +} + +// for BitcoinNoncedOutputClaimHandler +fn find_claim_data( + tx_hash: &str, + call: &CallFrame, + swap_contract_address: &EvmAddress, + escrow_hash: &[u8; 32], +) -> anyhow::Result> { + if call.to == Some(*swap_contract_address) + && let Ok(calldata) = IEscrowManagerCalls::abi_decode(&call.input) + && let IEscrowManagerCalls::claim(args) = calldata + { + let computed_hash = hash_escrow_data(&args.escrow); + if &computed_hash.0 == escrow_hash { + return match claim_data_from_witness(&args.witness) { + Ok(claim_data) => Ok(Some(claim_data)), + Err(e) => { + warn!("fail to decode claim data for tx: {tx_hash}, error:{e}"); + Ok(None) + } + }; + } + } + + for sub_call in &call.calls { + if let Some(claim_data) = + find_claim_data(tx_hash, sub_call, swap_contract_address, escrow_hash)? + { + return Ok(Some(claim_data)); + } + } + Ok(None) +} + +pub async fn extract_claim_data_from_tx( + goat_client: &GOATClient, + tx_hash: &str, + swap_contract_address: &EvmAddress, + escrow_hash: &[u8; 32], +) -> anyhow::Result> { + let trace_opts = GethDebugTracingOptions::call_tracer(CallConfig::default()); + let trace_raw = goat_client.debug_trace_tx(tx_hash, Some(trace_opts)).await?; + let call_trace = trace_raw.try_into_call_frame()?; + if let Some(claim_data) = + find_claim_data(tx_hash, &call_trace, swap_contract_address, escrow_hash)? + { + Ok(Some(claim_data)) + } else { + Ok(None) + } +} + +#[tokio::test] +async fn test_find_escrow_data() { + unsafe { + std::env::set_var(crate::env::ENV_GOAT_CHAIN_URL, "https://rpc.testnet3.goat.network"); + } + let tx_hash = "0x6027024dc57b847120074efed67c3e31534988f70b6b1e5043b248e7740a295c"; + let goat_client = GOATClient::new( + crate::env::goat_config_from_env().await, + client::goat_chain::GoatNetwork::Test, + ); + let swap_contract_address = + EvmAddress::from_str("0xe510D5781C6C849284Fb25Dc20b1684cEC445C8B").unwrap(); + let escrow_hash: [u8; 32] = + hex::decode("521a1d007f9fdf41b18ad6f1ccfeaf8fd67d0b04608ce3d8950526e55e4eca28") + .unwrap() + .try_into() + .unwrap(); + let escrow_data = + extract_escrow_data_from_tx(&goat_client, tx_hash, &swap_contract_address, &escrow_hash) + .await + .unwrap(); + assert!(escrow_data.is_some()); + let expected_claim_hash = + B256::from_str("0xc69e4a62e0c904b341245656ba191790356d771bd4a7a00bed2780a5abad8c63") + .unwrap(); + assert_eq!(escrow_data.unwrap().claimData, expected_claim_hash); +} + +#[test] +fn test_hash_claim_commitment() { + // example data from an actual swap on testnet: + // goat initialize txid: 0x6027024dc57b847120074efed67c3e31534988f70b6b1e5043b248e7740a295c + // goat claim txid: 0xc2b26508a28f349c7ee1e189914dc5815b77d1abaa5ce6a60449f69bd1e7e64a + // btc payout txid: 033d4024aca7f6dda6b01e7f0a2bb0fdd15160cc9b2559b55c6f65962362d74e + let claim_data = ClaimData { + txid: Txid::from_slice(&[0_u8; 32]).unwrap(), + nonce: 17872110975047329u64, + output_amount: 9511u64, + output_script: hex::decode( + "5120a5d06cb76aaf6287b93a8ee73d9678e32b039354e6df4019bbd60087e347f5cc", + ) + .unwrap(), + confirmations: 2u32, + btc_relay_contract: EvmAddress::from_str("0x3887B02217726bB36958Dd595e57293fB63D5082") + .unwrap(), + witness: "".to_string(), + }; + let commitment_hash = hash_claim_commitment(&claim_data); + + let expected_hash = + B256::from_str("0xc69e4a62e0c904b341245656ba191790356d771bd4a7a00bed2780a5abad8c63") + .unwrap(); + assert_eq!(commitment_hash, expected_hash); +} + +#[tokio::test] +async fn test_find_claim_data() { + unsafe { + std::env::set_var(crate::env::ENV_GOAT_CHAIN_URL, "https://rpc.testnet3.goat.network"); + } + let tx_hash = "0xc2b26508a28f349c7ee1e189914dc5815b77d1abaa5ce6a60449f69bd1e7e64a"; + let goat_client = GOATClient::new( + crate::env::goat_config_from_env().await, + client::goat_chain::GoatNetwork::Test, + ); + let swap_contract_address = + EvmAddress::from_str("0xe510D5781C6C849284Fb25Dc20b1684cEC445C8B").unwrap(); + let escrow_hash: [u8; 32] = + hex::decode("521a1d007f9fdf41b18ad6f1ccfeaf8fd67d0b04608ce3d8950526e55e4eca28") + .unwrap() + .try_into() + .unwrap(); + let expected_claim_data = ClaimData { + txid: Txid::from_str("033d4024aca7f6dda6b01e7f0a2bb0fdd15160cc9b2559b55c6f65962362d74e") + .unwrap(), + nonce: 17872110975047329u64, + output_amount: 9511u64, + output_script: hex::decode( + "5120a5d06cb76aaf6287b93a8ee73d9678e32b039354e6df4019bbd60087e347f5cc", + ) + .unwrap(), + confirmations: 2u32, + btc_relay_contract: EvmAddress::from_str("0x3887B02217726bB36958Dd595e57293fB63D5082") + .unwrap(), + witness: "8977831297fa2d7156898a8d26bb9e276a83cf907d739eb64ff98a0092e9250e000000023887b022\ + 17726bb36958dd595e57293fb63d508200600020bfe2760399ccb567289b120361316911b13e937aa0f2742bb7\ + 0b000000000000efaf19dc26fadb8b1cd69e76c6d1f51123f6d22407755f865b65b446cf863c3c5fbe3769f0ff\ + 0f1ad88a60f10000000000000000000000000000000000000000000017b6e253602b5f4cc25000494e876937b2\ + 636937bd246937bd2a6937bd2d6937bd306937bd6f6937bd8f6937bda66937bde76937bdf96937be07000000000\ + 00000000000000000000000000000000000000000000000000000000000008902000000018c3db50ef29dbdddf9\ + 628cd968667688cd37560ef5e64131efc3a1e4cf4e739c0100000000a1d60dfe022725000000000000225120a5d\ + 06cb76aaf6287b93a8ee73d9678e32b039354e6df4019bbd60087e347f5ccf61a010000000000225120eb3e0c2d\ + d6b344c6efaa771306a29e864c65d536dfac8f89a87680285af7acad1afc4b5d000000060000000000000000000\ + 000000000000000000000000000000000000000000003f7cfe6eed4929eea954c8e1358704c6dec19e2826f7223\ + ff7d0eff92c1addf20fd2b876e05846a100c38bd3c30619c66437abb97bc5fc21e2595fbb8a514f5341ebf1f8dd\ + 26752879de5c581667aaf183b18c1b5a111f4a24061d2c44aea2fd1" + .to_string(), + }; + let claim_data = + extract_claim_data_from_tx(&goat_client, tx_hash, &swap_contract_address, &escrow_hash) + .await + .unwrap(); + assert!(claim_data.is_some()); + assert_eq!(claim_data.unwrap(), expected_claim_data); +} diff --git a/node/src/utils/goat_chain.rs b/node/src/utils/goat_chain.rs new file mode 100644 index 000000000..81ada15e6 --- /dev/null +++ b/node/src/utils/goat_chain.rs @@ -0,0 +1,275 @@ +//! Reads from the Goat gateway: committees, peg-in requests, instances, graph +//! status and the posted graph data. + +use crate::env::*; +use crate::error::SpecialError; +use alloy::primitives::Address as EvmAddress; +use anyhow::{Result, anyhow, bail}; +use bitcoin::hashes::Hash; +use bitcoin::{Address, Amount, OutPoint, PublicKey, Txid, XOnlyPublicKey}; +use bitvm_lib::types::{BitvmGcGraph, BitvmGcInstanceParameters, UserInfo}; +use client::{btc_chain::BTCClient, goat_chain::GOATClient}; +use goat::contexts::base::generate_n_of_n_public_key; +use goat::transactions::base::Input; +use libp2p::PeerId; +use std::str::FromStr; +use std::time::Instant; +use store::localdb::LocalDB; + +use bitvm_lib::transactions::base::BaseTransaction; +use client::goat_chain::{GraphData, PeginStatus}; +use store::GraphStatus; +use uuid::Uuid; + +use super::btc::outpoint_available; + +pub fn build_graph_data(graph: &BitvmGcGraph) -> Result { + // operator pubkey: first byte is prefix, next 32 bytes are key + let op_pk_bytes = graph.parameters.operator_pubkey.to_bytes(); + let operator_pubkey_prefix = op_pk_bytes[0]; + let operator_pubkey: [u8; 32] = + op_pk_bytes[1..33].try_into().map_err(|_| anyhow!("invalid operator pubkey length"))?; + + // compute txids for all required transactions + let pegin_txid = graph.pegin.finalize().compute_txid().to_byte_array(); + let kickoff_txid = graph.kickoff.finalize().compute_txid().to_byte_array(); + let take1_txid = graph.take1.finalize().compute_txid().to_byte_array(); + let take2_txid = graph.take2.finalize().compute_txid().to_byte_array(); + let watchtower_challenge_init_txid = + graph.watchtower_challenge_init.finalize().compute_txid().to_byte_array(); + let prover_assert_txid = graph.operator_assert.finalize().compute_txid().to_byte_array(); + let disprove_txids: Vec<[u8; 32]> = + graph.disproves.iter().map(|tx| tx.finalize().compute_txid().to_byte_array()).collect(); + let watchtower_challenge_timeout_txids: Vec<[u8; 32]> = graph + .watchtower_challenge_timeouts + .iter() + .map(|tx| tx.finalize().compute_txid().to_byte_array()) + .collect(); + let operator_challenge_nack_txids: Vec<[u8; 32]> = graph + .operator_challenge_nacks + .iter() + .map(|tx| tx.finalize().compute_txid().to_byte_array()) + .collect(); + let operator_commit_timeout_txid = + graph.operator_commit_timeout.finalize().compute_txid().to_byte_array(); + + Ok(GraphData { + operator_pubkey_prefix, + operator_pubkey, + pegin_txid, + kickoff_txid, + take1_txid, + take2_txid, + watchtower_challenge_init_txid, + prover_assert_txid, + disprove_txids, + watchtower_challenge_timeout_txids, + operator_challenge_nack_txids, + operator_commit_timeout_txid, + }) +} + +pub async fn validate_committee( + goat_client: &GOATClient, + peer_id: &PeerId, + instance_id: Uuid, + committee_pubkey: &PublicKey, +) -> Result<()> { + // return SpecialError::InvalidCommittee if not valid + let registry = crate::p2p_admission::peer_registry(); + // A peer the registry has fresh word is unregistered gets no chain reads. + if registry.is_known_unregistered(peer_id, Instant::now()) { + bail!(SpecialError::InvalidCommittee("author is not a registered peer".to_string())); + } + // A peer the registry holds as committee whose binding on Goat does not + // match yet (a re-registered peer id, or a lagging Goat node) is asked + // again later instead of being refused for good; the job's deadline + // bounds the retries. Anyone else is refused. + let refused = |detail: &str| -> anyhow::Error { + if registry.is_committee(peer_id, Instant::now()) { + anyhow!("committee binding of {peer_id} on Goat does not match yet: {detail}") + } else { + anyhow!(SpecialError::InvalidCommittee(detail.to_string())) + } + }; + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + for (i, pk) in pegin_data.committee_pubkeys.iter().enumerate() { + let pk = PublicKey::from_slice(pk)?; + if &pk == committee_pubkey { + let addr = pegin_data.committee_addresses[i]; + let stored_peer_id = goat_client.committee_mana_get_committee_peer_id(&addr).await?; + if stored_peer_id.to_vec() != peer_id.to_bytes() { + return Err(refused("committee pubkey & peer id mismatch")); + } + return Ok(()); + } + } + Err(refused("committee pubkey not found in instance's committee pubkeys")) +} + +pub async fn validate_committee_with_evm_address( + goat_client: &GOATClient, + peer_id: &PeerId, + instance_id: Uuid, + committee_pubkey: &PublicKey, + committee_evm_address: &EvmAddress, +) -> Result<()> { + // return SpecialError::InvalidCommittee if not valid + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + for i in 0..pegin_data.committee_pubkeys.len() { + let pk = PublicKey::from_slice(&pegin_data.committee_pubkeys[i])?; + let addr = &pegin_data.committee_addresses[i]; + if addr == committee_evm_address { + if &pk != committee_pubkey { + bail!(SpecialError::InvalidCommittee( + "committee evm address & pubkey mismatch".to_string() + )); + } + let stored_peer_id = goat_client.committee_mana_get_committee_peer_id(addr).await?; + if stored_peer_id.to_vec() != peer_id.to_bytes() { + bail!(SpecialError::InvalidCommittee( + "committee evm address & peer id mismatch".to_string() + )); + } + return Ok(()); + } + } + bail!(SpecialError::InvalidCommittee( + "committee evm address not found in instance's committee addresses".to_string() + )) +} + +pub async fn read_pegin_request( + btc_client: &BTCClient, + goat_client: &GOATClient, + instance_id: Uuid, +) -> Result<(UserInfo, Amount, Vec)> { + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + if pegin_data.status != PeginStatus::Pending { + bail!(SpecialError::InvalidPeginRequest(format!( + "pegin request is {} on Goat: expired or already processed", + pegin_data.status + ))); + } + let network = get_network(); + let user_change_address = Address::from_str(&pegin_data.user_change_addr) + .map_err(|e| SpecialError::InvalidPeginRequest(format!("invalid user_change_addr: {e}")))? + .require_network(network) + .map_err(|e| { + SpecialError::InvalidPeginRequest(format!("invalid user_change_addr network: {e}")) + })?; + let user_refund_address = Address::from_str(&pegin_data.user_refund_addr) + .map_err(|e| SpecialError::InvalidPeginRequest(format!("invalid user_refund_addr: {e}")))? + .require_network(network) + .map_err(|e| { + SpecialError::InvalidPeginRequest(format!("invalid user_refund_addr network: {e}")) + })?; + let user_xonly_pubkey = + XOnlyPublicKey::from_slice(&pegin_data.user_xonly_pubkey).map_err(|e| { + SpecialError::InvalidPeginRequest(format!("invalid user_xonly_pubkey: {e}")) + })?; + let inputs: Vec = pegin_data + .user_inputs + .iter() + .map(|u| Input { + outpoint: OutPoint { txid: Txid::from_byte_array(u.txid), vout: u.vout }, + amount: Amount::from_sat(u.amount_sats), + }) + .collect(); + // TODO: we need to run our own bitcoin node in case of downtime or ddos attack. + for input in &inputs { + if !outpoint_available(btc_client, &input.outpoint.txid, input.outpoint.vout.into()).await? + { + bail!(SpecialError::InvalidPeginRequest(format!( + "input {}:{} is not available", + input.outpoint.txid, input.outpoint.vout + ))); + } + } + let user_info = UserInfo { + depositor_evm_address: pegin_data.depositor_address, + txn_fees: pegin_data.txn_fees, + inputs, + user_change_address, + user_refund_address, + user_xonly_pubkey, + }; + Ok((user_info, Amount::from_sat(pegin_data.pegin_amount_sats), pegin_data.committee_addresses)) +} + +pub async fn read_instance_info_from_goat( + goat_client: &GOATClient, + instance_id: Uuid, +) -> Result { + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + let network = get_network(); + let user_change_address = Address::from_str(&pegin_data.user_change_addr) + .map_err(|e| SpecialError::InvalidPeginData(format!("invalid user_change_addr: {e}")))? + .require_network(network) + .map_err(|e| { + SpecialError::InvalidPeginData(format!("invalid user_change_addr network: {e}")) + })?; + let user_refund_address = Address::from_str(&pegin_data.user_refund_addr) + .map_err(|e| SpecialError::InvalidPeginData(format!("invalid user_refund_addr: {e}")))? + .require_network(network) + .map_err(|e| { + SpecialError::InvalidPeginData(format!("invalid user_refund_addr network: {e}")) + })?; + let user_xonly_pubkey = XOnlyPublicKey::from_slice(&pegin_data.user_xonly_pubkey) + .map_err(|e| SpecialError::InvalidPeginData(format!("invalid user_xonly_pubkey: {e}")))?; + let inputs: Vec = pegin_data + .user_inputs + .iter() + .map(|u| Input { + outpoint: OutPoint { txid: Txid::from_byte_array(u.txid), vout: u.vout }, + amount: Amount::from_sat(u.amount_sats), + }) + .collect(); + let user_info = UserInfo { + depositor_evm_address: pegin_data.depositor_address, + txn_fees: pegin_data.txn_fees, + inputs, + user_change_address, + user_refund_address, + user_xonly_pubkey, + }; + let committee_pubkeys = match goat_client.gateway_get_committee_pubkeys(&instance_id).await { + Ok(pks) => pks, + Err(e) if crate::error::is_contract_revert(&e) => { + bail!(SpecialError::InvalidPeginData(format!("fail to get committee pubkeys: {e:#}"))) + } + Err(e) => bail!(e), + }; + let committee_agg_pubkey = generate_n_of_n_public_key(&committee_pubkeys).0; + Ok(BitvmGcInstanceParameters { + network, + instance_id, + user_info, + pegin_amount: Amount::from_sat(pegin_data.pegin_amount_sats), + committee_pubkeys, + committee_agg_pubkey, + }) +} + +pub async fn get_graph_status( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_process = local_db.acquire().await?; + let graph_op = storage_process.find_graph(&graph_id).await?; + if graph_op.is_none() { + return Ok(None); + }; + let graph = graph_op.unwrap(); + if graph.instance_id.ne(&instance_id) { + bail!( + "grap with graph_id:{graph_id} has instance_id:{} not match exp instance:{instance_id}", + graph.instance_id, + ); + } + Ok(Some( + GraphStatus::from_str(&graph.status) + .map_err(|_| anyhow!("unknown graph status: {}", graph.status))?, + )) +} diff --git a/node/src/utils/graph_chain_state.rs b/node/src/utils/graph_chain_state.rs new file mode 100644 index 000000000..623eadabf --- /dev/null +++ b/node/src/utils/graph_chain_state.rs @@ -0,0 +1,455 @@ +//! What both chains say about a graph: its stage and challenge branches. + +use anyhow::{Result, anyhow, bail}; +use bitcoin::{OutPoint, Txid}; +use bitvm_lib::types::BitvmGcGraph; +use client::{btc_chain::BTCClient, goat_chain::GOATClient}; +use goat::transactions::base::output_topology; +use goat::transactions::pre_signed::PreSignedTransaction; +use store::localdb::{GraphRuntimeUpdate, LocalDB}; + +use crate::aggregates::graph_scan::{ChallengeSubStatus, VerifierChallengeStatus}; +use client::goat_chain::DisproveTxType; +use store::{GraphStatus, GraphStatusTransitionOutcome}; +use tracing::warn; +use uuid::Uuid; + +use super::btc::{outpoint_spent_confirmed, outpoint_spent_txid, tx_confirmed, tx_visible}; +use super::graph_store::update_graph_status; + +/// Re-derive the graph's stage from both chains and record it +/// (ChainReconcile); a stale stage is ignored with a warning. +pub(crate) async fn refresh_graph( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + instance_id: Uuid, + graph_id: Uuid, + graph: &BitvmGcGraph, +) -> Result<()> { + if graph.parameters.instance_parameters.instance_id != instance_id + || graph.parameters.graph_id != graph_id + { + bail!( + "refuse to refresh graph {instance_id}:{graph_id} with mismatched graph parameters {}:{}", + graph.parameters.instance_parameters.instance_id, + graph.parameters.graph_id + ); + } + + let scan = scan_graph_chain_state(btc_client, goat_client, graph).await?; + let outcome = update_graph_status( + local_db, + instance_id, + graph_id, + scan.status, + Some(scan.sub_status.clone()), + ) + .await?; + match outcome { + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent => { + if let Some(challenge_txid) = scan.challenge_txid { + update_graph_challenge_txid_if_needed(local_db, graph_id, challenge_txid).await?; + } + } + GraphStatusTransitionOutcome::Rejected { current } => warn!( + "ignore stale chain scan for graph {instance_id}:{graph_id}: candidate={}, current={current}", + scan.status + ), + GraphStatusTransitionOutcome::NotFound => { + warn!("graph {instance_id}:{graph_id} disappeared while applying chain scan") + } + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub(crate) struct DetectedDisprove { + pub(crate) disprove_type: DisproveTxType, + pub(crate) index: usize, + pub(crate) challenge_finish_txid: Txid, +} + +/// What both chains say about a graph's transactions: the stage it reached +/// and, in a challenge, how far each branch got. +#[derive(Clone, Debug)] +pub(crate) struct GraphChainScan { + pub(crate) status: GraphStatus, + pub(crate) sub_status: ChallengeSubStatus, + pub(crate) challenge_txid: Option, + pub(crate) disprove: Option, +} + +#[cfg(test)] +impl GraphChainScan { + pub(crate) fn for_test(status: GraphStatus) -> Self { + Self { + status, + sub_status: ChallengeSubStatus::default(), + challenge_txid: None, + disprove: None, + } + } +} + +fn initial_challenge_sub_status(watchtower_num: usize, verifier_num: usize) -> ChallengeSubStatus { + let mut sub_status = ChallengeSubStatus::default(); + sub_status.watchtower_challenge_status.resize(watchtower_num, false); + sub_status.verifier_challenge_status.resize(verifier_num, VerifierChallengeStatus::None); + sub_status +} + +fn connector_a_outpoint(graph: &BitvmGcGraph) -> Result { + graph + .kickoff + .connector_a_input() + .map(|input| input.outpoint) + .map_err(|e| anyhow!("failed to get connector-a input: {e}")) +} + +fn connector_d_outpoint(graph: &BitvmGcGraph) -> Result { + graph + .operator_assert + .connector_d_input() + .map(|input| input.outpoint) + .map_err(|e| anyhow!("failed to get connector-d input: {e}")) +} + +async fn update_graph_challenge_txid_if_needed( + local_db: &LocalDB, + graph_id: Uuid, + challenge_txid: Txid, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let Some(graph) = storage_processor.find_graph(&graph_id).await? else { + warn!("graph: {graph_id} not found, skip updating challenge txid"); + return Ok(()); + }; + if graph.challenge_txid.as_ref().map(|txid| txid.0) != Some(challenge_txid) { + storage_processor + .update_graph_runtime( + &GraphRuntimeUpdate::new(graph.instance_id, graph_id) + .with_challenge_txid(challenge_txid.into()), + ) + .await?; + } + Ok(()) +} + +async fn detect_guardian_disprove( + btc_client: &BTCClient, + graph: &BitvmGcGraph, +) -> Result> { + let quick_challenge_txid = graph.quick_challenge.tx().compute_txid(); + if tx_confirmed(btc_client, &quick_challenge_txid).await? { + return Ok(Some(DetectedDisprove { + disprove_type: DisproveTxType::QuickChallenge, + index: 0, + challenge_finish_txid: quick_challenge_txid, + })); + } + + let challenge_incomplete_kickoff_txid = graph.challenge_incomplete_kickoff.tx().compute_txid(); + if tx_confirmed(btc_client, &challenge_incomplete_kickoff_txid).await? { + return Ok(Some(DetectedDisprove { + disprove_type: DisproveTxType::ChallengeIncompleteKickoff, + index: 0, + challenge_finish_txid: challenge_incomplete_kickoff_txid, + })); + } + Ok(None) +} + +async fn detect_connector_d_disprove( + btc_client: &BTCClient, + graph: &BitvmGcGraph, +) -> Result> { + let connector_d = connector_d_outpoint(graph)?; + let Some(spent_txid) = + outpoint_spent_confirmed(btc_client, &connector_d.txid, connector_d.vout as u64).await? + else { + return Ok(None); + }; + + if spent_txid == graph.take2.tx().compute_txid() { + return Ok(None); + } + + for (index, disprove) in graph.disproves.iter().enumerate() { + if spent_txid == disprove.tx().compute_txid() { + return Ok(Some(DetectedDisprove { + disprove_type: DisproveTxType::Disprove, + index, + challenge_finish_txid: spent_txid, + })); + } + } + + Ok(Some(DetectedDisprove { + disprove_type: DisproveTxType::PubinDisprove, + index: 0, + challenge_finish_txid: spent_txid, + })) +} + +async fn detect_watchtower_flow_disprove( + btc_client: &BTCClient, + graph: &BitvmGcGraph, +) -> Result> { + for (index, nack) in graph.operator_challenge_nacks.iter().enumerate() { + let txid = nack.tx().compute_txid(); + if tx_confirmed(btc_client, &txid).await? { + return Ok(Some(DetectedDisprove { + disprove_type: DisproveTxType::OperatorChallengeNack, + index, + challenge_finish_txid: txid, + })); + } + } + + let txid = graph.operator_commit_timeout.tx().compute_txid(); + if tx_confirmed(btc_client, &txid).await? { + return Ok(Some(DetectedDisprove { + disprove_type: DisproveTxType::OperatorCommitTimeout, + index: 0, + challenge_finish_txid: txid, + })); + } + + Ok(None) +} + +pub(crate) async fn scan_graph_chain_state( + btc_client: &BTCClient, + goat_client: &GOATClient, + graph: &BitvmGcGraph, +) -> Result { + let graph_id = graph.parameters.graph_id; + let watchtower_num = graph.parameters.watchtower_pubkeys.len(); + let verifier_num = graph.verifier_asserts.len(); + let mut sub_status = initial_challenge_sub_status(watchtower_num, verifier_num); + // Derive the candidate from the graph and both chains, rather than using + // the locally persisted status as the scan start. The stored status is a + // projection and can be stale after a restart or missed listener event. + let mut current_status = if graph.committee_pre_signed() { + GraphStatus::CommitteePresigned + } else { + GraphStatus::OperatorPresigned + }; + + let prekickoff_txid = graph.cur_prekickoff.tx().compute_txid(); + let kickoff_txid = graph.kickoff.tx().compute_txid(); + let take1_txid = graph.take1.tx().compute_txid(); + let take2_txid = graph.take2.tx().compute_txid(); + + // GraphData is a Goat-chain fact and must be checked regardless of the + // current local projection. A fully synced node may otherwise remain at + // OperatorPresigned forever after a prior status regression. + let graph_data_on_goat = goat_client.gateway_get_graph_data(&graph_id).await?; + if graph_data_on_goat.operator_pubkey != [0u8; 32] { + current_status = GraphStatus::OperatorDataPushed; + } + // Derive stages from confirmed chain progress, not local Obsoleted state. + // Mempool transactions can be replaced and must not decide a terminal stage. + if !tx_confirmed(btc_client, &prekickoff_txid).await? { + return Ok(GraphChainScan { + status: current_status, + sub_status, + challenge_txid: None, + disprove: None, + }); + } + // check Kickoff/SkipKickoff. Only a confirmed spend decides: Skipped is + // closed and never scanned again, so a Skip-Kickoff that is only in the + // mempool must not hide the Kickoff that replaces it, and a mempool + // Kickoff replaced by the Skip must not strand the graph at OperatorKickOff. + let kickoff_connector_vout = output_topology::prekickoff::kickoff_connector() as u64; + let kickoff_connector_spent = + outpoint_spent_confirmed(btc_client, &prekickoff_txid, kickoff_connector_vout).await?; + match kickoff_connector_spent { + Some(spent_txid) if spent_txid != kickoff_txid => { + return Ok(GraphChainScan { + status: GraphStatus::Skipped, + sub_status, + challenge_txid: None, + disprove: None, + }); + } + Some(_) => current_status = GraphStatus::OperatorKickOff, + None => { + return Ok(GraphChainScan { + status: GraphStatus::PreKickoff, + sub_status, + challenge_txid: None, + disprove: None, + }); + } + } + + let mut challenge_txid = None; + if current_status == GraphStatus::OperatorKickOff + && let Some(disprove) = detect_guardian_disprove(btc_client, graph).await? + { + sub_status.disprove_type = Some(disprove.disprove_type); + sub_status.disprove_index = disprove.index as i32; + return Ok(GraphChainScan { + status: GraphStatus::Disprove, + sub_status, + challenge_txid, + disprove: Some(disprove), + }); + } + if current_status == GraphStatus::OperatorKickOff { + let connector_a = connector_a_outpoint(graph)?; + if let Some(spent_txid) = + outpoint_spent_confirmed(btc_client, &connector_a.txid, connector_a.vout as u64).await? + { + if spent_txid != take1_txid { + current_status = GraphStatus::Challenge; + challenge_txid = Some(spent_txid); + } else { + return Ok(GraphChainScan { + status: GraphStatus::OperatorTake1, + sub_status, + challenge_txid: None, + disprove: None, + }); + } + } else { + return Ok(GraphChainScan { + status: GraphStatus::OperatorKickOff, + sub_status, + challenge_txid: None, + disprove: None, + }); + } + } + + if current_status == GraphStatus::Challenge { + if let Some(disprove) = detect_guardian_disprove(btc_client, graph).await? { + sub_status.disprove_type = Some(disprove.disprove_type); + sub_status.disprove_index = disprove.index as i32; + return Ok(GraphChainScan { + status: GraphStatus::Disprove, + sub_status, + challenge_txid, + disprove: Some(disprove), + }); + } + + if let Some(disprove) = detect_watchtower_flow_disprove(btc_client, graph).await? { + sub_status.disprove_type = Some(disprove.disprove_type); + sub_status.disprove_index = disprove.index as i32; + return Ok(GraphChainScan { + status: GraphStatus::Disprove, + sub_status, + challenge_txid, + disprove: Some(disprove), + }); + } + + let watchtower_challenge_init_txid = graph.watchtower_challenge_init.tx().compute_txid(); + let watchtower_challenge_init_on_chain = + tx_visible(btc_client, &watchtower_challenge_init_txid).await?; + if watchtower_challenge_init_on_chain { + for watchtower_index in 0..watchtower_num { + let watchtower_vout = + output_topology::watchtower_challenge_init::watchtower_connector( + watchtower_index, + ) as u64; + let spent = outpoint_spent_txid( + btc_client, + &watchtower_challenge_init_txid, + watchtower_vout, + ) + .await? + .is_some(); + if let Some(status) = + sub_status.watchtower_challenge_status.get_mut(watchtower_index) + { + *status = spent; + } + } + } + if !watchtower_challenge_init_on_chain { + return Ok(GraphChainScan { + status: current_status, + sub_status, + challenge_txid, + disprove: None, + }); + } + let operator_assert_txid = graph.operator_assert.tx().compute_txid(); + let operator_assert_on_chain = tx_visible(btc_client, &operator_assert_txid).await?; + if !operator_assert_on_chain { + return Ok(GraphChainScan { + status: current_status, + sub_status, + challenge_txid, + disprove: None, + }); + } + // TBD: add GraphStatus::Assert + } + + if current_status == GraphStatus::Challenge { + // TBD: add GraphStatus::Assert + let mut detected_disprove = None; + for verifier_index in 0..verifier_num { + let verifier_assert_txid = graph.verifier_asserts[verifier_index].tx().compute_txid(); + let disprove_txid = graph.disproves[verifier_index].tx().compute_txid(); + if tx_visible(btc_client, &verifier_assert_txid).await? { + if let Some(spent_txid) = + outpoint_spent_confirmed(btc_client, &verifier_assert_txid, 0).await? + { + if spent_txid == disprove_txid { + sub_status.verifier_challenge_status[verifier_index] = + VerifierChallengeStatus::Disproved; + sub_status.disprove_type = Some(DisproveTxType::Disprove); + sub_status.disprove_index = verifier_index as i32; + detected_disprove = Some(DetectedDisprove { + disprove_type: DisproveTxType::Disprove, + index: verifier_index, + challenge_finish_txid: spent_txid, + }); + current_status = GraphStatus::Disprove; + } else { + sub_status.verifier_challenge_status[verifier_index] = + VerifierChallengeStatus::ProverAnswered; + } + } else { + sub_status.verifier_challenge_status[verifier_index] = + VerifierChallengeStatus::VerifierAsserted; + } + } + } + if current_status == GraphStatus::Disprove { + return Ok(GraphChainScan { + status: current_status, + sub_status, + challenge_txid, + disprove: detected_disprove, + }); + } + if let Some(disprove) = detect_connector_d_disprove(btc_client, graph).await? { + sub_status.disprove_type = Some(disprove.disprove_type); + sub_status.disprove_index = disprove.index as i32; + return Ok(GraphChainScan { + status: GraphStatus::Disprove, + sub_status, + challenge_txid, + disprove: Some(disprove), + }); + } + if tx_confirmed(btc_client, &take2_txid).await? { + return Ok(GraphChainScan { + status: GraphStatus::OperatorTake2, + sub_status, + challenge_txid, + disprove: None, + }); + } + } + + Ok(GraphChainScan { status: current_status, sub_status, challenge_txid, disprove: None }) +} diff --git a/node/src/utils/graph_store.rs b/node/src/utils/graph_store.rs new file mode 100644 index 000000000..9fbbaebfb --- /dev/null +++ b/node/src/utils/graph_store.rs @@ -0,0 +1,700 @@ +//! Graph definitions in the store: ingest, raw data, lookups and status +//! transitions. + +use crate::clock::current_time_secs; +use crate::env::*; +use crate::error::SpecialError; +use anyhow::{Result, bail}; +use bitcoin::PublicKey; +use bitvm_lib::committee::*; +use bitvm_lib::operator::*; +use bitvm_lib::types::{BitvmGcGraph, SimplifiedBitvmGcGraph}; +use goat::transactions::prekickoff::PrekickoffTransaction; +use indexmap::IndexMap; +use store::localdb::{GraphQuery, LocalDB, StorageProcessor}; + +use crate::aggregates::graph_scan::ChallengeSubStatus; +use bitvm_lib::transactions::base::BaseTransaction; +use store::{ + Graph, GraphRawData, GraphStatus, GraphStatusSource, GraphStatusTransitionOutcome, + PeginGraphProcessData, +}; +use tracing::error; +use uuid::Uuid; + +use super::btc::node_p2wsh_address; +use super::rounds::GraphProcessDataMap; + +// db operations +pub async fn get_current_prekickoff_tx( + local_db: &LocalDB, + operator_pubkey: &PublicKey, +) -> Result> { + // return (latest_graph.nonce + 1 , latest_graph.next_prekickoff_tx) + // return None if no graph yet + let mut storage_processor = local_db.acquire().await?; + let graphs = storage_processor + .get_operator_graphs( + GraphQuery::default() + .with_operator_pubkey(operator_pubkey.to_string()) + .with_order("kickoff_index DESC".to_string()) + .with_limit(1), + ) + .await?; + + if let Some(graph) = graphs.first() + && let Some(simplified_graph) = load_validated_graph_definition( + &mut storage_processor, + graph.instance_id, + graph.graph_id, + ) + .await? + { + Ok(Some(( + (graph.kickoff_index + 1) as u64, + BitvmGcGraph::from_simplified(&simplified_graph)?.next_prekickoff, + ))) + } else { + Ok(None) + } +} + +fn convert_graph( + bitvm_graph: &BitvmGcGraph, + current_time: i64, + initial_status: GraphStatus, + definition_hash: String, +) -> Graph { + Graph { + graph_id: bitvm_graph.parameters.graph_id, + instance_id: bitvm_graph.parameters.instance_parameters.instance_id, + kickoff_index: bitvm_graph.parameters.graph_nonce as i64, + from_addr: "".to_string(), + to_addr: "".to_string(), + amount: bitvm_graph.parameters.instance_parameters.pegin_amount.to_sat() as i64, + challenge_amount: bitvm_graph.parameters.challenge_amount.to_sat() as i64, + status: initial_status.to_string(), + sub_status: "".to_string(), + operator_pubkey: bitvm_graph.parameters.operator_pubkey.to_string(), + definition_hash, + cur_prekickoff_txid: Some(bitvm_graph.cur_prekickoff.finalize().compute_txid().into()), + next_prekickoff: Some(bitvm_graph.next_prekickoff.finalize().compute_txid().into()), + force_skip_kickoff_txid: Some( + bitvm_graph.force_skip_kickoff.finalize().compute_txid().into(), + ), + quick_challenge_txid: Some(bitvm_graph.quick_challenge.finalize().compute_txid().into()), + challenge_incomplete_kickoff_txid: Some( + bitvm_graph.challenge_incomplete_kickoff.finalize().compute_txid().into(), + ), + pegin_txid: Some(bitvm_graph.pegin.finalize().compute_txid().into()), + kickoff_txid: Some(bitvm_graph.kickoff.finalize().compute_txid().into()), + take1_txid: Some(bitvm_graph.take1.finalize().compute_txid().into()), + challenge_txid: None, + take2_txid: Some(bitvm_graph.take2.finalize().compute_txid().into()), + watchtower_challenge_init_txid: Some( + bitvm_graph.watchtower_challenge_init.finalize().compute_txid().into(), + ), + operator_assert_txid: Some(bitvm_graph.operator_assert.finalize().compute_txid().into()), + verifier_assert_txids: bitvm_graph + .verifier_asserts + .iter() + .map(|tx| tx.finalize().compute_txid().into()) + .collect(), + disprove_txids: bitvm_graph + .disproves + .iter() + .map(|tx| tx.finalize().compute_txid().into()) + .collect(), + watchtower_challenge_timeout_txids: bitvm_graph + .watchtower_challenge_timeouts + .iter() + .map(|tx| tx.finalize().compute_txid().into()) + .collect(), + operator_challenge_nack_txids: bitvm_graph + .operator_challenge_nacks + .iter() + .map(|tx| tx.finalize().compute_txid().into()) + .collect(), + operator_commit_timeout_txid: Some( + bitvm_graph.operator_commit_timeout.finalize().compute_txid().into(), + ), + init_withdraw_tx_hash: None, + bridge_out_start_at: 0, + status_updated_at: current_time, + proceed_withdraw_height: 0, + created_at: current_time, + updated_at: current_time, + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum FinalizedGraphStoreOutcome { + NewlyStored, + DefinitionUpgraded, + AlreadyFinalized, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum GraphDefinitionIngestKind { + OperatorPresigned, + Finalized, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum GraphDefinitionIngestOutcome { + Inserted, + FinalizedUpgrade, + Replay, + AlreadyFinalized, +} + +/// Store a verified operator-pre-signed graph definition. +/// +/// This path is intentionally unable to advance `GraphStatus`: receiving a +/// CreateGraph message is not evidence of committee finalization. +pub(crate) async fn store_operator_presigned_graph( + local_db: &LocalDB, + simple_graph: &SimplifiedBitvmGcGraph, +) -> Result<()> { + if !simple_graph.operator_pre_signed() || simple_graph.committee_pre_signed() { + bail!(SpecialError::InvalidGraph(format!( + "graph {} is not an operator-pre-signed proposal", + simple_graph.parameters.graph_id + ))); + } + + // The definition path reads existing state before updating it. Reserve the + // writer up front so another task cannot invalidate that read snapshot. + let mut tx = local_db.start_immediate_transaction().await?; + ingest_graph_definition(&mut tx, simple_graph, GraphDefinitionIngestKind::OperatorPresigned) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Store a verified finalized graph once without replacing a graph whose +/// finalized form is already known. +pub(crate) async fn store_finalized_graph_if_needed( + local_db: &LocalDB, + simple_graph: &SimplifiedBitvmGcGraph, +) -> Result { + let graph_id = simple_graph.parameters.graph_id; + if !simple_graph.operator_pre_signed() || !simple_graph.committee_pre_signed() { + bail!(SpecialError::InvalidGraph(format!("graph {graph_id} is not fully pre-signed"))); + } + + // The definition path reads existing state before updating it. Reserve the + // writer up front so another task cannot invalidate that read snapshot. + let mut tx = local_db.start_immediate_transaction().await?; + let outcome = + ingest_graph_definition(&mut tx, simple_graph, GraphDefinitionIngestKind::Finalized) + .await?; + tx.commit().await?; + + Ok(match outcome { + GraphDefinitionIngestOutcome::Inserted => FinalizedGraphStoreOutcome::NewlyStored, + GraphDefinitionIngestOutcome::FinalizedUpgrade => { + FinalizedGraphStoreOutcome::DefinitionUpgraded + } + GraphDefinitionIngestOutcome::AlreadyFinalized => { + FinalizedGraphStoreOutcome::AlreadyFinalized + } + GraphDefinitionIngestOutcome::Replay => { + unreachable!("a finalized graph replay must be classified as already finalized") + } + }) +} + +async fn ingest_graph_definition( + tx: &mut StorageProcessor<'_>, + simple_graph: &SimplifiedBitvmGcGraph, + kind: GraphDefinitionIngestKind, +) -> Result { + let bitvm_graph: BitvmGcGraph = BitvmGcGraph::from_simplified(simple_graph)?; + let graph_id = simple_graph.parameters.graph_id; + verify_graph_operator_pre_signatures(&bitvm_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "graph {graph_id} has invalid operator pre-signatures: {error}" + )) + })?; + if kind == GraphDefinitionIngestKind::Finalized { + verify_graph_committee_pre_signatures(&bitvm_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "graph {graph_id} has invalid committee pre-signatures: {error}" + )) + })?; + } + let current_time = current_time_secs(); + let definition_hash = hex::encode(simple_graph.parameters_hash()?); + let existing_graph_row = tx.find_graph(&graph_id).await?; + let existing_raw_data = tx.find_graph_raw_data(&graph_id).await?; + let existing_graph = match (existing_graph_row.as_ref(), existing_raw_data) { + (None, None) => None, + (Some(_), None) => { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} has runtime data but no raw definition; explicit repair is required" + ))); + } + (None, Some(_)) => { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} has raw definition but no graph row; explicit repair is required" + ))); + } + (Some(existing_row), Some(existing_raw_data)) => { + if existing_row.definition_hash.is_empty() { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} is missing its stored definition hash; explicit repair is required" + ))); + } + if existing_row.definition_hash != definition_hash { + bail!(SpecialError::InvalidGraph(format!( + "graph parameters changed for graph_id {graph_id}: existing={}, incoming={definition_hash}", + existing_row.definition_hash + ))); + } + + let existing_graph = parse_graph_raw_data(existing_raw_data.raw_data, graph_id).await?; + let existing_raw_hash = hex::encode(existing_graph.parameters_hash()?); + if existing_raw_hash != existing_row.definition_hash { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} definition hash does not match its stored raw definition" + ))); + } + let existing_full_graph = BitvmGcGraph::from_simplified(&existing_graph)?; + verify_graph_operator_pre_signatures(&existing_full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored graph {graph_id} has invalid operator pre-signatures: {error}" + )) + })?; + if existing_full_graph.committee_pre_signed() { + verify_graph_committee_pre_signatures(&existing_full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored graph {graph_id} has invalid committee pre-signatures: {error}" + )) + })?; + } + Some(existing_graph) + } + }; + + let existing_is_finalized = + existing_graph.as_ref().is_some_and(SimplifiedBitvmGcGraph::committee_pre_signed); + if existing_is_finalized { + let existing_graph = existing_graph.as_ref().expect("checked above"); + if kind == GraphDefinitionIngestKind::Finalized && existing_graph != simple_graph { + bail!(SpecialError::InvalidGraph(format!( + "conflicting finalized graph for graph_id {graph_id}" + ))); + } + + // An old CreateGraph replay is harmless once a verified finalized + // graph is stored. Do not replace its raw signatures or runtime. + if kind == GraphDefinitionIngestKind::Finalized { + confirm_finalized_graph_definition( + tx, + bitvm_graph.parameters.instance_parameters.instance_id, + graph_id, + ) + .await?; + } + return Ok(GraphDefinitionIngestOutcome::AlreadyFinalized); + } + + if kind == GraphDefinitionIngestKind::OperatorPresigned + && let Some(existing_graph) = existing_graph.as_ref() + && existing_graph != simple_graph + { + bail!(SpecialError::InvalidGraph(format!( + "conflicting operator-pre-signed graph for graph_id {graph_id}" + ))); + } + if kind == GraphDefinitionIngestKind::Finalized + && let Some(existing_graph) = existing_graph.as_ref() + && existing_graph.operator_pre_sigs != simple_graph.operator_pre_sigs + { + bail!(SpecialError::InvalidGraph(format!( + "finalized graph {graph_id} changes the stored operator pre-signatures" + ))); + } + + if existing_graph.is_none() { + // One definition per operator slot: a second graph for a taken slot is refused for + // good, not retried on the unique index. + let operator_pubkey = bitvm_graph.parameters.operator_pubkey.to_string(); + let graph_nonce = bitvm_graph.parameters.graph_nonce; + if let Some(holder) = + tx.find_graph_by_operator_slot(&operator_pubkey, graph_nonce as i64).await? + && holder.graph_id != graph_id + { + bail!(SpecialError::InvalidGraph(format!( + "slot {graph_nonce} of operator {operator_pubkey} is held by graph {}; graph {graph_id} is refused", + holder.graph_id + ))); + } + } + + let mut graph = convert_graph( + &bitvm_graph, + current_time, + // A graph row is always created at the operator-presigned baseline. + // Only a verified finalized definition may advance it through the + // dedicated Definition transition below. + GraphStatus::OperatorPresigned, + definition_hash.clone(), + ); + + if let Some(node_info) = + tx.get_node_by_btc_pub_key(&bitvm_graph.parameters.operator_pubkey.to_string()).await? + { + graph.from_addr = node_info.goat_addr.clone(); + graph.to_addr = + node_p2wsh_address(get_network(), &bitvm_graph.parameters.operator_pubkey).to_string(); + } + + tx.upsert_graph_definition(&graph).await?; + + let outcome = if existing_graph.is_none() { + let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; + tx.upsert_graph_raw_data( + bitvm_graph.parameters.instance_parameters.instance_id, + GraphRawData { graph_id, raw_data, created_at: current_time, updated_at: current_time }, + &definition_hash, + ) + .await?; + if kind == GraphDefinitionIngestKind::Finalized { + confirm_finalized_graph_definition( + tx, + bitvm_graph.parameters.instance_parameters.instance_id, + graph_id, + ) + .await?; + } + GraphDefinitionIngestOutcome::Inserted + } else if kind == GraphDefinitionIngestKind::Finalized { + let raw_data = serialize_graph_raw_data(simple_graph, graph_id).await?; + tx.upsert_graph_raw_data( + bitvm_graph.parameters.instance_parameters.instance_id, + GraphRawData { graph_id, raw_data, created_at: current_time, updated_at: current_time }, + &definition_hash, + ) + .await?; + + // Only a verified finalized definition can advance an existing locally + // operator-pre-signed graph. A later chain-observed state is retained. + confirm_finalized_graph_definition( + tx, + bitvm_graph.parameters.instance_parameters.instance_id, + graph_id, + ) + .await?; + GraphDefinitionIngestOutcome::FinalizedUpgrade + } else { + GraphDefinitionIngestOutcome::Replay + }; + + Ok(outcome) +} + +/// Confirm that a fully verified graph definition has reached the committee +/// pre-signing stage without allowing it to overwrite a later runtime state. +async fn confirm_finalized_graph_definition( + tx: &mut StorageProcessor<'_>, + instance_id: Uuid, + graph_id: Uuid, +) -> Result<()> { + match tx + .transition_graph_status( + instance_id, + graph_id, + GraphStatus::CommitteePresigned, + GraphStatusSource::Definition, + None, + ) + .await? + { + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent => { + Ok(()) + } + GraphStatusTransitionOutcome::Rejected { current } => { + tracing::debug!( + "keep later graph status while storing finalized definition: graph={graph_id}, current={current}" + ); + Ok(()) + } + GraphStatusTransitionOutcome::NotFound => { + bail!("graph {graph_id} disappeared while confirming its finalized definition") + } + } +} + +/// Parse raw graph data JSON string to SimplifiedBitvmGcGraph using spawn_blocking +/// to handle large data and potential stack overflow issues +pub async fn parse_graph_raw_data( + raw_data: String, + graph_id: Uuid, +) -> Result { + let raw_data_len = raw_data.len(); + let raw_data_clone = raw_data.clone(); + let parse_result = tokio::task::spawn_blocking(move || { + serde_json::from_str::(&raw_data_clone) + }) + .await; + + match parse_result { + Ok(Ok(data)) => Ok(data), + Ok(Err(e)) => { + // Normal JSON parsing error + error!("Failed to parse graph data for graph_id {graph_id}: {e}"); + error!("Raw data length: {raw_data_len} bytes"); + Err(e.into()) + } + Err(join_err) => { + // spawn_blocking task failed (thread panic or task cancelled) + let msg = if join_err.is_panic() { + format!("Thread panic while parsing graph data for graph_id {graph_id}") + } else if join_err.is_cancelled() { + format!("Task cancelled while parsing graph data for graph_id {graph_id}") + } else { + format!( + "Task join error while parsing graph data for graph_id {graph_id}: {join_err}" + ) + }; + error!("{msg}"); + error!("Raw data length: {raw_data_len} bytes"); + Err(anyhow::anyhow!("{msg}")) + } + } +} + +/// Serialize SimplifiedBitvmGcGraph to JSON string using spawn_blocking +/// to handle large data and potential stack overflow issues +pub async fn serialize_graph_raw_data( + graph: &SimplifiedBitvmGcGraph, + graph_id: Uuid, +) -> Result { + let graph_clone = graph.clone(); + let serialize_result = + tokio::task::spawn_blocking(move || serde_json::to_string(&graph_clone)).await; + + match serialize_result { + Ok(Ok(data)) => Ok(data), + Ok(Err(e)) => { + // Normal JSON serialization error + error!("Failed to serialize graph data for graph_id {graph_id}: {e}"); + Err(e.into()) + } + Err(join_err) => { + // spawn_blocking task failed (thread panic or task cancelled) + let msg = if join_err.is_panic() { + format!("Thread panic while serializing graph data for graph_id {graph_id}") + } else if join_err.is_cancelled() { + format!("Task cancelled while serializing graph data for graph_id {graph_id}") + } else { + format!( + "Task join error while serializing graph data for graph_id {graph_id}: {join_err}", + ) + }; + error!("{msg}"); + Err(anyhow::anyhow!("{graph_id}")) + } + } +} + +/// Load a graph only after binding its runtime row, raw definition, canonical +/// parameters and pre-signatures together. Callers must use this instead of +/// parsing `graph_raw_data` directly. +pub(crate) async fn load_validated_graph_definition( + storage_processor: &mut StorageProcessor<'_>, + instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let Some(graph_row) = storage_processor.find_graph(&graph_id).await? else { + return Ok(None); + }; + if graph_row.instance_id != instance_id { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} belongs to instance {}, not {instance_id}", + graph_row.instance_id + ))); + } + if graph_row.definition_hash.is_empty() { + bail!(SpecialError::InvalidGraph(format!( + "graph {graph_id} is missing its definition hash; explicit repair is required" + ))); + } + + let graph_raw_data = storage_processor + .find_graph_raw_data(&graph_id) + .await? + .ok_or_else(|| { + SpecialError::InvalidGraph(format!( + "graph {graph_id} has a runtime row but no raw definition; explicit repair is required" + )) + })?; + let simplified_graph = parse_graph_raw_data(graph_raw_data.raw_data, graph_id).await?; + if simplified_graph.parameters.graph_id != graph_id + || simplified_graph.parameters.instance_parameters.instance_id != instance_id + { + bail!(SpecialError::InvalidGraph(format!( + "stored raw definition identifiers do not match graph {instance_id}:{graph_id}" + ))); + } + let definition_hash = hex::encode(simplified_graph.parameters_hash()?); + if definition_hash != graph_row.definition_hash { + bail!(SpecialError::InvalidGraph(format!( + "stored raw definition hash does not match graph row for {instance_id}:{graph_id}" + ))); + } + + let full_graph = BitvmGcGraph::from_simplified(&simplified_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored raw definition cannot rebuild graph for {instance_id}:{graph_id}: {error}" + )) + })?; + verify_graph_operator_pre_signatures(&full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored raw definition has invalid operator pre-signatures for {instance_id}:{graph_id}: {error}" + )) + })?; + if full_graph.committee_pre_signed() { + verify_graph_committee_pre_signatures(&full_graph).map_err(|error| { + SpecialError::InvalidGraph(format!( + "stored raw definition has invalid committee pre-signatures for {instance_id}:{graph_id}: {error}" + )) + })?; + } + + Ok(Some(simplified_graph)) +} + +pub async fn get_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + load_validated_graph_definition(&mut storage_processor, instance_id, graph_id).await +} + +pub async fn get_graph_by_instance_id_and_operator_pubkey( + local_db: &LocalDB, + instance_id: Uuid, + operator_pubkey: &PublicKey, +) -> Result> { + let mut storage_process = local_db.acquire().await?; + if let Some(graph_id) = storage_process + .get_graph_id_by_instance_id_and_operator_pubkey(&instance_id, &operator_pubkey.to_string()) + .await? + { + load_validated_graph_definition(&mut storage_process, instance_id, graph_id).await + } else { + Ok(None) + } +} + +pub async fn upsert_pegin_graph_process_data( + storage_processor: &mut StorageProcessor<'_>, + graph_id: Uuid, + instance_id: Uuid, + is_endorsed: bool, + process_data_map: &GraphProcessDataMap, +) -> Result<()> { + let current_time = current_time_secs(); + storage_processor + .upsert_pegin_graph_process_data(&PeginGraphProcessData { + graph_id, + instance_id, + is_endorsed, + process_data: serde_json::to_string(process_data_map)?, + updated_at: current_time, + created_at: current_time, + }) + .await?; + Ok(()) +} + +pub async fn find_pegin_graph_process_data( + storage_processor: &mut StorageProcessor<'_>, + graph_id: Uuid, +) -> Result<(bool, GraphProcessDataMap)> { + match storage_processor.find_pegin_graph_process_data(&graph_id).await? { + Some(data) => Ok((data.is_endorsed, serde_json::from_str(&data.process_data)?)), + None => Ok((false, IndexMap::new())), + } +} + +pub async fn update_graph_status( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + new_status: GraphStatus, + sub_status: Option, +) -> Result { + let mut storage_processor = local_db.acquire().await?; + storage_processor + .transition_graph_status( + instance_id, + graph_id, + new_status, + GraphStatusSource::ChainReconcile, + sub_status.map(|sub_status| serde_json::to_string(&sub_status)).transpose()?, + ) + .await +} + +/// Mark one graph obsolete and cancel its pending messages. +pub(crate) async fn obsolete_graph( + storage_processor: &mut StorageProcessor<'_>, + instance_id: Uuid, + graph_id: Uuid, +) -> Result { + let outcome = storage_processor + .transition_graph_status( + instance_id, + graph_id, + GraphStatus::Obsoleted, + GraphStatusSource::ChainReconcile, + None, + ) + .await?; + Ok(matches!( + outcome, + GraphStatusTransitionOutcome::Applied | GraphStatusTransitionOutcome::AlreadyCurrent + )) +} + +/// Mark every non-terminal graph for an already claimed instance as obsolete. +/// +/// `retained_graph_id` is the graph which completed the withdrawal. The caller +/// must only use this after a terminal Gateway withdraw event confirms that +/// this graph completed the withdrawal. +pub(crate) async fn obsolete_instance_graphs_except( + storage_processor: &mut StorageProcessor<'_>, + instance_id: Uuid, + retained_graph_id: Option, +) -> Result> { + let graphs = storage_processor.get_graphs_by_instance_id(&instance_id).await?; + let mut obsoleted_graph_ids = Vec::new(); + + for graph in graphs { + if Some(graph.graph_id) != retained_graph_id + && obsolete_graph(storage_processor, instance_id, graph.graph_id).await? + { + obsoleted_graph_ids.push(graph.graph_id); + } + } + + Ok(obsoleted_graph_ids) +} + +pub(crate) async fn pending_graph_belongs_to_operator( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + operator_pubkey: &PublicKey, +) -> Result { + let mut storage = local_db.acquire().await?; + Ok(storage.find_pending_graph_init_by_graph_id(&graph_id).await?.is_some_and(|pending| { + pending.instance_id == instance_id && pending.operator_pubkey == operator_pubkey.to_string() + })) +} diff --git a/node/src/utils/graph_tx.rs b/node/src/utils/graph_tx.rs new file mode 100644 index 000000000..47e24b944 --- /dev/null +++ b/node/src/utils/graph_tx.rs @@ -0,0 +1,476 @@ +//! Building and sending the graph transactions: pre-kickoff, kickoff, skip and +//! challenges. + +use crate::env::*; +use crate::error::SpecialError; +use anyhow::{Result, anyhow, bail}; +use bitcoin::hashes::Hash; +use bitcoin::key::Keypair; +use bitcoin::{Address, Amount, EcdsaSighashType, OutPoint, TxOut, Txid}; +use bitvm_lib::keys::{OperatorMasterKey, VerifierMasterKey, WatchtowerMasterKey}; +use bitvm_lib::operator::*; +use bitvm_lib::timelocks::default_timelock_config; +use bitvm_lib::types::{ + BitvmGcCircuitData, BitvmGcGraph, BitvmGcGraphParameters, BitvmGcInstanceParameters, + PrekickoffParameters, +}; +use bitvm_lib::verifier::*; +use bitvm_lib::watchtower::*; +use client::{btc_chain::BTCClient, goat_chain::GOATClient}; +use goat::connectors::{ + base::TaprootConnector, + kickoff_connectors::{ForceSkipConnector, KickoffConnector, PrekickoffConnector}, +}; +use goat::scripts::generate_opreturn_script; +use goat::transactions::base::Input; +use goat::transactions::pre_signed::PreSignedTransaction; +use goat::transactions::prekickoff::PrekickoffTransaction; +use store::localdb::LocalDB; + +use uuid::Uuid; + +use super::btc::{ + broadcast_package, broadcast_tx, build_cpfp_txns, build_sign_and_broadcast_tx, fund_address, + get_fee_rate, get_proper_utxo_set, node_p2wsh_address, node_sign, tx_visible, +}; +use super::graph_validate::{ + challenge_amount, min_prekickoff_input_amount, min_required_verifier, prekickoff_fee_amount, + prekickoff_replenishment_amount, validate_watchtower_registry_selection, +}; + +pub async fn verifier_force_skip_kickoff(client: &BTCClient, graph: &BitvmGcGraph) -> Result { + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); + let verifier_master_keypair = verifier_master_key.master_keypair(); + let verifier_receive_address = + node_p2wsh_address(get_network(), &verifier_master_keypair.public_key().into()); + let fee_rate = get_fee_rate(client).await?; + let (force_skip_kickoff_tx, anchor_added) = + build_force_skip_kickoff_tx(graph, verifier_receive_address, fee_rate)?; + if anchor_added { + let anchor_vout = force_skip_kickoff_tx.output.len() as u64 - 1; + let force_skip_kickoff_tx_total_input_amount = + graph.force_skip_kickoff.prev_outs().iter().map(|o| o.value).sum::(); + let child_tx = build_cpfp_txns( + client, + &force_skip_kickoff_tx, + anchor_vout, + force_skip_kickoff_tx_total_input_amount, + ) + .await?; + match child_tx { + Some(tx) => { + broadcast_package(client, &[force_skip_kickoff_tx.clone(), tx], true).await? + } + None => broadcast_tx(client, &force_skip_kickoff_tx).await?, + } + } else { + broadcast_tx(client, &force_skip_kickoff_tx).await?; + } + Ok(force_skip_kickoff_tx.compute_txid()) +} + +pub async fn verifier_quick_challenge(client: &BTCClient, graph: &BitvmGcGraph) -> Result { + let verifier_master_key = VerifierMasterKey::new(get_bitvm_key()?); + let verifier_master_keypair = verifier_master_key.master_keypair(); + let verifier_receive_address = + node_p2wsh_address(get_network(), &verifier_master_keypair.public_key().into()); + let fee_rate = get_fee_rate(client).await?; + let (quick_challenge_tx, anchor_added) = + build_quick_challenge_tx(graph, verifier_receive_address, fee_rate)?; + if anchor_added { + let anchor_vout = quick_challenge_tx.output.len() as u64 - 1; + let quick_challenge_tx_total_input_amount = + graph.quick_challenge.prev_outs().iter().map(|o| o.value).sum::(); + let child_tx = build_cpfp_txns( + client, + &quick_challenge_tx, + anchor_vout, + quick_challenge_tx_total_input_amount, + ) + .await?; + match child_tx { + Some(tx) => broadcast_package(client, &[quick_challenge_tx.clone(), tx], true).await?, + None => broadcast_tx(client, &quick_challenge_tx).await?, + } + } else { + broadcast_tx(client, &quick_challenge_tx).await?; + } + Ok(quick_challenge_tx.compute_txid()) +} + +pub async fn build_genesis_prekickoff_tx( + btc_client: &BTCClient, + goat_client: &GOATClient, +) -> Result { + let watchtower_num = goat_client.committee_mana_get_watchtowers().await?.len(); + let verifier_num = min_required_verifier(); + let network = get_network(); + let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); + let node_keypair = operator_master_key.master_keypair(); + let operator_taproot_public_key = node_keypair.x_only_public_key().0; + let cur_prekickoff_connector = PrekickoffConnector::new(network, &operator_taproot_public_key); + let next_force_skip_connector = ForceSkipConnector::new(network, &operator_taproot_public_key); + let next_kickoff_connector = KickoffConnector::new(network, &operator_taproot_public_key); + let next_prekickoff_connector = PrekickoffConnector::new(network, &operator_taproot_public_key); + let init_amount = prekickoff_replenishment_amount(); + let cur_prekickoff_connector_input = genesis_funding_input( + btc_client, + node_keypair, + cur_prekickoff_connector.generate_taproot_address(), + init_amount, + ) + .await?; + let fee_amount = prekickoff_fee_amount(0); + PrekickoffTransaction::new_for_validation( + &cur_prekickoff_connector, + &next_force_skip_connector, + &next_kickoff_connector, + &next_prekickoff_connector, + cur_prekickoff_connector_input, + vec![], + vec![], + fee_amount.to_sat(), + watchtower_num, + verifier_num, + ) + .map_err(|e| anyhow::anyhow!("failed to create pre-kickoff txn: {e}")) +} + +/// Reuse a matching unspent output, or fund the genesis address. +async fn genesis_funding_input( + btc_client: &BTCClient, + node_keypair: Keypair, + address: Address, + amount: Amount, +) -> Result { + let existing = btc_client + .get_address_utxo(address.clone()) + .await? + .into_iter() + .filter(|utxo| utxo.value == amount) + .min_by_key(|utxo| (utxo.txid, utxo.vout)); + let outpoint = match existing { + Some(utxo) => OutPoint { txid: utxo.txid, vout: utxo.vout }, + None => fund_address(btc_client, node_keypair, address, amount).await?, + }; + Ok(Input { outpoint, amount }) +} + +pub async fn build_prekickoff_params( + btc_client: &BTCClient, + graph_nonce: u64, + cur_prekickoff_txn: PrekickoffTransaction, +) -> Result { + let prekickoff_remaining_amount = cur_prekickoff_txn + .prekickoff_connector_input() + .map_err(|e| anyhow!("failed to get pre-kickoff connector input: {e}"))? + .amount; + let (replenish_fee_inputs, replenish_fee_prev_outs, fee_amount) = if prekickoff_remaining_amount + >= min_prekickoff_input_amount() + { + // no need to replenish funds + (vec![], vec![], prekickoff_fee_amount(0)) + } else { + let network = get_network(); + let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); + let master_keypair = operator_master_key.master_keypair(); + let nonce_keypair = operator_master_key.keypair_for_nonce(graph_nonce); + let nonce_address = node_p2wsh_address(network, &nonce_keypair.public_key().into()); + let replenishment_amount = prekickoff_replenishment_amount(); + let mut replenish_fee_inputs: Vec = btc_client + .get_address_utxo(nonce_address.clone()) + .await? + .into_iter() + .map(|u| Input { outpoint: OutPoint { txid: u.txid, vout: u.vout }, amount: u.value }) + .collect(); + let current_balance: Amount = replenish_fee_inputs.iter().map(|i| i.amount).sum(); + if current_balance < replenishment_amount { + let shortfall = replenishment_amount - current_balance; + let extra_input = Input { + outpoint: fund_address( + btc_client, + master_keypair, + nonce_address.clone(), + shortfall, + ) + .await?, + amount: shortfall, + }; + replenish_fee_inputs.push(extra_input); + }; + let replenish_fee_prev_outs: Vec = replenish_fee_inputs + .iter() + .map(|i| TxOut { value: i.amount, script_pubkey: nonce_address.script_pubkey() }) + .collect(); + let fee_amount = prekickoff_fee_amount(replenish_fee_inputs.len()); + (replenish_fee_inputs, replenish_fee_prev_outs, fee_amount) + }; + Ok(PrekickoffParameters { + cur_prekickoff_txn, + replenish_fee_inputs, + replenish_fee_prev_outs, + fee_amount: fee_amount.to_sat(), + }) +} + +pub async fn build_graph_params( + _local_db: &LocalDB, + goat_client: &GOATClient, + instance_parameters: BitvmGcInstanceParameters, + prekickoff_parameters: PrekickoffParameters, + bitvm_gc_circuit_datas: Vec, + graph_nonce: u64, + graph_id: Uuid, +) -> Result { + let network = instance_parameters.network; + let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); + let operator_master_keypair = operator_master_key.master_keypair(); + let operator_pubkey = operator_master_keypair.public_key().into(); + let operator_receive_address = + node_p2wsh_address(instance_parameters.network, &operator_pubkey); + let (_, operator_assert_wots_pubkey) = + operator_master_key.assert_wots_keypair_for_graph(graph_id); + let (_, operator_commit_pubin_wots_pubkey) = + operator_master_key.commit_pubin_wots_keypair_for_graph(graph_id); + let watchtower_pubkeys = goat_client.committee_mana_get_watchtowers().await?; + let watchtower_ack_hashlocks = (0..watchtower_pubkeys.len()) + .map(|index| { + bitcoin::hashes::hash160::Hash::hash( + &operator_master_key.preimage_for_graph(graph_id, index), + ) + .to_byte_array() + }) + .collect(); + validate_watchtower_registry_selection(&watchtower_pubkeys, &watchtower_pubkeys)?; + Ok(BitvmGcGraphParameters { + instance_parameters, + prekickoff_parameters, + timelock_config: default_timelock_config(network), + graph_id, + graph_nonce, + challenge_amount: challenge_amount(), + operator_pubkey, + operator_assert_wots_pubkey, + operator_commit_pubin_wots_pubkey, + operator_receive_address, + watchtower_pubkeys, + watchtower_ack_hashlocks, + pubin_disprove_constant: [0u8; 32], + gc_data: bitvm_gc_circuit_datas, + }) +} + +pub async fn operator_skip_graph(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result<()> { + let graph_nonce = graph.parameters.graph_nonce; + let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); + let operator_master_keypair = operator_master_key.master_keypair(); + let operator_receive_address = + node_p2wsh_address(get_network(), &operator_master_keypair.public_key().into()); + let operator_graph_keypair = operator_master_key.master_keypair(); + let mut prekickoff_tx = operator_sign_prekickoff_input_0(operator_graph_keypair, graph)?; + if prekickoff_tx.input.len() != 1 { + let operator_nonce_keypair = operator_master_key.keypair_for_nonce(graph_nonce - 1); + for i in 1..prekickoff_tx.input.len() { + let input_value = graph.cur_prekickoff.input_amounts[i]; + node_sign( + &mut prekickoff_tx, + i, + input_value, + bitcoin::EcdsaSighashType::All, + &operator_nonce_keypair, + )?; + } + } + let anchor_vout = prekickoff_tx.output.len() as u64 - 1; + let prekickoff_tx_total_input_amount = + graph.cur_prekickoff.input_amounts.clone().into_iter().sum(); + let child_tx = + build_cpfp_txns(btc_client, &prekickoff_tx, anchor_vout, prekickoff_tx_total_input_amount) + .await?; + let prekickoff_txid = prekickoff_tx.compute_txid(); + match operator_sign_skip_kickoff( + operator_graph_keypair, + graph, + operator_receive_address, + get_fee_rate(btc_client).await?, + )? { + Some(skip_kickoff_tx) => { + let skip_kickoff_txid = skip_kickoff_tx.compute_txid(); + if !tx_visible(btc_client, &prekickoff_txid).await? { + broadcast_package(btc_client, &[prekickoff_tx, skip_kickoff_tx], true).await?; + if let Some(child_tx) = child_tx { + broadcast_tx(btc_client, &child_tx).await?; + } + } else if !tx_visible(btc_client, &skip_kickoff_txid).await? { + if let Some(child_tx) = child_tx { + broadcast_package(btc_client, &[skip_kickoff_tx, child_tx], true).await?; + } else { + broadcast_tx(btc_client, &skip_kickoff_tx).await?; + } + } + } + None => match child_tx { + Some(tx) => { + if !tx_visible(btc_client, &prekickoff_txid).await? { + broadcast_package(btc_client, &[prekickoff_tx, tx], true).await?; + } + } + None => { + if !tx_visible(btc_client, &prekickoff_txid).await? { + broadcast_tx(btc_client, &prekickoff_tx).await?; + } + } + }, + }; + Ok(()) +} + +pub async fn operator_kickoff(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result<()> { + let graph_nonce = graph.parameters.graph_nonce; + let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); + let operator_graph_keypair = operator_master_key.master_keypair(); + let mut prekickoff_tx = operator_sign_prekickoff_input_0(operator_graph_keypair, graph)?; + if prekickoff_tx.input.len() != 1 { + let operator_nonce_keypair = operator_master_key.keypair_for_nonce(graph_nonce - 1); + for i in 1..prekickoff_tx.input.len() { + let input_value = graph.cur_prekickoff.input_amounts[i]; + node_sign( + &mut prekickoff_tx, + i, + input_value, + bitcoin::EcdsaSighashType::All, + &operator_nonce_keypair, + )?; + } + } + let prekickoff_txid = prekickoff_tx.compute_txid(); + let anchor_vout = prekickoff_tx.output.len() as u64 - 1; + let prekickoff_tx_total_input_amount = + graph.cur_prekickoff.input_amounts.clone().into_iter().sum(); + let prekickoff_child_tx = + build_cpfp_txns(btc_client, &prekickoff_tx, anchor_vout, prekickoff_tx_total_input_amount) + .await?; + + let kickoff_tx = operator_sign_kickoff(operator_graph_keypair, graph)?; + let kickoff_txid = kickoff_tx.compute_txid(); + let anchor_vout = kickoff_tx.output.len() as u64 - 1; + let kickoff_tx_total_input_amount = graph.kickoff.prev_outs().iter().map(|o| o.value).sum(); + let kickoff_child_tx = + build_cpfp_txns(btc_client, &kickoff_tx, anchor_vout, kickoff_tx_total_input_amount) + .await?; + + // If a tx is already on-chain, skip rebroadcasting and move to the next one. + let mut kickoff_child_broadcasted = false; + if !tx_visible(btc_client, &prekickoff_txid).await? { + // Parent not on-chain yet: broadcast parent and kickoff together as a package. + broadcast_package(btc_client, &[prekickoff_tx, kickoff_tx], true).await?; + } else if !tx_visible(btc_client, &kickoff_txid).await? { + // Parent is on-chain, but kickoff isn't: try kickoff (and its CPFP child if present). + if let Some(child) = kickoff_child_tx.as_ref() { + broadcast_package(btc_client, &[kickoff_tx, child.clone()], true).await?; + kickoff_child_broadcasted = true; + } else { + broadcast_tx(btc_client, &kickoff_tx).await?; + } + } + + // Ensure both transactions are seen on-chain and then handle CPFP children. + if !tx_visible(btc_client, &prekickoff_txid).await? { + bail!("prekickoff tx not on chain after broadcasting"); + } + if let Some(prekickoff_child_tx) = prekickoff_child_tx + && let Err(e) = broadcast_tx(btc_client, &prekickoff_child_tx).await + { + tracing::warn!("failed to broadcast prekickoff child tx: {e}"); + } + if !tx_visible(btc_client, &kickoff_txid).await? { + bail!("kickoff tx not on chain after broadcasting"); + } + if !kickoff_child_broadcasted + && let Some(kickoff_child_tx) = kickoff_child_tx + && let Err(e) = broadcast_tx(btc_client, &kickoff_child_tx).await + { + tracing::warn!("failed to broadcast kickoff child tx: {e}"); + } + Ok(()) +} + +pub async fn send_challenge_tx(btc_client: &BTCClient, graph: &BitvmGcGraph) -> Result { + let (mut challenge_tx, _) = export_challenge_tx(graph)?; + let challenge_keypair = VerifierMasterKey::new(get_bitvm_key()?).master_keypair(); + let verifier_evm_address = get_node_goat_address() + .ok_or_else(|| anyhow::anyhow!("failed to get node goat address".to_string()))?; + challenge_tx.output.push(bitcoin::TxOut { + value: Amount::ZERO, + script_pubkey: generate_opreturn_script(verifier_evm_address.to_vec()), + }); + let connector_a_input = graph + .kickoff + .connector_a_input() + .map_err(|e| anyhow!("failed to get connector-a input: {e}"))?; + build_sign_and_broadcast_tx( + btc_client, + challenge_keypair, + challenge_tx.input, + connector_a_input.amount, + challenge_tx.output, + ) + .await +} + +pub async fn send_watchtower_challenge_tx( + btc_client: &BTCClient, + graph: &BitvmGcGraph, + watchtower_index: usize, + commitment_data: Vec, +) -> Result { + let watchtower_keypair = WatchtowerMasterKey::new(get_bitvm_key()?).master_keypair(); + let fee_rate = get_fee_rate(btc_client).await?; + let watchtower_challenge_tx_base_vbytes = + estimate_watchtower_challenge_vbytes(commitment_data.len()); + let node_address = node_p2wsh_address(get_network(), &watchtower_keypair.public_key().into()); + match get_proper_utxo_set( + btc_client, + watchtower_challenge_tx_base_vbytes as u64, + node_address.clone(), + Amount::ZERO, + fee_rate, + ) + .await? + { + Some((inputs, fee_amount, _)) => { + let mut watchtower_challenge_tx = build_watchtower_challenge_tx( + graph, + &watchtower_keypair, + watchtower_index, + &commitment_data, + inputs.clone(), + &node_address, + fee_amount, + ) + .unwrap(); + for (i, input) in inputs.iter().enumerate() { + node_sign( + &mut watchtower_challenge_tx, + i + 1, + input.amount, + EcdsaSighashType::All, + &watchtower_keypair, + )?; + } + broadcast_tx(btc_client, &watchtower_challenge_tx).await?; + Ok(watchtower_challenge_tx.compute_txid()) + } + None => { + let current_balance = btc_client + .get_address_utxo(node_address) + .await? + .iter() + .map(|u| u.value) + .sum::(); + bail!(SpecialError::InsufficientBalance(format!( + "Not enough balance to complete the transaction, current_balance: {current_balance}" + ))); + } + } +} diff --git a/node/src/utils/graph_validate.rs b/node/src/utils/graph_validate.rs new file mode 100644 index 000000000..bf76fc612 --- /dev/null +++ b/node/src/utils/graph_validate.rs @@ -0,0 +1,632 @@ +//! Validation of graph parameters, policy and the init and finalized graphs. + +use crate::env::*; +use crate::error::SpecialError; +use alloy::primitives::Address as EvmAddress; +use anyhow::{Context, Result, anyhow, bail}; +use bitcoin::hashes::Hash; +use bitcoin::{Amount, Network, PublicKey, XOnlyPublicKey}; +use bitvm_lib::committee::*; +use bitvm_lib::operator::*; +use bitvm_lib::timelocks::estimated_block_interval_secs; +use bitvm_lib::types::{ + BitvmGcGraph, BitvmGcGraphParameters, BitvmGcInstanceParameters, SimplifiedBitvmGcGraph, +}; +use client::{btc_chain::BTCClient, goat_chain::GOATClient}; +use goat::transactions::pre_signed::PreSignedTransaction; +use libp2p::PeerId; +use secp256k1::schnorr::Signature as SchnorrSignature; +use std::str::FromStr; +use store::localdb::{GraphQuery, LocalDB}; + +use bitcoin_light_client_circuit::hash_operator_constant; +use bitvm_lib::babe_adapter::BABE_M_CC; +use bitvm_lib::transactions::base::BaseTransaction; + +use super::endorse::{ + get_graph_digest, verify_graph_endorsement_digest, verify_graph_params_endorsement, + verify_verifier_graph_params_endorsement, +}; +use super::goat_chain::read_instance_info_from_goat; +use super::graph_store::load_validated_graph_definition; + +// other operations +pub fn avg_block_time_secs(network: Network) -> u64 { + estimated_block_interval_secs(network) as u64 +} + +pub fn min_required_operator() -> usize { + // todo!("get min required operator number") + 1 +} + +pub fn min_required_watchtower() -> usize { + // todo!("get min required watchtower number") + 1 +} + +pub fn min_required_verifier() -> usize { + get_min_required_verifier() + .unwrap_or_else(|error| panic!("invalid verifier count configuration: {error}")) +} + +/// Checks graph Watchtower membership, uniqueness, and supported count. +pub(super) fn validate_watchtower_registry_selection( + selected: &[XOnlyPublicKey], + registered: &[XOnlyPublicKey], +) -> Result<()> { + use std::collections::HashSet; + + if selected.len() < min_required_watchtower() { + bail!(SpecialError::InvalidGraph(format!( + "insufficient watchtowers: have {}, required {}", + selected.len(), + min_required_watchtower() + ))); + } + if selected.len() > 256 { + bail!(SpecialError::InvalidGraph(format!( + "too many watchtowers: {}, max 256", + selected.len() + ))); + } + + let mut seen = HashSet::new(); + for key in selected { + if !seen.insert(*key) { + bail!(SpecialError::InvalidGraph("duplicate watchtower pubkey in graph".to_string())); + } + if !registered.contains(key) { + bail!(SpecialError::InvalidGraph(format!("watchtower {} is not registered", key))); + } + } + Ok(()) +} + +/// Validates the graph's ordered watchtower selection and its bound Operator constant. +fn validate_watchtower_selection( + selected: &[XOnlyPublicKey], + registered: &[XOnlyPublicKey], + graph_id: [u8; 16], + genesis_txid: [u8; 32], + challenge_init_txid: [u8; 32], + constant: [u8; 32], +) -> Result<()> { + validate_watchtower_registry_selection(selected, registered)?; + let key_bytes = selected.iter().map(XOnlyPublicKey::serialize).collect::>(); + let expected = hash_operator_constant(graph_id, genesis_txid, challenge_init_txid, &key_bytes); + if constant != expected { + bail!(SpecialError::InvalidGraph( + "operator constant mismatch for graph watchtower list".to_string() + )); + } + Ok(()) +} + +pub async fn validate_graph_instance_parameters( + btc_client: &BTCClient, + goat_client: &GOATClient, + parameters: &BitvmGcInstanceParameters, +) -> Result<()> { + let expected = read_instance_info_from_goat(goat_client, parameters.instance_id) + .await + .context("failed to load instance parameters from GoatChain")?; + if parameters != &expected { + bail!(SpecialError::InvalidGraph( + "instance parameters mismatch with GoatChain peg-in data".to_string() + )); + } + + for input in &expected.user_info.inputs { + let funding_tx = btc_client.get_tx(&input.outpoint.txid).await.with_context(|| { + format!("failed to load peg-in funding transaction {}", input.outpoint.txid) + })?; + // The parameters already match GoatChain: a transaction the Bitcoin + // backend does not know yet is the backend lagging, not a bad graph. + let Some(funding_tx) = funding_tx else { + bail!("peg-in funding transaction {} not found on Bitcoin", input.outpoint.txid); + }; + let funding_output = + funding_tx.output.get(input.outpoint.vout as usize).ok_or_else(|| { + SpecialError::InvalidGraph(format!( + "peg-in funding outpoint {}:{} does not exist", + input.outpoint.txid, input.outpoint.vout + )) + })?; + if funding_output.value != input.amount { + bail!(SpecialError::InvalidGraph(format!( + "peg-in funding amount mismatch for {}:{}: graph={}, bitcoin={}", + input.outpoint.txid, + input.outpoint.vout, + input.amount.to_sat(), + funding_output.value.to_sat() + ))); + } + } + + let pegin_deposit_txid = expected + .build_pegin_tx() + .map_err(|e| { + SpecialError::InvalidGraph(format!("failed to reconstruct peg-in deposit: {e}")) + })? + .0 + .tx() + .compute_txid(); + if btc_client + .get_tx(&pegin_deposit_txid) + .await + .with_context(|| format!("failed to load peg-in deposit transaction {pegin_deposit_txid}"))? + .is_none() + { + bail!("peg-in deposit transaction {pegin_deposit_txid} not found on Bitcoin"); + } + + Ok(()) +} + +fn validate_graph_policy(parameters: &BitvmGcGraphParameters) -> Result<()> { + let verifier_num = parameters.gc_data.len(); + if verifier_num < min_required_verifier() { + bail!(SpecialError::InvalidGraph(format!( + "insufficient verifier GC slots: have {}, required {}", + verifier_num, + min_required_verifier() + ))); + } + + let mut verifier_pubkeys = std::collections::HashSet::new(); + for gc_data in ¶meters.gc_data { + if !verifier_pubkeys.insert(XOnlyPublicKey::from(gc_data.verifier_pubkey)) { + bail!(SpecialError::InvalidGraph(format!( + "duplicate verifier pubkey in GC data: {}", + gc_data.verifier_pubkey + ))); + } + if gc_data.final_msg_hashlocks.len() != BABE_M_CC { + bail!(SpecialError::InvalidGraph(format!( + "verifier {} has {} final message hashlocks, expected {}", + gc_data.verifier_pubkey, + gc_data.final_msg_hashlocks.len(), + BABE_M_CC + ))); + } + } + + let watchtower_num = parameters.watchtower_pubkeys.len(); + if watchtower_num < min_required_watchtower() { + bail!(SpecialError::InvalidGraph(format!( + "insufficient watchtower slots: have {}, required {}", + watchtower_num, + min_required_watchtower() + ))); + } + let mut watchtower_pubkeys = std::collections::HashSet::new(); + for pubkey in ¶meters.watchtower_pubkeys { + if !watchtower_pubkeys.insert(*pubkey) { + bail!(SpecialError::InvalidGraph(format!( + "duplicate watchtower pubkey in graph: {pubkey}" + ))); + } + } + let mut watchtower_hashlocks = std::collections::HashSet::new(); + for hashlock in ¶meters.watchtower_ack_hashlocks { + if !watchtower_hashlocks.insert(*hashlock) { + bail!(SpecialError::InvalidGraph( + "duplicate watchtower ack hashlock in graph".to_string() + )); + } + } + + Ok(()) +} + +async fn validate_prekickoff_continuity( + local_db: &LocalDB, + full_graph: &BitvmGcGraph, +) -> Result<()> { + if full_graph.parameters.graph_nonce == 0 { + return Ok(()); + } + + let mut storage_processor = local_db.acquire().await?; + let previous_nonce = full_graph.parameters.graph_nonce - 1; + let graphs = storage_processor + .get_operator_graphs( + GraphQuery::default() + .with_operator_pubkey(full_graph.parameters.operator_pubkey.to_string()) + .with_kickoff_index(previous_nonce as i64) + .with_limit(1), + ) + .await?; + if let Some(previous_graph) = graphs.first() { + let simplified_graph = load_validated_graph_definition( + &mut storage_processor, + previous_graph.instance_id, + previous_graph.graph_id, + ) + .await? + .ok_or_else(|| { + anyhow!("previous graph raw data is missing for graph nonce {previous_nonce}") + })?; + let expected_cur_prekickoff = + BitvmGcGraph::from_simplified(&simplified_graph)?.next_prekickoff; + let expected_txid = expected_cur_prekickoff.finalize().compute_txid(); + let actual_txid = full_graph.cur_prekickoff.finalize().compute_txid(); + if actual_txid != expected_txid { + bail!(SpecialError::InvalidGraph(format!( + "cur prekickoff continuity mismatch: graph={actual_txid}, expected={expected_txid}" + ))); + } + Ok(()) + } else { + // Not stored yet: the job retries until the definition arrives by + // gossip or a pull, or the graph retires. + bail!("previous graph is missing for graph nonce {previous_nonce}"); + } +} + +pub async fn validate_init_graph_base( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + graph: &SimplifiedBitvmGcGraph, +) -> Result<()> { + if !graph.operator_pre_signed() { + bail!(SpecialError::InvalidGraph("graph is missing operator pre-signatures".to_string())); + } + // Basic structural and on-chain consistency checks for an incoming graph proposal. + // Return SpecialError::InvalidGraph on any validation failure. + // 1) Rebuild full graph (ensures signatures present if flags are set and tx graph is coherent) + let full_graph = BitvmGcGraph::from_simplified(graph) + .map_err(|e| SpecialError::InvalidGraph(format!("invalid graph structure: {e}")))?; + validate_graph_policy(&graph.parameters)?; + validate_prekickoff_continuity(local_db, &full_graph).await?; + verify_graph_operator_pre_signatures(&full_graph) + .map_err(|e| SpecialError::InvalidGraph(format!("invalid operator pre-signatures: {e}")))?; + + // 2) Bind all instance and peg-in parameters to GoatChain and Bitcoin. + validate_graph_instance_parameters( + btc_client, + goat_client, + &graph.parameters.instance_parameters, + ) + .await?; + + // 3) Challenge amount and assert-commit count must match local constants + if graph.parameters.challenge_amount != challenge_amount() { + bail!(SpecialError::InvalidGraph("unexpected challenge amount".to_string())); + } + + // 4) Watchtower config sanity: number of watchtowers should match number of hashlocks and registry size + let watchtowers_on_chain = goat_client + .committee_mana_get_watchtowers() + .await + .context("load the watchtower registry from GoatChain")?; + + validate_watchtower_selection( + &graph.parameters.watchtower_pubkeys, + &watchtowers_on_chain, + *graph.parameters.graph_id.as_bytes(), + get_genesis_sequencer_commit_id(), + full_graph.watchtower_challenge_init.tx().compute_txid().to_byte_array(), + graph.parameters.pubin_disprove_constant, + )?; + + // 5) Operator stake sanity: verify operator is registered and has enough locked stake. + // The query may fail transiently; only the verdict is a rejection. + match operator_stake_status(goat_client, &graph.parameters.operator_pubkey) + .await + .context("query operator stake")? + { + OperatorStakeStatus::Staked => {} + OperatorStakeStatus::NotRegistered => { + bail!(SpecialError::InvalidGraph("operator not registered".to_string())) + } + OperatorStakeStatus::Insufficient { locked, min } => { + bail!(SpecialError::InvalidGraph(format!( + "insufficient operator stake: locked={locked}, min={min}" + ))) + } + } + + Ok(()) +} + +/// Operator stake verdict; RPC failures are returned as errors. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OperatorStakeStatus { + Staked, + NotRegistered, + Insufficient { locked: u64, min: u64 }, +} + +pub async fn operator_stake_status( + goat_client: &GOATClient, + operator_pubkey: &PublicKey, +) -> Result { + let operator_xonly_pubkey = XOnlyPublicKey::from(*operator_pubkey).serialize(); + let operator_addr = goat_client + .stake_mana_pubkey_to_address(&operator_xonly_pubkey) + .await + .context("query operator address")?; + if operator_addr == [0; 20] { + return Ok(OperatorStakeStatus::NotRegistered); + } + let min_stake_amount = + goat_client.gateway_get_min_stake_amount().await.context("query minimum operator stake")?; + let locked_stake = goat_client + .stake_mana_lock_stake_of(&operator_addr) + .await + .context("query operator locked stake")?; + if locked_stake < min_stake_amount { + return Ok(OperatorStakeStatus::Insufficient { + locked: locked_stake, + min: min_stake_amount, + }); + } + Ok(OperatorStakeStatus::Staked) +} + +pub async fn validate_verifier_graph_params_endorsements( + goat_client: &GOATClient, + graph: &SimplifiedBitvmGcGraph, + verifier_endorsements: &[(PublicKey, usize, String, SchnorrSignature)], +) -> Result<()> { + let expected_verifier_num = graph.parameters.gc_data.len(); + if verifier_endorsements.len() != expected_verifier_num { + bail!(SpecialError::InvalidGraph(format!( + "verifier params endorsement count {} does not match GC slot count {}", + verifier_endorsements.len(), + expected_verifier_num + ))); + } + + let mut seen_pubkeys = std::collections::HashSet::new(); + let mut seen_indices = std::collections::HashSet::new(); + let mut seen_peers = std::collections::HashSet::new(); + for (verifier_pubkey, verifier_index, verifier_peer_id, signature) in verifier_endorsements { + let peer = + PeerId::from_str(verifier_peer_id).context("invalid verifier endorsement peer")?; + if !seen_peers.insert(peer) { + bail!(SpecialError::InvalidGraph("duplicate verifier endorsement peer".into())); + } + if *verifier_index >= expected_verifier_num { + bail!(SpecialError::InvalidGraph(format!( + "verifier params endorsement index {verifier_index} out of range" + ))); + } + let slot_pubkey = graph.parameters.gc_data[*verifier_index].verifier_pubkey; + if slot_pubkey != *verifier_pubkey { + bail!(SpecialError::InvalidGraph(format!( + "verifier params endorsement slot mismatch at index {verifier_index}: signature pubkey={}, graph pubkey={slot_pubkey}", + verifier_pubkey + ))); + } + if !seen_pubkeys.insert(XOnlyPublicKey::from(*verifier_pubkey)) { + bail!(SpecialError::InvalidGraph(format!( + "duplicate verifier params endorsement pubkey: {verifier_pubkey}" + ))); + } + if !seen_indices.insert(*verifier_index) { + bail!(SpecialError::InvalidGraph(format!( + "duplicate verifier params endorsement index: {verifier_index}" + ))); + } + let ok = verify_verifier_graph_params_endorsement(verifier_pubkey, graph, signature)?; + if !ok { + bail!(SpecialError::InvalidGraph(format!( + "invalid verifier params endorsement signature for {verifier_pubkey}" + ))); + } + } + + for (_, _, peer, _) in verifier_endorsements { + let peer = PeerId::from_str(peer)?; + if !goat_client + .committee_mana_is_verifier(&peer.to_bytes()) + .await + .context("recheck verifier registration before committee setup")? + { + bail!(SpecialError::InvalidGraph( + "endorsement peer is not a registered verifier".into() + )); + } + } + Ok(()) +} + +pub async fn validate_init_graph( + local_db: &LocalDB, + btc_client: &BTCClient, + goat_client: &GOATClient, + graph: &SimplifiedBitvmGcGraph, + verifier_endorsements: &[(PublicKey, usize, String, SchnorrSignature)], +) -> Result<()> { + validate_init_graph_base(local_db, btc_client, goat_client, graph).await?; + validate_verifier_graph_params_endorsements(goat_client, graph, verifier_endorsements).await +} + +pub async fn validate_finalized_graph( + btc_client: &BTCClient, + goat_client: &GOATClient, + graph: &SimplifiedBitvmGcGraph, + endorse_sigs: &[(PublicKey, EvmAddress, Vec)], + params_endorse_sigs: &[(PublicKey, EvmAddress, Vec)], +) -> Result<()> { + if !graph.operator_pre_signed() || !graph.committee_pre_signed() { + bail!(SpecialError::InvalidGraph( + "finalized graph is missing operator or committee pre-signatures".to_string() + )); + } + // 1) Rebuild full graph to ensure structure is coherent and txns derivable + let full_graph = BitvmGcGraph::from_simplified(graph) + .map_err(|e| SpecialError::InvalidGraph(format!("invalid graph structure: {e}")))?; + validate_graph_policy(&graph.parameters)?; + verify_graph_operator_pre_signatures(&full_graph) + .map_err(|e| SpecialError::InvalidGraph(format!("invalid operator pre-signatures: {e}")))?; + verify_graph_committee_pre_signatures(&full_graph).map_err(|e| { + SpecialError::InvalidGraph(format!("invalid committee pre-signatures: {e}")) + })?; + + // 2) Repeat the full instance and peg-in binding for nodes that did not see CreateGraph. + validate_graph_instance_parameters( + btc_client, + goat_client, + &graph.parameters.instance_parameters, + ) + .await?; + + let instance_id = graph.parameters.instance_parameters.instance_id; + if graph.parameters.challenge_amount != challenge_amount() { + bail!(SpecialError::InvalidGraph("unexpected challenge amount".to_string())); + } + + let watchtowers_on_chain = goat_client + .committee_mana_get_watchtowers() + .await + .context("load the watchtower registry from GoatChain")?; + validate_watchtower_selection( + &graph.parameters.watchtower_pubkeys, + &watchtowers_on_chain, + *graph.parameters.graph_id.as_bytes(), + get_genesis_sequencer_commit_id(), + full_graph.watchtower_challenge_init.tx().compute_txid().to_byte_array(), + graph.parameters.pubin_disprove_constant, + )?; + + // 3) Validate endorsements: unique, from legitimate committee members, and signatures recover to the provided EVM address + use std::collections::HashSet; + let mut seen_committee: HashSet = HashSet::new(); + let mut seen_evm: HashSet = HashSet::new(); + let pegin_data = goat_client + .gateway_get_pegin_data(&instance_id) + .await + .context("load the instance's peg-in data from GoatChain")?; + // One digest per graph; computing it is a GoatChain call and may fail + // transiently, while a signature that does not recover is a rejection. + let graph_digest = get_graph_digest(goat_client, &full_graph) + .await + .context("load the graph digest from GoatChain")?; + if pegin_data.committee_pubkeys.len() != pegin_data.committee_addresses.len() { + bail!(SpecialError::InvalidGraph( + "on-chain committee pubkey and address counts differ".to_string() + )); + } + if endorse_sigs.len() != pegin_data.committee_pubkeys.len() { + bail!(SpecialError::InvalidGraph(format!( + "endorsement count {} does not match instance committee count {}", + endorse_sigs.len(), + pegin_data.committee_pubkeys.len() + ))); + } + if params_endorse_sigs.len() != pegin_data.committee_pubkeys.len() { + bail!(SpecialError::InvalidGraph(format!( + "params endorsement count {} does not match instance committee count {}", + params_endorse_sigs.len(), + pegin_data.committee_pubkeys.len() + ))); + } + + for (pk, evm_addr, sig) in endorse_sigs.iter() { + // no duplicates + if !seen_committee.insert(*pk) { + bail!(SpecialError::InvalidGraph( + "duplicate committee pubkey in endorsements".to_string() + )); + } + if !seen_evm.insert(*evm_addr) { + bail!(SpecialError::InvalidGraph("duplicate evm address in endorsements".to_string())); + } + + // map pubkey -> expected evm address from GoatChain + let mut found = false; + for i in 0..pegin_data.committee_pubkeys.len() { + let on_chain_pk = + PublicKey::from_slice(&pegin_data.committee_pubkeys[i]).map_err(|e| { + SpecialError::InvalidGraph(format!("invalid committee pubkey on-chain: {e}")) + })?; + if &on_chain_pk == pk { + found = true; + let expected_addr = pegin_data.committee_addresses[i]; + if &expected_addr != evm_addr { + bail!(SpecialError::InvalidGraph("committee evm address mismatch".to_string())); + } + break; + } + } + if !found { + bail!(SpecialError::InvalidGraph("endorser not in committee set".to_string())); + } + + // cryptographically verify the endorsement against the graph digest + let ok = verify_graph_endorsement_digest(evm_addr, &graph_digest, sig).map_err(|e| { + SpecialError::InvalidGraph(format!("failed to verify endorsement: {e}")) + })?; + if !ok { + bail!(SpecialError::InvalidGraph("invalid endorsement signature".to_string())); + } + } + + let mut seen_committee: HashSet = HashSet::new(); + let mut seen_evm: HashSet = HashSet::new(); + for (pk, evm_addr, sig) in params_endorse_sigs.iter() { + if !seen_committee.insert(*pk) { + bail!(SpecialError::InvalidGraph( + "duplicate committee pubkey in params endorsements".to_string() + )); + } + if !seen_evm.insert(*evm_addr) { + bail!(SpecialError::InvalidGraph( + "duplicate evm address in params endorsements".to_string() + )); + } + + let mut found = false; + for i in 0..pegin_data.committee_pubkeys.len() { + let on_chain_pk = + PublicKey::from_slice(&pegin_data.committee_pubkeys[i]).map_err(|e| { + SpecialError::InvalidGraph(format!("invalid committee pubkey on-chain: {e}")) + })?; + if &on_chain_pk == pk { + found = true; + let expected_addr = pegin_data.committee_addresses[i]; + if &expected_addr != evm_addr { + bail!(SpecialError::InvalidGraph( + "committee evm address mismatch in params endorsements".to_string() + )); + } + break; + } + } + if !found { + bail!(SpecialError::InvalidGraph("params endorser not in committee set".to_string())); + } + + let ok = verify_graph_params_endorsement(evm_addr, &full_graph, sig).map_err(|e| { + SpecialError::InvalidGraph(format!("failed to verify params endorsement: {e}")) + })?; + if !ok { + bail!(SpecialError::InvalidGraph("invalid params endorsement signature".to_string())); + } + } + + Ok(()) +} + +pub fn prekickoff_replenishment_amount() -> Amount { + Amount::from_sat(500000) +} + +pub fn min_prekickoff_input_amount() -> Amount { + Amount::from_sat(200000) +} + +pub fn challenge_amount() -> Amount { + Amount::from_sat(20000) +} + +pub fn prekickoff_fee_amount(replenish_fee_inputs_num: usize) -> Amount { + let tx_vbytes = PRE_KICKOFF_BASE_VBYTES + + (replenish_fee_inputs_num as u64 * CHECKSIG_P2WSH_INPUT_VBYTES_ESTIMATE); + Amount::from_sat(tx_vbytes) +} diff --git a/node/src/utils/instance_store.rs b/node/src/utils/instance_store.rs new file mode 100644 index 000000000..6c5fec415 --- /dev/null +++ b/node/src/utils/instance_store.rs @@ -0,0 +1,298 @@ +//! Instance parameters and process data. + +use crate::clock::current_time_secs; +use crate::env::*; +use crate::error::SpecialError; +use alloy::primitives::Address as EvmAddress; +use anyhow::{Context, Result, bail}; +use bitcoin::address::NetworkUnchecked; +use bitcoin::hashes::Hash; +use bitcoin::{Address, Amount, Network, OutPoint, PublicKey, Txid, XOnlyPublicKey}; +use bitvm_lib::types::{BitvmGcInstanceParameters, UserInfo}; +use client::Utxo as ClientUtxo; +use client::btc_chain::BTCClient; +use client::goat_chain::GOATClient; +use client::graphs::graph_query::BridgeInRequestEvent; +use goat::contexts::base::generate_n_of_n_public_key; +use goat::transactions::base::Input; +use indexmap::IndexMap; +use std::str::FromStr; +use store::localdb::{LocalDB, StorageProcessor}; + +use crate::env; + +use store::{ + ByteArray32, Instance, InstanceBridgeInStatus, PeginInstanceProcessData, UInt64Array3, +}; +use tracing::warn; +use uuid::Uuid; + +use super::rounds::InstanceProcessDataMap; +use super::strip_hex_prefix_owned; + +pub struct GenerateInstanceParams { + pub instance_id: Uuid, + pub user_info: UserInfo, + pub pegin_amount: Amount, + pub pegin_request_tx_hash: String, + pub pegin_request_height: i64, + pub pegin_timestamp: i64, +} + +pub async fn generate_instance( + btc_client: &BTCClient, + params: GenerateInstanceParams, +) -> Result { + let from_addr = if !params.user_info.inputs.is_empty() + && let Some(tx) = btc_client.get_tx(¶ms.user_info.inputs[0].outpoint.txid).await? + { + let tx_scripts = + tx.output[params.user_info.inputs[0].outpoint.vout as usize].script_pubkey.clone(); + Address::from_script(&tx_scripts, env::get_network()) + .map(|addr| addr.to_string()) + .unwrap_or_default() + } else { + warn!( + "failed to decode instance {} from_address from pegin_request as input_utxos is empty or decode address failed", + params.instance_id + ); + "".to_string() + }; + let input_utxos = params + .user_info + .inputs + .iter() + .map(|input| ClientUtxo { + txid: input.outpoint.txid.to_byte_array(), + vout: input.outpoint.vout, + amount_sats: input.amount.to_sat(), + }) + .collect::>(); + let current_time = current_time_secs(); + + Ok(Instance { + instance_id: params.instance_id, + network: get_network().to_string(), + from_addr, + to_addr: EvmAddress::from(¶ms.user_info.depositor_evm_address).to_string(), + amount: params.pegin_amount.to_sat() as i64, + fees: UInt64Array3(params.user_info.txn_fees), + input_utxos: serde_json::to_string(&input_utxos)?, + status: InstanceBridgeInStatus::UserInited.to_string(), + goat_tx_hash: params.pegin_request_tx_hash, + goat_tx_height: params.pegin_request_height, + user_xonly_pubkey: ByteArray32(params.user_info.user_xonly_pubkey.clone().serialize()), + user_change_addr: params.user_info.user_change_address.clone().to_string(), + user_refund_addr: params.user_info.user_refund_address.clone().to_string(), + btc_txid: None, + pegin_confirm_txid: None, + pegin_cancel_txid: None, + committees_answers: IndexMap::new(), + pegin_data_tx_hash: "".to_string(), + btc_height: 0, + parameters: None, + post_pegin_txhash: None, + key_deleted: false, + status_updated_at: params.pegin_timestamp, + created_at: current_time, + updated_at: current_time, + }) +} + +pub async fn store_instance_parameters( + local_db: &LocalDB, + instance_params: &BitvmGcInstanceParameters, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let incoming_parameters_hash = instance_params.parameters_hash()?; + if let Some(existing_parameters) = + storage_processor.get_instance_parameters_by_id(&instance_params.instance_id).await? + { + let existing_instance_params: BitvmGcInstanceParameters = + serde_json::from_str(&existing_parameters)?; + let existing_parameters_hash = existing_instance_params.parameters_hash()?; + if existing_parameters_hash != incoming_parameters_hash { + bail!(SpecialError::InvalidPeginData(format!( + "instance parameters changed for instance_id {}: existing={}, incoming={}", + instance_params.instance_id, + hex::encode(existing_parameters_hash), + hex::encode(incoming_parameters_hash) + ))); + } + } + storage_processor + .update_instance_parameters( + &instance_params.instance_id, + &serde_json::to_string(&instance_params)?, + ) + .await?; + Ok(()) +} + +pub async fn get_instance_parameters( + local_db: &LocalDB, + instance_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let Some(instance) = storage_processor.find_instance(&instance_id).await? else { + return Ok(None); + }; + + if let Some(parameters) = instance.parameters { + return Ok(Some(serde_json::from_str(¶meters)?)); + } + + Ok(Some( + gen_instance_parameters_local(&instance).with_context(|| { + format!("failed to reconstruct parameters for instance {instance_id}") + })?, + )) +} + +pub async fn upsert_pegin_instance_process_data( + storage_processor: &mut StorageProcessor<'_>, + instance_id: Uuid, + process_data_map: &InstanceProcessDataMap, +) -> Result<()> { + let current_time = current_time_secs(); + storage_processor + .upsert_pegin_instance_process_data(&PeginInstanceProcessData { + instance_id, + process_data: serde_json::to_string(process_data_map)?, + updated_at: current_time, + created_at: current_time, + }) + .await?; + Ok(()) +} + +pub async fn find_pegin_instance_process_data( + storage_processor: &mut StorageProcessor<'_>, + instance_id: Uuid, +) -> Result { + if let Ok(Some(data)) = storage_processor.find_pegin_instance_process_data(&instance_id).await + && let Ok(process_data) = serde_json::from_str(data.process_data.as_str()) + { + Ok(process_data) + } else { + Ok(IndexMap::new()) + } +} + +pub fn gen_instance_parameters_local( + instance: &Instance, +) -> anyhow::Result { + let network = Network::from_str(&instance.network)?; + let committee_pubkeys: Vec = instance + .committees_answers + .iter() + .map(|(committee, pubkey)| { + PublicKey::from_slice(pubkey).with_context(|| { + format!( + "invalid committee public key for committee {committee} in instance {}", + instance.instance_id + ) + }) + }) + .collect::>()?; + if committee_pubkeys.is_empty() { + bail!( + "instance {} has no committee answers to derive parameters from", + instance.instance_id + ); + } + + let committee_agg_pubkey = generate_n_of_n_public_key(&committee_pubkeys).0; + let utxos: Vec = serde_json::from_str(&instance.input_utxos)?; + Ok(BitvmGcInstanceParameters { + network, + instance_id: instance.instance_id, + user_info: gen_user_info( + network, + &instance.to_addr, + &instance.user_change_addr.clone(), + &instance.user_refund_addr.clone(), + utxos, + instance.fees.0, + &instance.user_xonly_pubkey.0, + )?, + pegin_amount: Amount::from_sat(instance.amount as u64), + committee_pubkeys, + committee_agg_pubkey, + }) +} + +fn gen_user_info( + network: Network, + depositor_evm_address: &str, + user_change_addr: &str, + user_refund_addr: &str, + utxos: Vec, + txn_fees: [u64; 3], + user_xonly_pubkey: &[u8; 32], +) -> anyhow::Result { + let user_change_address: Address = Address::from_str(user_change_addr)?; + let user_refund_addr: Address = Address::from_str(user_refund_addr)?; + let inputs = utxos + .into_iter() + .map(|utxo| Input { + outpoint: OutPoint { txid: Txid::from_slice(&utxo.txid).unwrap(), vout: utxo.vout }, + amount: Amount::from_sat(utxo.amount_sats), + }) + .collect(); + Ok(UserInfo { + depositor_evm_address: EvmAddress::from_str(depositor_evm_address)?.into_array(), + txn_fees, + inputs, + user_xonly_pubkey: XOnlyPublicKey::from_slice(user_xonly_pubkey)?, + user_change_address: user_change_address.require_network(network)?, + user_refund_address: user_refund_addr.require_network(network)?, + }) +} + +/// Build an instance from BridgeInRequest and Goat pegin data. Input-spend checks +/// are deferred until the Prepare transaction can be identified. +pub(crate) async fn generate_instance_from_bridge_in_request_event( + btc_client: &BTCClient, + goat_client: &GOATClient, + event: &BridgeInRequestEvent, +) -> anyhow::Result { + let instance_id = Uuid::from_str(&strip_hex_prefix_owned(&event.instance_id))?; + let pegin_data = goat_client.gateway_get_pegin_data(&instance_id).await?; + let inputs: Vec = pegin_data + .user_inputs + .iter() + .map(|u| Input { + outpoint: OutPoint { txid: Txid::from_byte_array(u.txid), vout: u.vout }, + amount: Amount::from_sat(u.amount_sats), + }) + .collect(); + + let user_change_address: Address = + Address::from_str(&event.user_change_address)?; + let user_refund_addr: Address = + Address::from_str(&event.user_refund_address)?; + let network = get_network(); + let user_info = UserInfo { + depositor_evm_address: EvmAddress::from_str(&event.depositor_address)?.into_array(), + txn_fees: event.txn_fees.clone().map(|s| s.parse::().unwrap_or(0)), + inputs, + user_xonly_pubkey: XOnlyPublicKey::from_str(&strip_hex_prefix_owned( + &event.user_xonly_pubkey, + ))?, + user_change_address: user_change_address.require_network(network)?, + user_refund_address: user_refund_addr.require_network(network)?, + }; + generate_instance( + btc_client, + GenerateInstanceParams { + instance_id, + user_info, + pegin_amount: Amount::from_sat(event.pegin_amount_sats.parse::().unwrap_or(0)), + pegin_request_tx_hash: event.transaction_hash.clone(), + pegin_request_height: event.block_number.parse::().unwrap_or(0), + pegin_timestamp: event.block_timestamp.parse::().unwrap_or(current_time_secs()), + }, + ) + .await +} diff --git a/node/src/utils/mod.rs b/node/src/utils/mod.rs new file mode 100644 index 000000000..76d4f41d9 --- /dev/null +++ b/node/src/utils/mod.rs @@ -0,0 +1,91 @@ +mod btc; +mod committee_keys; +mod endorse; +pub mod evm_swap_utils; +mod goat_chain; +mod graph_chain_state; +mod graph_store; +mod graph_tx; +mod graph_validate; +mod instance_store; +mod node_info; +mod proof_client; +mod rounds; +mod setup_state; + +pub use btc::*; +pub use committee_keys::*; +pub use endorse::*; +pub use goat_chain::*; +pub(crate) use graph_chain_state::*; +pub use graph_store::*; +pub use graph_tx::*; +pub use graph_validate::*; +pub use instance_store::*; +pub use node_info::*; +pub use proof_client::*; +pub use rounds::*; +pub use setup_state::*; + +use alloy::primitives::Address as EvmAddress; +use anyhow::Result; +use bitcoin::{Address, CompressedPublicKey, Network, PrivateKey}; +use rand::Rng; +use secp256k1::Secp256k1; +use store::BridgeOutGlobalStats; +use store::localdb::StorageProcessor; + +pub fn generate_random_bytes(len: usize) -> Vec { + let mut rng = rand::thread_rng(); + (0..len).map(|_| rng.gen_range(0..255)).collect() +} + +pub fn get_rand_btc_address_p2wpkh(network: Network) -> String { + let secp = Secp256k1::new(); + Address::p2wpkh( + &CompressedPublicKey::try_from(PrivateKey::generate(network).public_key(&secp)) + .expect("Could not compress public key"), + network, + ) + .to_string() +} + +pub fn get_rand_goat_address() -> String { + EvmAddress::from_slice(&generate_random_bytes(20)).to_string() +} + +pub fn strip_hex_prefix_owned(s: &str) -> String { + if s.starts_with("0x") || s.starts_with("0X") { s[2..].to_string() } else { s.to_string() } +} + +pub fn generate_local_key() -> libp2p::identity::Keypair { + libp2p::identity::Keypair::generate_ed25519() +} + +pub fn temp_sqlite_db_path() -> String { + let tmp_db = tempfile::NamedTempFile::new().unwrap(); + format!("sqlite:{}", tmp_db.path().as_os_str().to_str().unwrap()) +} + +// contract calls + +pub(crate) const BRIDGE_OUT_GLOBAL_STATS_ID: i64 = 1; + +pub(crate) async fn get_bridge_out_global_stats<'a>( + storage_processor: &mut StorageProcessor<'a>, +) -> Result { + match storage_processor.find_bridge_out_global_stats_by_id(BRIDGE_OUT_GLOBAL_STATS_ID).await? { + Some(global_stats) => Ok(global_stats), + None => Ok(BridgeOutGlobalStats { + id: BRIDGE_OUT_GLOBAL_STATS_ID, + initial_txn: 0, + initial_amount: "0".to_string(), + claim_txn: 0, + claim_amount: "0".to_string(), + refund_txn: 0, + refund_amount: "0".to_string(), + created_at: 0, + updated_at: 0, + }), + } +} diff --git a/node/src/utils/node_info.rs b/node/src/utils/node_info.rs new file mode 100644 index 000000000..68ade44b8 --- /dev/null +++ b/node/src/utils/node_info.rs @@ -0,0 +1,279 @@ +//! NodeInfo validation and the node table rows it keeps current. + +use crate::env::*; +use crate::messages::{GOATMessage, GOATMessageContent, NodeInfo}; +use crate::middleware::AllBehaviours; +use crate::middleware::publisher::publish_to_role; +use alloy::primitives::Address as EvmAddress; +use anyhow::Result; +use bitcoin::PublicKey; +use bitvm_lib::actors::Actor; +use libp2p::{PeerId, Swarm}; +use std::net::SocketAddr; +use std::str::FromStr; +use std::time::{SystemTime, UNIX_EPOCH}; +use store::localdb::LocalDB; + +use store::Node; +use stun_client::{Attribute, Class, Client}; +use tracing::warn; + +/// Upper bound for the operator-supplied node name, which `GET /v1/nodes` +/// returns verbatim. +const MAX_NODE_NAME_LEN: usize = 64; + +/// Upper bound for the advertised `host:port` string. +const MAX_SOCKET_ADDR_LEN: usize = 128; + +/// Every `NodeInfo` field is self-declared by the sender, yet the node registry +/// is keyed by `peer_id`. The one identity a peer cannot forge is the gossipsub +/// message source: the swarm runs with `MessageAuthenticity::Signed` and strict +/// validation, so `message.source` is bound to the sender's libp2p key. Key the +/// registry row on that identity and never on the `peer_id` inside the payload, +/// otherwise any peer can overwrite any other node's record. +pub fn node_info_matches_sender(node_info: &NodeInfo, from_peer_id: &PeerId) -> bool { + node_info.peer_id == from_peer_id.to_string() +} + +/// Reject payloads that cannot be persisted safely. These fields are served +/// verbatim by `GET /v1/nodes` and drive on-chain lookups and reward +/// attribution in `node_available_pbtc_update_monitor` and `add_node_reward`. +pub fn validate_node_info_payload(node_info: &NodeInfo) -> std::result::Result<(), String> { + if PeerId::from_str(&node_info.peer_id).is_err() { + return Err(format!("invalid peer_id {}", node_info.peer_id)); + } + if Actor::from_str(&node_info.actor).is_err() { + return Err(format!("invalid actor {}", node_info.actor)); + } + if !node_info.goat_addr.is_empty() && EvmAddress::from_str(&node_info.goat_addr).is_err() { + return Err(format!("invalid goat_addr {}", node_info.goat_addr)); + } + if !node_info.btc_pub_key.is_empty() && PublicKey::from_str(&node_info.btc_pub_key).is_err() { + return Err(format!("invalid btc_pub_key {}", node_info.btc_pub_key)); + } + if !node_info.available_peg_btc.is_empty() + && alloy::primitives::U256::from_str(&node_info.available_peg_btc).is_err() + { + return Err(format!("invalid available_peg_btc {}", node_info.available_peg_btc)); + } + if !node_info.service_fee_rate.is_finite() || !(0.0..=1.0).contains(&node_info.service_fee_rate) + { + return Err(format!("invalid service_fee_rate {}", node_info.service_fee_rate)); + } + if node_info.node_name.chars().count() > MAX_NODE_NAME_LEN + || node_info.node_name.chars().any(char::is_control) + { + return Err(format!("invalid node_name of {} chars", node_info.node_name.chars().count())); + } + validate_node_socket_addr(&node_info.socket_addr) +} + +/// The advertised socket address is only ever displayed and dialed by clients, +/// so require a plain `host:port` string that cannot smuggle control characters +/// into API responses. +fn validate_node_socket_addr(socket_addr: &str) -> std::result::Result<(), String> { + if socket_addr.is_empty() { + return Ok(()); + } + if socket_addr.chars().count() > MAX_SOCKET_ADDR_LEN + || socket_addr.chars().any(|c| c.is_control() || c.is_whitespace()) + { + return Err(format!("invalid socket_addr of {} chars", socket_addr.chars().count())); + } + let Some((host, port)) = socket_addr.rsplit_once(':') else { + return Err(format!("socket_addr {socket_addr} is missing a port")); + }; + if host.is_empty() || port.parse::().is_err() { + return Err(format!("socket_addr {socket_addr} is not a valid host:port")); + } + Ok(()) +} + +pub async fn save_node_info(local_db: &LocalDB, node_info: &NodeInfo) -> Result<()> { + // Reachable once per announcement from any peer: not louder than debug. + tracing::debug!("save_node_info for {}", node_info.peer_id); + let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; + let mut storage_process = local_db.acquire().await?; + storage_process + .upsert_node(&Node { + peer_id: node_info.peer_id.clone(), + actor: node_info.actor.clone(), + node_name: node_info.node_name.clone(), + goat_addr: node_info.goat_addr.clone(), + btc_pub_key: node_info.btc_pub_key.clone(), + socket_addr: node_info.socket_addr.clone(), + reward: "0".to_string(), + service_fee_rate: node_info.service_fee_rate, + available_peg_btc: node_info.available_peg_btc.clone(), + binding_sig: node_info.binding_sig.clone(), + binding_issued_at: node_info.binding_issued_at, + updated_at: current_time, + created_at: current_time, + }) + .await?; + Ok(()) +} + +pub async fn save_local_info(local_db: &LocalDB) { + let node = get_local_node_info(); + // Peers apply the same validation to the NodeInfo this node broadcasts, so a + // malformed local config would silently drop this node from every peer's + // registry. Surface it here instead, where the operator reads the logs. + if let Err(reason) = validate_node_info_payload(&node) { + tracing::error!( + "local node info will be rejected by peers: {reason}; fix the node configuration" + ); + } + match save_node_info(local_db, &node).await { + Ok(_) => {} + Err(err) => tracing::error!("save local node err: {err}"), + } +} + +pub async fn update_node_timestamp(local_db: &LocalDB, peer_id: &str) -> Result<()> { + tracing::debug!("update timestamp for {peer_id}"); + let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; + let mut storage_process = local_db.acquire().await?; + match storage_process.update_node_timestamp(peer_id, current_time).await { + Ok(_) => {} + Err(err) => warn!("{err}"), + }; + Ok(()) +} + +pub async fn detect_heart_beat(swarm: &mut Swarm) -> Result<()> { + tracing::info!("start detect_heart_beat"); + let message_content = GOATMessageContent::RequestNodeInfo(get_local_node_info()); + // send to actor + let actors = get_rpc_support_actors(); + for actor in actors { + match publish_to_role(swarm, GOATMessage::new(actor, message_content.clone())).await { + Ok(_) => {} + Err(err) => warn!("{err}"), + } + } + Ok(()) +} + +/// Retrieve the server's public IP via NAT protocol and combine it with +/// the configured RPC monitoring port`rpc_addr` to generate the external RPC service address. +pub async fn set_node_external_socket_addr_env(rpc_addr: &str) -> Result<()> { + if get_proof_server_url().is_some() || std::env::var(ENV_EXTERNAL_SOCKET_ADDR).is_ok() { + // not provide proof server + return Ok(()); + } + let addr = SocketAddr::from_str(rpc_addr)?; + let mut client = Client::new("0.0.0.0:0", None).await?; + let message_res = client.binding_request("stun.l.google.com:19302", None).await; + if message_res.is_err() { + warn!("fail to get message from stun.l.google.com:19302, err :{:?}", message_res.err()); + return Ok(()); + } + let message = message_res?; + if message.get_class() != Class::SuccessResponse { + warn!( + "fail to get message from stun.l.google.com:19302, return class :{:?}", + message.get_class() + ); + return Ok(()); + } + if let Some(socket_addr) = Attribute::get_xor_mapped_address(&message) { + unsafe { + std::env::set_var( + ENV_EXTERNAL_SOCKET_ADDR, + SocketAddr::new(socket_addr.ip(), addr.port()).to_string(), + ); + } + } + Ok(()) +} + +pub fn reflect_goat_address(addr_op: Option) -> (bool, Option) { + if let Some(addr) = addr_op + && let Ok(addr) = EvmAddress::from_str(&addr) + { + return (true, Some(addr.to_string())); + } + (false, None) +} + +#[cfg(test)] +mod node_info_tests { + use super::*; + + /// One field mutation applied to an otherwise valid payload. + type NodeInfoMutation = fn(&mut NodeInfo); + + fn valid_node_info(peer_id: &str) -> NodeInfo { + NodeInfo { + peer_id: peer_id.to_string(), + actor: Actor::Operator.to_string(), + goat_addr: "0x1234567890abcdef1234567890abcdef12345678".to_string(), + btc_pub_key: "0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798" + .to_string(), + socket_addr: "127.0.0.1:8080".to_string(), + node_name: "zkm".to_string(), + service_fee_rate: 0.001, + available_peg_btc: "1000".to_string(), + ..Default::default() + } + } + + #[test] + fn node_info_is_accepted_only_for_its_own_sender() { + let sender = PeerId::random(); + let other = PeerId::random(); + let node_info = valid_node_info(&sender.to_string()); + + assert!(node_info_matches_sender(&node_info, &sender)); + assert!(!node_info_matches_sender(&node_info, &other)); + } + + #[test] + fn valid_node_info_payload_is_accepted() { + let node_info = valid_node_info(&PeerId::random().to_string()); + assert!(validate_node_info_payload(&node_info).is_ok()); + } + + #[test] + fn optional_node_info_fields_may_be_empty() { + let mut node_info = valid_node_info(&PeerId::random().to_string()); + node_info.goat_addr = String::new(); + node_info.btc_pub_key = String::new(); + node_info.socket_addr = String::new(); + node_info.available_peg_btc = String::new(); + node_info.node_name = String::new(); + assert!(validate_node_info_payload(&node_info).is_ok()); + } + + #[test] + fn malformed_node_info_fields_are_rejected() { + let peer_id = PeerId::random().to_string(); + let cases: Vec<(&str, NodeInfoMutation)> = vec![ + ("peer_id", |n| n.peer_id = "not-a-peer-id".to_string()), + ("actor", |n| n.actor = "Auditor".to_string()), + ("goat_addr", |n| n.goat_addr = "0xdeadbeef".to_string()), + ("btc_pub_key", |n| n.btc_pub_key = "02zz".to_string()), + ("available_peg_btc", |n| n.available_peg_btc = "lots".to_string()), + ("service_fee_rate NaN", |n| n.service_fee_rate = f64::NAN), + ("service_fee_rate range", |n| n.service_fee_rate = 42.0), + ("node_name length", |n| n.node_name = "a".repeat(MAX_NODE_NAME_LEN + 1)), + ("node_name control", |n| n.node_name = "a\nb".to_string()), + ("socket_addr length", |n| { + n.socket_addr = format!("{}:8080", "a".repeat(MAX_SOCKET_ADDR_LEN)) + }), + ("socket_addr port", |n| n.socket_addr = "host".to_string()), + ("socket_addr host", |n| n.socket_addr = ":8080".to_string()), + ("socket_addr control", |n| n.socket_addr = "127.0.0.1 :8080".to_string()), + ]; + + for (label, mutate) in cases { + let mut node_info = valid_node_info(&peer_id); + mutate(&mut node_info); + assert!( + validate_node_info_payload(&node_info).is_err(), + "expected {label} to be rejected" + ); + } + } +} diff --git a/node/src/utils/proof_client.rs b/node/src/utils/proof_client.rs new file mode 100644 index 000000000..05d20e665 --- /dev/null +++ b/node/src/utils/proof_client.rs @@ -0,0 +1,798 @@ +//! The proof builder client: watchtower commitments and challenge data, operator +//! proofs and the statements they are checked against. + +use crate::env::*; +use anyhow::{Context, Result, anyhow, bail}; +use bitcoin::hashes::Hash; +use bitcoin::key::Keypair; +use bitcoin::{BlockHash, Txid, XOnlyPublicKey}; +use bitcoin_light_client_circuit::{ + VK_HASH_SIZE, build_watchtower_commitment, decode_operator_public_outputs, +}; +use bitvm_lib::babe_adapter::{ + DYNAMIC_SLOTS, NUM_PUBLIC_INPUTS, STATIC_SLOTS, StaticInputs, ziren_static_inputs, +}; + +use bitvm_lib::types::BitvmGcGraph; +use client::btc_chain::BTCClient; +use goat::transactions::base::output_topology; +use goat::transactions::pre_signed::PreSignedTransaction; +use p3_bn254_fr::Bn254Fr; +use p3_field::{PrimeCharacteristicRing, PrimeField}; +use reqwest::Url; +use serde::Serialize; +use store::localdb::LocalDB; + +use crate::env; +use crate::rpc_service::routes::v1::{ + NODES_OPERATOR_BASE, NODES_WATCHTOWER_BASE, PROOFS_WATCHTOWER_PROOF_TIMEOUT, +}; + +use bitcoin_light_client_circuit::hash_operator_constant; +use client::http_client::async_client::HttpAsyncClient; +use proof_builder::api_auth::{ProofBuilderAuthRole, sign_proof_builder_request}; +use proof_builder::{ + OperatorProofRequest, OperatorProofResponse, ProofData, WatchtowerProofRequest, + WatchtowerProofResponse, WatchtowerProofTimeoutUpdateRequest, + WatchtowerProofTimeoutUpdateResponse, +}; +use store::SerializableTxid; +use tracing::{info, warn}; +use uuid::Uuid; +use zkm_pcs::PartStarkVerifyingKey; +use zkm_recursion_core::stark::KoalaBearPoseidon2Outer; +use zkm_sdk::ZKMProofWithPublicValues; +use zkm_verifier::{ + Groth16Verifier, IMM_GROTH16_VK_BYTES, VK_ROOT_BYTES, convert_ark_imm_wrap_vk, + decode_zkm_vkey_hash, +}; + +use super::btc::outpoint_spent_txid; + +pub type VerifyingKey = ark_groth16::VerifyingKey; + +pub type Groth16Proof = ark_groth16::Proof; + +pub type PublicInputs = Vec; + +#[derive(Clone)] +pub struct ValidatedOperatorProof { + pub proof: Groth16Proof, + pub public_inputs: PublicInputs, + pub verifying_key: VerifyingKey, + pub public_values: Vec, + pub vk_hash: String, + pub zkm_version: String, +} + +#[derive(Clone)] +pub struct OperatorStatement { + pub static_input: StaticInputs, + pub vk_hash: [u8; 32], + pub zkm_version: String, + pub constant: [u8; 32], +} + +fn gen_watchtower_commitment(graph_id: Uuid, proof_data: ProofData) -> Result> { + let graph_id = graph_id.as_bytes(); + let proof = + proof_data.proof.as_slice().try_into().map_err(|_| anyhow!("invalid proof length"))?; + let public_inputs = proof_data + .public_inputs + .as_slice() + .try_into() + .map_err(|_| anyhow!("invalid public inputs length"))?; + if proof_data.vk.len() != VK_HASH_SIZE { + bail!("invalid vk_hash length"); + } + if proof_data.zkm_version.is_empty() { + bail!("missing zkm_version"); + } + + Ok(build_watchtower_commitment( + graph_id, + proof, + &public_inputs, + &proof_data.vk, + &proof_data.zkm_version, + )) +} + +// proof network +/// Builds fresh Proof Builder authentication headers for one outbound request attempt. +fn proof_builder_auth_headers( + keypair: &Keypair, + role: ProofBuilderAuthRole, + path: &str, + body: &B, +) -> Result> { + Ok(sign_proof_builder_request(keypair, role, "POST", path, body)?.to_header_pairs()) +} + +/// Returns: +/// - `Ok(Some(WatchtowerCommitment), _)` if watchtower proof is available +/// - `Ok(None, wait_secs)` if watchtower proof is not yet available, with suggested wait time +pub async fn get_watchtower_commitment( + local_db: &LocalDB, + btc_client: &BTCClient, + http_client: &HttpAsyncClient, + instance_id: Uuid, + graph_id: Uuid, +) -> Result<(Option>, usize)> { + let mut storage_processor = local_db.acquire().await?; + if let Some(graph) = storage_processor.find_graph(&graph_id).await? + && let Some(challenge_init_txid) = graph.watchtower_challenge_init_txid + { + // check if challenge_txid is confirmed + let tx_status = btc_client.get_tx_status(&challenge_init_txid.0).await?; + if !tx_status.confirmed { + warn!("graph {graph_id} challenge tx is not confirmed"); + return Ok((None, get_watchtower_proof_wait_secs())); + } + + let base_url = Url::parse( + &get_proof_build_rpc_host() + .ok_or_else(|| anyhow::anyhow!("failed to get proof_build_rpc_host"))?, + )?; + let url = base_url.join(NODES_WATCHTOWER_BASE)?; + + let payload = WatchtowerProofRequest { + instance_id: instance_id.to_string(), + graph_id: graph_id.to_string(), + gateway_address: Some(env::get_goat_gateway_contract_from_env().to_string()), + public_key: env::get_node_pubkey()?.to_string(), + challenge_init_txid: challenge_init_txid.0.to_string(), + execution_layer_block_number: graph.proceed_withdraw_height, // NOTE: this number may be zero + }; + let auth_keypair = get_bitvm_key()?; + let response = http_client + .post_response_json_with_dynamic_headers::( + url.as_str(), + &payload, + || { + proof_builder_auth_headers( + &auth_keypair, + ProofBuilderAuthRole::Watchtower, + NODES_WATCHTOWER_BASE, + &payload, + ) + }, + ) + .await + .inspect(|_| crate::metrics_service::record_backend_ok("proof")) + .inspect_err(|error| { + crate::metrics_service::record_backend_failure("proof", "watchtower_proofs", error) + })?; + match response.proof_data { + Some(proof_data) => { + let watchtower_commitment = gen_watchtower_commitment(graph_id, proof_data)?; + Ok((Some(watchtower_commitment), 0)) + } + None => Ok((None, get_watchtower_proof_wait_secs())), + } + } else { + warn!("graph:{graph_id} not found or related txn is none",); + bail!("No graph in db"); + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct WatchtowerChallengeInfo { + pub challenge_txids: Vec>, + pub included_watchtowers: Vec, + pub resolved_branch_txids: Vec, +} + +#[tracing::instrument(name = "get_watchtower_challenge_info", skip(btc_client))] +pub async fn get_watchtower_challenge_info( + btc_client: &BTCClient, + watchtower_challenge_init_txid: &SerializableTxid, + watchtower_timeout_txids: &[Txid], + num_watchtowers: usize, +) -> Result { + let mut challenge_txids = Vec::with_capacity(num_watchtowers); + let mut included_watchtowers = Vec::with_capacity(num_watchtowers); + let mut resolved_branch_txids = Vec::with_capacity(num_watchtowers); + for index in 0..num_watchtowers { + let challenge_vout = + output_topology::watchtower_challenge_init::watchtower_connector(index) as u64; + let spent_txid = + outpoint_spent_txid(btc_client, &watchtower_challenge_init_txid.0, challenge_vout) + .await?; + match spent_txid { + Some(txid) => { + let status = btc_client.get_tx_status(&txid).await?; + if !status.confirmed { + bail!("watchtower branch tx {txid} at index {index} is not confirmed yet"); + } + resolved_branch_txids.push(txid); + + let is_timeout = watchtower_timeout_txids + .get(index) + .is_some_and(|timeout_txid| txid == *timeout_txid); + if is_timeout { + challenge_txids.push(None); + included_watchtowers.push(false); + } else { + challenge_txids.push(Some(txid.to_string())); + included_watchtowers.push(true); + } + } + None => { + challenge_txids.push(None); + included_watchtowers.push(false); + } + } + } + Ok(WatchtowerChallengeInfo { challenge_txids, included_watchtowers, resolved_branch_txids }) +} + +/// Returns `(btc_best_block_hash, included_watchtowers_bitmap)` using every finalized +/// challenge-branch resolution for the block hash while only challenges set bitmap bits. +/// +/// The selected hash is signed into the operator's pubin and cannot be replaced after a +/// reorganization. Require every branch that determines the bitmap to be at least as deeply +/// buried as the header/SPV finality window before returning it. +pub async fn compute_operator_pubin_blockhash_and_bitmap( + btc_client: &BTCClient, + resolved_branch_txids: &[Txid], + included_watchtowers_bits: &[bool], +) -> Result<([u8; 32], [u8; 32])> { + let tip_height = btc_client.get_height().await?; + let required_burial_depth = get_btc_block_confirms(btc_client.network()); + let btc_best_block_hash = { + let mut largest: Option<(u32, BlockHash)> = None; + for txid in resolved_branch_txids { + let status = btc_client.get_tx_status(txid).await?; + let (height, hash) = match (status.confirmed, status.block_height, status.block_hash) { + (true, Some(height), Some(hash)) => (height, hash), + _ => bail!("watchtower branch resolution tx {txid} is not confirmed yet"), + }; + // `BTC_BLOCK_CONFIRMS` is the number of blocks that must follow a block before + // the SPV/header pipeline consumes it. Match the proof-builder's strict boundary: + // the anchor must have more than this many blocks after it. + let burial_depth = tip_height.saturating_sub(height); + if burial_depth <= required_burial_depth { + bail!( + "watchtower branch resolution tx {txid} is not final enough for operator pubin: height={height}, tip={tip_height}, burial_depth={burial_depth}, required_burial_depth>{required_burial_depth}" + ); + } + if largest.is_none_or(|(h, _)| height > h) { + largest = Some((height, hash)); + } + } + largest + .map(|(_, hash)| hash.to_byte_array()) + .ok_or_else(|| anyhow!("no confirmed watchtower branch resolution tx available"))? + }; + + let mut included_watchtowers = [0u8; 32]; + for (i, &included) in included_watchtowers_bits.iter().enumerate() { + if included && i < 256 { + included_watchtowers[i / 8] |= 1 << (i % 8); + } + } + + Ok((btc_best_block_hash, included_watchtowers)) +} + +async fn get_operator_committed_blockhash( + btc_client: &BTCClient, + resolved_branch_txids: &[Txid], + included_watchtowers_bits: &[bool], + graph_id: Uuid, +) -> Result> { + match compute_operator_pubin_blockhash_and_bitmap( + btc_client, + resolved_branch_txids, + included_watchtowers_bits, + ) + .await + { + Ok((btc_best_block_hash, _)) => { + Ok(Some(BlockHash::from_byte_array(btc_best_block_hash).to_string())) + } + Err(e) => { + warn!("operator proof inputs are not ready for graph {graph_id}: {e}"); + Ok(None) + } + } +} + +/// Assembles the 96-byte guest pubin: +pub fn build_operator_guest_pubin( + btc_best_block_hash: &[u8; 32], + pubin_disprove_constant: &[u8; 32], + included_watchtowers: &[u8; 32], +) -> [u8; 96] { + let mut pubin = [0u8; 96]; + pubin[0..32].copy_from_slice(btc_best_block_hash); + pubin[32..64].copy_from_slice(pubin_disprove_constant); + pubin[64..96].copy_from_slice(included_watchtowers); + pubin +} + +fn load_part_stark_vk_for_zkm_version(zkm_version: &str) -> Result> { + let part_stark_vk = Groth16Verifier::try_get_part_stark_vk(zkm_version).ok_or_else(|| { + anyhow!("no partial STARK verifying key is bundled for Ziren {zkm_version}") + })?; + Ok(Vec::from(part_stark_vk)) +} + +fn convert_operator_proof_to_ark( + proof: &ZKMProofWithPublicValues, + vk_hash: &str, +) -> Result { + let part_stark_vk = load_part_stark_vk_for_zkm_version(&proof.zkm_version)?; + convert_ark_imm_wrap_vk(proof, vk_hash, &IMM_GROTH16_VK_BYTES, &part_stark_vk) + .map_err(|err| anyhow!("failed to convert operator proof to ark format: {err}")) +} + +fn check_operator_public_input_count(public_inputs: &[ark_bn254::Fr]) -> Result<()> { + if public_inputs.len() != NUM_PUBLIC_INPUTS { + bail!( + "operator proof has {} public inputs; expected {NUM_PUBLIC_INPUTS}", + public_inputs.len() + ); + } + Ok(()) +} + +fn operator_static_inputs(public_inputs: &[ark_bn254::Fr]) -> Result { + check_operator_public_input_count(public_inputs)?; + Ok(STATIC_SLOTS.map(|slot| public_inputs[slot])) +} + +pub fn operator_assert_dynamic_input(public_inputs: &[ark_bn254::Fr]) -> Result { + check_operator_public_input_count(public_inputs)?; + Ok(public_inputs[DYNAMIC_SLOTS[0]]) +} + +fn combined_operator_vk_hash(operator_vk_hash: &str, zkm_version: &str) -> Result<[u8; 32]> { + if !operator_vk_hash.starts_with("0x") { + bail!("configured operator vk hash must use 0x-prefixed Ziren encoding"); + } + let raw_vk_hash = decode_zkm_vkey_hash(operator_vk_hash) + .map_err(|e| anyhow!("invalid configured operator vk hash: {e:?}"))?; + let part_vk: PartStarkVerifyingKey = + bincode::deserialize(&load_part_stark_vk_for_zkm_version(zkm_version)?) + .context("deserialize operator partial STARK verifying key")?; + let base = Bn254Fr::from_u32(256); + let mut field_hash = Bn254Fr::ZERO; + for byte in raw_vk_hash { + field_hash = field_hash * base + Bn254Fr::from_u32(byte as u32); + } + let combined = zkm_recursion_core::hash_vkey_with_part_vk(&part_vk, field_hash); + let bytes = combined.as_canonical_biguint().to_bytes_be(); + if bytes.len() > 32 { + bail!("combined operator verifying key hash exceeds BN254 field encoding"); + } + let mut encoded = [0u8; 32]; + encoded[32 - bytes.len()..].copy_from_slice(&bytes); + Ok(encoded) +} + +fn operator_identity() -> Result<([u8; 32], String, StaticInputs)> { + let vk_hash = get_operator_vk_hash()?; + let zkm_version = get_operator_zkm_version()?; + let combined_hash = + combined_operator_vk_hash(&format!("0x{}", hex::encode(vk_hash)), &zkm_version)?; + let static_input = ziren_static_inputs(&combined_hash, &VK_ROOT_BYTES); + Ok((vk_hash, zkm_version, static_input)) +} + +pub fn derive_operator_static_input() -> Result { + Ok(operator_identity()?.2) +} + +/// Derives the Operator proof identity and graph constant for one challenge-init transaction. +pub fn derive_operator_statement( + graph_id: Uuid, + challenge_init_txid: [u8; 32], + watchtower_pubkeys: &[XOnlyPublicKey], +) -> Result { + let (vk_hash, zkm_version, static_input) = operator_identity()?; + let key_bytes = watchtower_pubkeys.iter().map(XOnlyPublicKey::serialize).collect::>(); + let constant = hash_operator_constant( + *graph_id.as_bytes(), + get_genesis_sequencer_commit_id(), + challenge_init_txid, + &key_bytes, + ); + Ok(OperatorStatement { static_input, vk_hash, zkm_version, constant }) +} + +/// Returns: +/// - `Ok(Some(OperatorProof), _)` if operator proof is available and valid +/// - `Ok(None, wait_secs)` if operator proof is not yet available +pub async fn get_operator_proof( + local_db: &LocalDB, + http_client: &HttpAsyncClient, + bitvm_graph: &BitvmGcGraph, + btc_client: &BTCClient, + instance_id: Uuid, + graph_id: Uuid, +) -> Result<(Option, usize)> { + let mut storage_processor = local_db.acquire().await?; + let Some(graph) = storage_processor.find_graph(&graph_id).await? else { + warn!("graph:{graph_id} not found"); + bail!("No graph in db"); + }; + drop(storage_processor); + + if graph.proceed_withdraw_height <= 0 { + warn!("graph {graph_id} proceed_withdraw_height <= 0, waiting to been updated"); + return Ok((None, get_operator_proof_wait_secs())); + } + + let watchtower_challenge_init_txid = graph + .watchtower_challenge_init_txid + .ok_or_else(|| anyhow::anyhow!("watchtower_challenge_init_txid is none"))?; + let num_challenger = bitvm_graph.parameters.watchtower_pubkeys.len(); + let watchtower_timeout_txids: Vec = + bitvm_graph.watchtower_challenge_timeouts.iter().map(|tx| tx.tx().compute_txid()).collect(); + let WatchtowerChallengeInfo { + challenge_txids: watchtower_challenge_txids, + included_watchtowers, + resolved_branch_txids, + } = match get_watchtower_challenge_info( + btc_client, + &watchtower_challenge_init_txid, + &watchtower_timeout_txids, + num_challenger, + ) + .await + { + Ok(info) => info, + Err(e) => { + warn!("Failed to get watchtower challenge info: {e}"); + return Ok((None, get_operator_proof_wait_secs())); + } + }; + let Some(operator_committed_blockhash) = get_operator_committed_blockhash( + btc_client, + &resolved_branch_txids, + &included_watchtowers, + graph_id, + ) + .await? + else { + return Ok((None, get_operator_proof_wait_secs())); + }; + + let base_url = Url::parse( + &get_proof_build_rpc_host() + .ok_or_else(|| anyhow::anyhow!("failed to get proof_build_rpc_host"))?, + )?; + let operator_url = base_url.join(NODES_OPERATOR_BASE)?; + let payload = OperatorProofRequest { + instance_id: instance_id.to_string(), + graph_id: graph_id.to_string(), + gateway_address: Some(env::get_goat_gateway_contract_from_env().to_string()), + operator_committed_blockhash, + execution_layer_block_number: graph.proceed_withdraw_height, + watchtower_challenge_txids, + included_watchtowers, + watchtower_challenge_init_txid: watchtower_challenge_init_txid.0.to_string(), + watchtower_challenge_pubkeys: bitvm_graph + .parameters + .watchtower_pubkeys + .iter() + .map(|pk| pk.public_key(secp256k1::Parity::Even).to_string()) + .collect(), + }; + let auth_keypair = get_bitvm_key()?; + let operator_response = http_client + .post_response_json_with_dynamic_headers::( + operator_url.as_str(), + &payload, + || { + proof_builder_auth_headers( + &auth_keypair, + ProofBuilderAuthRole::Operator, + NODES_OPERATOR_BASE, + &payload, + ) + }, + ) + .await + .inspect(|_| crate::metrics_service::record_backend_ok("proof")) + .inspect_err(|error| { + crate::metrics_service::record_backend_failure("proof", "operator_proofs", error) + })?; + + let Some(proof_data) = operator_response.proof_data else { + if let Some(error) = operator_response.error { + info!("operator proof is not ready for graph_id:{graph_id}: {error}"); + } + return Ok((None, get_operator_proof_wait_secs())); + }; + + let statement = derive_operator_statement( + graph_id, + bitvm_graph.watchtower_challenge_init.tx().compute_txid().to_byte_array(), + &bitvm_graph.parameters.watchtower_pubkeys, + )?; + if statement.constant != bitvm_graph.parameters.pubin_disprove_constant { + bail!("graph operator constant does not match its watchtower list"); + } + + let proof: ZKMProofWithPublicValues = bincode::deserialize(proof_data.proof.as_slice()) + .map_err(|err| anyhow!("failed to deserialize operator proof: {err}"))?; + + let operator_vk_hash_raw = decode_zkm_vkey_hash(&proof_data.vk) + .map_err(|e| anyhow!("invalid operator proof vk hash: {e:?}"))?; + if operator_vk_hash_raw != statement.vk_hash { + bail!("operator proof vk hash does not match configured operator identity"); + } + if proof.zkm_version != statement.zkm_version || proof_data.zkm_version != statement.zkm_version + { + bail!("operator proof Ziren version does not match configured operator identity"); + } + + let outputs = decode_operator_public_outputs(&proof.public_values.to_vec()) + .map_err(|e| anyhow!("invalid operator public outputs: {e}"))?; + if outputs.constant != statement.constant { + bail!("operator proof constant does not match graph setup"); + } + + let ark_proof = convert_operator_proof_to_ark(&proof, &proof_data.vk)?; + if operator_static_inputs(&ark_proof.public_inputs)? != statement.static_input { + bail!("operator proof static public inputs do not match graph setup statement"); + } + + Ok(( + Some(ValidatedOperatorProof { + proof: ark_proof.proof, + public_inputs: ark_proof.public_inputs.into(), + verifying_key: ark_proof.groth16_vk.into(), + public_values: proof.public_values.to_vec(), + vk_hash: proof_data.vk, + zkm_version: proof.zkm_version, + }), + 0, + )) +} + +/// Tell the proof builder that this watchtower's proof for the graph is no +/// longer needed: the operator asserted, the watchtower window is over. +/// Returns whether the builder accepted it; without a builder host, `false`. +pub async fn notify_watchtower_proof_timeout(instance_id: Uuid, graph_id: Uuid) -> Result { + let Some(host) = get_proof_build_rpc_host() else { + warn!("notify_watchtower_proof_timeout: proof builder host is not configured"); + return Ok(false); + }; + let url = Url::parse(&host)?.join(PROOFS_WATCHTOWER_PROOF_TIMEOUT)?; + let payload = WatchtowerProofTimeoutUpdateRequest { + instance_id: instance_id.to_string(), + graph_id: graph_id.to_string(), + gateway_address: Some(env::get_goat_gateway_contract_from_env().to_string()), + public_key: get_node_pubkey()?.to_string(), + }; + let auth_keypair = get_bitvm_key()?; + // Called once per graph at most; a fresh client is enough. + let response = HttpAsyncClient::new(None) + .post_response_json_with_dynamic_headers::( + url.as_str(), + &payload, + || { + proof_builder_auth_headers( + &auth_keypair, + ProofBuilderAuthRole::Watchtower, + PROOFS_WATCHTOWER_PROOF_TIMEOUT, + &payload, + ) + }, + ) + .await?; + info!("call {}, response:{:?}", PROOFS_WATCHTOWER_PROOF_TIMEOUT, response); + Ok(response.data.is_some()) +} + +/// Computes the graph constant committed by the Operator guest and Connector D. +pub fn get_guest_constant_value( + graph_id: Uuid, + challenge_init_txid: [u8; 32], + watchtower_pubkeys: &[XOnlyPublicKey], +) -> Result<[u8; 32]> { + let key_bytes = watchtower_pubkeys.iter().map(XOnlyPublicKey::serialize).collect::>(); + Ok(hash_operator_constant( + graph_id.into_bytes(), + get_genesis_sequencer_commit_id(), + challenge_init_txid, + &key_bytes, + )) +} + +#[cfg(test)] +mod commit_pubin_tests { + use super::*; + use ark_serialize::CanonicalSerialize; + use bitcoin::BlockHash; + use bitcoin::ScriptBuf; + use client::btc_chain::BTCClient; + use esplora_client::{Tx, TxStatus, Vin}; + use store::SerializableTxid; + + fn make_txid(byte: u8) -> Txid { + Txid::from_slice(&[byte; 32]).unwrap() + } + + fn make_block_hash(byte: u8) -> BlockHash { + BlockHash::from_slice(&[byte; 32]).unwrap() + } + + fn create_confirmed_tx( + txid: Txid, + spends: &[(Txid, u32)], + height: u32, + block_hash: BlockHash, + ) -> Tx { + Tx { + txid, + version: 2, + locktime: 0, + vin: spends + .iter() + .map(|(prev_txid, vout)| Vin { + txid: *prev_txid, + vout: *vout, + prevout: None, + scriptsig: ScriptBuf::default(), + witness: vec![], + sequence: 1, + is_coinbase: false, + }) + .collect(), + vout: vec![], + size: 1, + weight: 1, + status: TxStatus { + confirmed: true, + block_height: Some(height), + block_hash: Some(block_hash), + block_time: Some(1_000_000), + }, + fee: 0, + } + } + + #[test] + fn assert_ready_extracts_blake3_xd_from_operator_public_values() { + let mut pubin = [0u8; 96]; + pubin[..32].copy_from_slice(&[0x11; 32]); + pubin[32..64].copy_from_slice(&[0x22; 32]); + pubin[64..].copy_from_slice(&[0x33; 32]); + let outputs = decode_operator_public_outputs(&pubin).unwrap(); + assert_eq!(outputs.btc_best_block_hash, [0x11; 32]); + assert_eq!(outputs.constant, [0x22; 32]); + assert_eq!(outputs.included_watchtowers, [0x33; 32]); + + let public_inputs: PublicInputs = zkm_verifier::load_ark_public_inputs_from_bytes( + &[0u8; 32], + &zkm_verifier::hash_public_inputs(&pubin), + &VK_ROOT_BYTES, + ) + .into(); + let dynamic_input = operator_assert_dynamic_input(&public_inputs).unwrap(); + assert_eq!( + operator_static_inputs(&public_inputs).unwrap(), + ziren_static_inputs(&[0u8; 32], &VK_ROOT_BYTES) + ); + + let mut expected_xd = *blake3::hash(&pubin).as_bytes(); + expected_xd[0] &= 0x1f; + let mut actual_xd = Vec::new(); + dynamic_input.serialize_uncompressed(&mut actual_xd).unwrap(); + // `pi1_xd_to_wots96_msg` serializes the field element most-significant byte first, + // whereas the public-input digest is defined in little-endian order. + actual_xd.reverse(); + assert_eq!(actual_xd, expected_xd); + } + + #[tokio::test] + async fn test_get_watchtower_challenge_info_partial_inclusion() { + let (btc_client, mock_adaptor) = BTCClient::new_mock_client(); + + let watchtower_init_txid = make_txid(0x00); + let challenge_txid_wt0 = make_txid(0x01); + let block_hash = make_block_hash(0xAA); + + let challenge_vout_0 = + output_topology::watchtower_challenge_init::watchtower_connector(0) as u32; + let ack_vout_0 = output_topology::watchtower_challenge_init::ack_connector(0) as u32; + + // watchtower 0 spent its challenge connector and ACK connector; watchtower 1 is unresolved. + mock_adaptor.set_tx( + challenge_txid_wt0, + create_confirmed_tx( + challenge_txid_wt0, + &[(watchtower_init_txid, challenge_vout_0)], + 100, + block_hash, + ), + ); + let ack_txid_wt0 = make_txid(0x02); + mock_adaptor.set_tx( + ack_txid_wt0, + create_confirmed_tx( + ack_txid_wt0, + &[(watchtower_init_txid, ack_vout_0)], + 101, + block_hash, + ), + ); + + let info = get_watchtower_challenge_info( + &btc_client, + &SerializableTxid::from(watchtower_init_txid), + &[], + 2, // 2 watchtowers + ) + .await + .unwrap(); + + assert_eq!(info.challenge_txids, vec![Some(challenge_txid_wt0.to_string()), None]); + assert_eq!(info.included_watchtowers, vec![true, false]); + assert_eq!(info.resolved_branch_txids, vec![challenge_txid_wt0]); + } + + #[tokio::test] + async fn test_compute_operator_pubin_blockhash_and_bitmap() { + let (btc_client, mock_adaptor) = BTCClient::new_mock_client(); + + let init_txid = make_txid(0x00); + let challenge_txid_wt0 = make_txid(0x01); + let timeout_txid_wt1 = make_txid(0x03); + let challenge_txid_wt2 = make_txid(0x02); + let block_hash_low = make_block_hash(0x10); + let block_hash_middle = make_block_hash(0x15); + let block_hash_high = make_block_hash(0x20); + + let challenge_vout_0 = + output_topology::watchtower_challenge_init::watchtower_connector(0) as u32; + let challenge_vout_2 = + output_topology::watchtower_challenge_init::watchtower_connector(2) as u32; + + // All three branches are finalized; wt2's height 200 block is the anchor. + let required_burial_depth = get_btc_block_confirms(btc_client.network()); + mock_adaptor.set_height(200 + required_burial_depth + 1); + mock_adaptor.set_tx( + challenge_txid_wt0, + create_confirmed_tx( + challenge_txid_wt0, + &[(init_txid, challenge_vout_0)], + 100, + block_hash_low, + ), + ); + mock_adaptor.set_tx( + timeout_txid_wt1, + create_confirmed_tx(timeout_txid_wt1, &[(init_txid, 1)], 150, block_hash_middle), + ); + mock_adaptor.set_tx( + challenge_txid_wt2, + create_confirmed_tx( + challenge_txid_wt2, + &[(init_txid, challenge_vout_2)], + 200, + block_hash_high, + ), + ); + + let resolved_branch_txids = vec![challenge_txid_wt0, timeout_txid_wt1, challenge_txid_wt2]; + let bits = vec![true, false, true]; + + let (best_hash, bitmap) = + compute_operator_pubin_blockhash_and_bitmap(&btc_client, &resolved_branch_txids, &bits) + .await + .unwrap(); + + // block at height 200 wins + assert_eq!(best_hash, block_hash_high.to_byte_array()); + + // bits 0 and 2 set → byte 0 = 0b0000_0101 + assert_eq!(bitmap[0], 0b0000_0101u8); + assert_eq!(&bitmap[1..], &[0u8; 31]); + } +} diff --git a/node/src/utils/rounds.rs b/node/src/utils/rounds.rs new file mode 100644 index 000000000..199c1fe6d --- /dev/null +++ b/node/src/utils/rounds.rs @@ -0,0 +1,703 @@ +//! Committee round material: nonces, partial signatures and endorsements per +//! graph and per instance. + +use crate::error::SpecialError; +use alloy::primitives::Address as EvmAddress; +use anyhow::{Result, anyhow, bail}; +use bitcoin::{PublicKey, XOnlyPublicKey}; +use bitvm_lib::committee::*; +use indexmap::IndexMap; +use musig2::{PartialSignature, PubNonce}; +use secp256k1::schnorr::Signature as SchnorrSignature; +use serde::{Deserialize, Serialize}; +use store::localdb::LocalDB; + +use uuid::Uuid; + +use super::graph_store::{find_pegin_graph_process_data, upsert_pegin_graph_process_data}; +use super::graph_validate::min_required_operator; +use super::instance_store::{find_pegin_instance_process_data, upsert_pegin_instance_process_data}; + +#[derive(Clone, Serialize, Deserialize, Debug, Default)] +pub struct InstanceProcessDataItem { + pub pub_nonce: Option, + pub agg_nonce_consensus_hash: Option<[u8; 32]>, + pub agg_nonce_consensus_signature: Option, + pub partial_sign: Option, + pub endorse_signature: Vec, +} + +pub type InstanceProcessDataMap = IndexMap; + +#[derive(Clone, Serialize, Deserialize, Default)] +pub struct GraphProcessDataItem { + pub committee_pub_nonce: Option, + pub agg_nonce_consensus_hash: Option<[u8; 32]>, + pub agg_nonce_consensus_signature: Option, + pub partial_sigs: Option, + pub committee_evm_address: Option, + pub endorse_signature: Vec, + pub params_endorse_signature: Vec, + pub verifier_index: Option, + pub verifier_peer_id: Option, + pub verifier_params_signature: Option, +} + +pub type GraphProcessDataMap = IndexMap; + +pub fn order_committee_values( + committee_pubkeys: &[PublicKey], + values: Vec<(PublicKey, T)>, + label: &str, +) -> Result> { + for (idx, (pubkey, _)) in values.iter().enumerate() { + if !committee_pubkeys.contains(pubkey) { + bail!("{label} contains non-committee pubkey {pubkey}"); + } + if values.iter().skip(idx + 1).any(|(other, _)| other == pubkey) { + bail!("{label} contains duplicate pubkey {pubkey}"); + } + } + + committee_pubkeys + .iter() + .map(|committee_pubkey| { + values + .iter() + .find(|(pubkey, _)| pubkey == committee_pubkey) + .map(|(_, value)| value.clone()) + .ok_or_else(|| anyhow!("{label} missing pubkey {committee_pubkey}")) + }) + .collect() +} + +pub async fn store_committee_pub_nonces_for_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + committee_pubkey: PublicKey, + pub_nonces: CommitteePubNonces, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (is_endorsed, mut process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + if let Some(existing_pub_nonces) = + process_data.get(&committee_pubkey).and_then(|v| v.committee_pub_nonce.as_ref()) + && existing_pub_nonces != &pub_nonces + { + bail!(SpecialError::InvalidGraph(format!( + "committee pub nonces changed for graph {graph_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|v| v.committee_pub_nonce = Some(pub_nonces.clone())) + .or_insert_with(|| GraphProcessDataItem { + committee_pub_nonce: Some(pub_nonces), + ..Default::default() + }); + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + is_endorsed, + &process_data, + ) + .await?; + Ok(()) +} + +pub async fn get_committee_pub_nonces_for_graph( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| v.committee_pub_nonce.as_ref().map(|nonce| (*k, nonce.clone()))) + .collect::>()) +} + +pub async fn store_committee_agg_nonce_consensus_for_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + committee_pubkey: PublicKey, + consensus_hash: [u8; 32], + signature: SchnorrSignature, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (is_endorsed, mut process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + if let Some((existing_hash, existing_signature)) = process_data + .get(&committee_pubkey) + .and_then(|item| item.agg_nonce_consensus_hash.zip(item.agg_nonce_consensus_signature)) + && (existing_hash != consensus_hash || existing_signature != signature) + { + tracing::error!( + event = "committee_agg_nonce_consensus_conflict", + %instance_id, + %graph_id, + %committee_pubkey, + existing_consensus_hash = %hex::encode(existing_hash), + existing_signature = %existing_signature, + conflicting_consensus_hash = %hex::encode(consensus_hash), + conflicting_signature = %signature, + "detected conflicting committee agg nonce consensus signatures" + ); + bail!(SpecialError::InvalidGraph(format!( + "committee agg nonce consensus changed for graph {graph_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|item| { + item.agg_nonce_consensus_hash = Some(consensus_hash); + item.agg_nonce_consensus_signature = Some(signature); + }) + .or_insert_with(|| GraphProcessDataItem { + agg_nonce_consensus_hash: Some(consensus_hash), + agg_nonce_consensus_signature: Some(signature), + ..Default::default() + }); + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + is_endorsed, + &process_data, + ) + .await?; + Ok(()) +} + +pub async fn get_committee_agg_nonce_consensus_for_graph( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data + .iter() + .filter_map(|(pubkey, item)| { + item.agg_nonce_consensus_hash + .zip(item.agg_nonce_consensus_signature) + .map(|(consensus_hash, signature)| (*pubkey, consensus_hash, signature)) + }) + .collect()) +} + +pub async fn store_committee_partial_sigs_for_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + committee_pubkey: PublicKey, + partial_sigs: CommitteePartialSignatures, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (is_endorsed, mut process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + if let Some(existing_partial_sigs) = + process_data.get(&committee_pubkey).and_then(|v| v.partial_sigs.as_ref()) + && existing_partial_sigs != &partial_sigs + { + bail!(SpecialError::InvalidGraph(format!( + "committee partial signatures changed for graph {graph_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|v| v.partial_sigs = Some(partial_sigs.clone())) + .or_insert_with(|| GraphProcessDataItem { + partial_sigs: Some(partial_sigs), + ..Default::default() + }); + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + is_endorsed, + &process_data, + ) + .await?; + Ok(()) +} + +pub async fn get_committee_partial_sigs_for_graph( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| v.partial_sigs.as_ref().map(|nonce| (*k, nonce.clone()))) + .collect::>()) +} + +pub async fn get_committee_partial_sigs_for_graph_member( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, + committee_pubkey: &PublicKey, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data.get(committee_pubkey).and_then(|v| v.partial_sigs.clone())) +} + +pub async fn store_committee_endorsement_for_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + committee_pubkey: PublicKey, + committee_evm_address: EvmAddress, + endorse_signature: Vec, + params_endorse_signature: Vec, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (is_endorsed, mut process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + process_data + .entry(committee_pubkey) + .and_modify(|v| { + v.endorse_signature = endorse_signature.clone(); + v.params_endorse_signature = params_endorse_signature.clone(); + v.committee_evm_address = Some(committee_evm_address); + }) + .or_insert_with(|| GraphProcessDataItem { + committee_evm_address: Some(committee_evm_address), + endorse_signature, + params_endorse_signature, + ..Default::default() + }); + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + is_endorsed, + &process_data, + ) + .await?; + Ok(()) +} + +pub async fn store_committee_endorsements_for_graph( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, + params_endorse_sigs: Vec<(PublicKey, EvmAddress, Vec)>, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (is_endorsed, mut process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + + for (committee_pubkey, committee_evm_address, endorse_signature) in endorse_sigs { + process_data + .entry(committee_pubkey) + .and_modify(|v| { + v.endorse_signature = endorse_signature.clone(); + v.committee_evm_address = Some(committee_evm_address); + }) + .or_insert_with(|| GraphProcessDataItem { + committee_evm_address: Some(committee_evm_address), + endorse_signature, + ..Default::default() + }); + } + for (committee_pubkey, committee_evm_address, params_endorse_signature) in params_endorse_sigs { + process_data + .entry(committee_pubkey) + .and_modify(|v| { + v.params_endorse_signature = params_endorse_signature.clone(); + v.committee_evm_address = Some(committee_evm_address); + }) + .or_insert_with(|| GraphProcessDataItem { + committee_evm_address: Some(committee_evm_address), + params_endorse_signature, + ..Default::default() + }); + } + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + is_endorsed, + &process_data, + ) + .await?; + Ok(()) +} + +pub async fn get_committee_endorsements_for_graph( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, +) -> Result)>> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| { + if !v.endorse_signature.is_empty() { + v.committee_evm_address + .as_ref() + .map(|evm_addr| (*k, *evm_addr, v.endorse_signature.clone())) + } else { + None + } + }) + .collect::)>>()) +} + +pub async fn get_committee_params_endorsements_for_graph( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, +) -> Result)>> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| { + if !v.params_endorse_signature.is_empty() { + v.committee_evm_address + .as_ref() + .map(|evm_addr| (*k, *evm_addr, v.params_endorse_signature.clone())) + } else { + None + } + }) + .collect::)>>()) +} + +pub async fn store_verifier_graph_params_endorsement( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + verifier_pubkey: PublicKey, + verifier_index: usize, + verifier_peer_id: &str, + signature: SchnorrSignature, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (is_endorsed, mut process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + for (key, item) in &process_data { + if let Some(peer) = &item.verifier_peer_id + && ((*key == verifier_pubkey && peer != verifier_peer_id) + || (*key != verifier_pubkey + && (peer == verifier_peer_id + || XOnlyPublicKey::from(*key) == XOnlyPublicKey::from(verifier_pubkey)))) + { + bail!(SpecialError::InvalidGraph("conflicting verifier peer/key binding".into())); + } + } + if let Some(existing_index) = process_data.get(&verifier_pubkey).and_then(|v| v.verifier_index) + && existing_index != verifier_index + { + bail!(SpecialError::InvalidGraph(format!( + "verifier params endorsement index changed for graph {graph_id} and verifier {verifier_pubkey}" + ))); + } + if let Some(existing_signature) = + process_data.get(&verifier_pubkey).and_then(|v| v.verifier_params_signature.as_ref()) + && existing_signature != &signature + { + bail!(SpecialError::InvalidGraph(format!( + "verifier params endorsement signature changed for graph {graph_id} and verifier {verifier_pubkey}" + ))); + } + process_data + .entry(verifier_pubkey) + .and_modify(|v| { + v.verifier_index = Some(verifier_index); + v.verifier_peer_id = Some(verifier_peer_id.to_owned()); + v.verifier_params_signature = Some(signature); + }) + .or_insert_with(|| GraphProcessDataItem { + verifier_index: Some(verifier_index), + verifier_peer_id: Some(verifier_peer_id.to_owned()), + verifier_params_signature: Some(signature), + ..Default::default() + }); + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + is_endorsed, + &process_data, + ) + .await?; + Ok(()) +} + +pub async fn get_verifier_graph_params_endorsements_for_graph( + local_db: &LocalDB, + _instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let (_is_endorsed, process_data) = + find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| { + v.verifier_index + .zip(v.verifier_params_signature.as_ref()) + .zip(v.verifier_peer_id.as_ref()) + .map(|((index, signature), peer)| (*k, index, peer.clone(), *signature)) + }) + .collect()) +} + +pub async fn mark_graph_as_endorsed( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let (_, process_data) = find_pegin_graph_process_data(&mut storage_processor, graph_id).await?; + upsert_pegin_graph_process_data( + &mut storage_processor, + graph_id, + instance_id, + true, + &process_data, + ) + .await +} + +pub async fn get_endorsed_graph_count(local_db: &LocalDB, instance_id: Uuid) -> Result { + let mut storage_processor = local_db.acquire().await?; + Ok(storage_processor.get_pegin_graph_endorsed_len_by_instance_id(&instance_id, true).await? + as usize) +} + +pub async fn has_required_presigned_graphs(local_db: &LocalDB, instance_id: Uuid) -> Result { + Ok(get_endorsed_graph_count(local_db, instance_id).await? >= min_required_operator()) +} + +pub async fn store_committee_pub_nonce_for_instance( + local_db: &LocalDB, + instance_id: Uuid, + committee_pubkey: PublicKey, + pub_nonce: PubNonce, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let mut process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + if let Some(existing_pub_nonce) = + process_data.get(&committee_pubkey).and_then(|v| v.pub_nonce.as_ref()) + && existing_pub_nonce != &pub_nonce + { + bail!(SpecialError::InvalidPeginData(format!( + "committee pub nonce changed for instance {instance_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|v| v.pub_nonce = Some(pub_nonce.clone())) + .or_insert_with(|| InstanceProcessDataItem { + pub_nonce: Some(pub_nonce), + ..Default::default() + }); + upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; + Ok(()) +} + +pub async fn get_committee_pub_nonce_for_instance( + local_db: &LocalDB, + instance_id: Uuid, + committee_pubkey: &PublicKey, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + Ok(process_data.get(committee_pubkey).and_then(|v| v.pub_nonce.clone())) +} + +pub async fn get_committee_pub_nonces_for_instance( + local_db: &LocalDB, + instance_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| v.pub_nonce.as_ref().map(|pub_nonce| (*k, pub_nonce.clone()))) + .collect()) +} + +pub async fn store_committee_agg_nonce_consensus_for_instance( + local_db: &LocalDB, + instance_id: Uuid, + committee_pubkey: PublicKey, + consensus_hash: [u8; 32], + signature: SchnorrSignature, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let mut process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + if let Some((existing_hash, existing_signature)) = process_data + .get(&committee_pubkey) + .and_then(|item| item.agg_nonce_consensus_hash.zip(item.agg_nonce_consensus_signature)) + && (existing_hash != consensus_hash || existing_signature != signature) + { + tracing::error!( + event = "committee_pegin_confirm_nonce_consensus_conflict", + %instance_id, + %committee_pubkey, + existing_consensus_hash = %hex::encode(existing_hash), + existing_signature = %existing_signature, + conflicting_consensus_hash = %hex::encode(consensus_hash), + conflicting_signature = %signature, + "detected conflicting committee PeginConfirm nonce consensus signatures" + ); + bail!(SpecialError::InvalidPeginData(format!( + "committee PeginConfirm nonce consensus changed for instance {instance_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|item| { + item.agg_nonce_consensus_hash = Some(consensus_hash); + item.agg_nonce_consensus_signature = Some(signature); + }) + .or_insert_with(|| InstanceProcessDataItem { + agg_nonce_consensus_hash: Some(consensus_hash), + agg_nonce_consensus_signature: Some(signature), + ..Default::default() + }); + upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; + Ok(()) +} + +pub async fn get_committee_agg_nonce_consensus_for_instance( + local_db: &LocalDB, + instance_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + Ok(process_data + .iter() + .filter_map(|(pubkey, item)| { + item.agg_nonce_consensus_hash + .zip(item.agg_nonce_consensus_signature) + .map(|(consensus_hash, signature)| (*pubkey, consensus_hash, signature)) + }) + .collect()) +} + +pub async fn store_committee_partial_sig_for_instance( + local_db: &LocalDB, + instance_id: Uuid, + committee_pubkey: PublicKey, + partial_sigs: PartialSignature, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let mut process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + if let Some(existing_partial_sig) = + process_data.get(&committee_pubkey).and_then(|v| v.partial_sign.as_ref()) + && existing_partial_sig != &partial_sigs + { + bail!(SpecialError::InvalidPeginData(format!( + "committee partial signature changed for instance {instance_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|v| v.partial_sign = Some(partial_sigs)) + .or_insert_with(|| InstanceProcessDataItem { + partial_sign: Some(partial_sigs), + ..Default::default() + }); + upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; + Ok(()) +} + +pub async fn get_committee_partial_sigs_for_instance( + local_db: &LocalDB, + instance_id: Uuid, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| v.partial_sign.as_ref().map(|partial_sign| (*k, *partial_sign))) + .collect()) +} + +pub async fn get_committee_partial_sig_for_instance( + local_db: &LocalDB, + instance_id: Uuid, + committee_pubkey: &PublicKey, +) -> Result> { + let mut storage_processor = local_db.acquire().await?; + let process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + Ok(process_data.get(committee_pubkey).and_then(|v| v.partial_sign)) +} + +pub async fn store_committee_endorse_sig_for_pegin( + local_db: &LocalDB, + instance_id: Uuid, + committee_pubkey: PublicKey, + endorse_sig: Vec, +) -> Result<()> { + let mut storage_processor = local_db.acquire().await?; + let mut process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + if let Some(existing_endorse_sig) = + process_data.get(&committee_pubkey).map(|v| v.endorse_signature.as_slice()) + && !existing_endorse_sig.is_empty() + && existing_endorse_sig != endorse_sig.as_slice() + { + bail!(SpecialError::InvalidPeginData(format!( + "committee endorse signature changed for instance {instance_id} and committee {committee_pubkey}" + ))); + } + process_data + .entry(committee_pubkey) + .and_modify(|v| v.endorse_signature = endorse_sig.clone()) + .or_insert_with(|| InstanceProcessDataItem { + endorse_signature: endorse_sig, + ..Default::default() + }); + upsert_pegin_instance_process_data(&mut storage_processor, instance_id, &process_data).await?; + Ok(()) +} + +pub async fn get_committee_endorse_sigs_for_pegin( + local_db: &LocalDB, + instance_id: Uuid, +) -> Result)>> { + let mut storage_processor = local_db.acquire().await?; + let process_data = + find_pegin_instance_process_data(&mut storage_processor, instance_id).await?; + Ok(process_data + .iter() + .filter_map(|(k, v)| { + if !v.endorse_signature.is_empty() { + Some((*k, v.endorse_signature.clone())) + } else { + None + } + }) + .collect::)>>()) +} diff --git a/node/src/utils/setup_state.rs b/node/src/utils/setup_state.rs new file mode 100644 index 000000000..1b2be61d1 --- /dev/null +++ b/node/src/utils/setup_state.rs @@ -0,0 +1,327 @@ +//! The BABE setup state and soldering payloads kept on disk during graph setup. + +use crate::env::*; +use crate::messages::SolderingProofReady; +use crate::soldering_payload_store::{ + delete_soldering_proof_store_payload, is_soldering_proof_s3_path, + soldering_proof_payload_store_path, write_soldering_proof_store_payload, +}; +use anyhow::{Context, Result, bail}; +use bitcoin::PublicKey; +use bitvm_lib::types::BitvmGcCircuitData; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::{Duration, SystemTime}; +use store::localdb::LocalDB; + +use bitvm_lib::babe_adapter::{ + BabeVerifierPrivateState, CACSetupPackage, FinalizedInstanceData, Seed, SolderingData, + compact_soldering_proof_payload, +}; +use store::GraphStatus; +use tracing::warn; +use uuid::Uuid; + +const BABE_SETUP_STATE_ORPHAN_RETENTION: Duration = Duration::from_secs(24 * 60 * 60); + +pub(crate) fn babe_setup_state_root(local_db: &LocalDB) -> PathBuf { + std::env::var_os(ENV_BABE_SETUP_STATE_DIR) + .filter(|path| !path.as_os_str().is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| { + let db_path = local_db + .path + .strip_prefix("sqlite://") + .or_else(|| local_db.path.strip_prefix("sqlite:")) + .unwrap_or(&local_db.path); + PathBuf::from(db_path) + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")) + .join(".bitvm-babe-state") + }) +} + +fn babe_setup_state_path(local_db: &LocalDB, instance_id: Uuid, graph_id: Uuid) -> PathBuf { + babe_setup_state_root(local_db).join(instance_id.to_string()).join(format!("{graph_id}.bin")) +} + +pub(crate) fn soldering_payload_hash(payload: &[u8]) -> [u8; 32] { + Sha256::digest(payload).into() +} + +pub(crate) fn soldering_payload_hash_hex(payload_hash: &[u8; 32]) -> String { + payload_hash.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn save_soldering_proof_payload( + instance_id: Uuid, + graph_id: Uuid, + candidate_index: usize, + opened: &[(usize, Seed)], + finalized: &[FinalizedInstanceData], + soldering: &SolderingData, +) -> Result { + let compact_payload = compact_soldering_proof_payload(opened, finalized, soldering)?; + let payload = bincode::serialize(&compact_payload) + .context("serialize compact soldering proof payload")?; + let payload_hash = soldering_payload_hash(&payload); + let total_len = payload.len(); + let store_base_path = get_soldering_proof_payload_store_path()?; + let payload_path = soldering_proof_payload_store_path( + &store_base_path, + instance_id, + graph_id, + candidate_index, + &payload_hash, + )?; + let store_mode = if is_soldering_proof_s3_path(&store_base_path) { "s3" } else { "local" }; + if let Err(err) = + write_soldering_proof_store_payload(&payload_path, &payload).await.with_context(|| { + format!("write soldering proof payload to configured store path {payload_path}") + }) + { + tracing::error!( + store_mode, + total_len, + payload_hash = %soldering_payload_hash_hex(&payload_hash), + payload_path = %payload_path, + error = %err, + "failed to write soldering proof payload to store" + ); + return Err(err); + } + tracing::info!( + store_mode, + total_len, + payload_hash = %soldering_payload_hash_hex(&payload_hash), + payload_path = %payload_path, + "saved compact soldering proof to payload store" + ); + Ok(SolderingProofReady { instance_id, graph_id, candidate_index, payload_hash, total_len }) +} + +fn load_babe_setup_state_from_path(path: &Path) -> Result> { + match std::fs::read(path) { + Ok(bytes) => Ok(Some( + bincode::deserialize(&bytes) + .with_context(|| format!("decode BABE setup state {}", path.display()))?, + )), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err).with_context(|| format!("read BABE setup state {}", path.display())), + } +} + +fn save_babe_setup_state_to_path(path: &Path, state: &BabeSetupState) -> Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create BABE setup state dir {}", parent.display()))?; + } + let bytes = bincode::serialize(state)?; + let mut temp_path = path.as_os_str().to_os_string(); + temp_path.push(".tmp"); + let temp_path = PathBuf::from(temp_path); + std::fs::write(&temp_path, bytes) + .with_context(|| format!("write BABE setup state {}", temp_path.display()))?; + std::fs::rename(&temp_path, path).with_context(|| { + format!("move BABE setup state {} to {}", temp_path.display(), path.display()) + }) +} + +pub(crate) fn load_babe_setup_state( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, +) -> Result> { + load_babe_setup_state_from_path(&babe_setup_state_path(local_db, instance_id, graph_id)) +} + +pub(crate) fn save_babe_setup_state( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + state: &BabeSetupState, +) -> Result<()> { + save_babe_setup_state_to_path(&babe_setup_state_path(local_db, instance_id, graph_id), state) +} + +pub(crate) fn update_babe_setup_state( + local_db: &LocalDB, + instance_id: Uuid, + graph_id: Uuid, + update: impl FnOnce(&mut BabeSetupState), +) -> Result { + let mut state = load_babe_setup_state(local_db, instance_id, graph_id)?.unwrap_or_default(); + update(&mut state); + save_babe_setup_state(local_db, instance_id, graph_id, &state)?; + Ok(state) +} + +fn babe_setup_state_is_stale(path: &Path, now: SystemTime) -> Result { + let modified = std::fs::metadata(path) + .with_context(|| format!("read BABE setup state metadata {}", path.display()))? + .modified() + .with_context(|| format!("read BABE setup state modified time {}", path.display()))?; + Ok(now.duration_since(modified).unwrap_or_default() >= BABE_SETUP_STATE_ORPHAN_RETENTION) +} + +fn should_cleanup_babe_setup_state(status: Option, is_stale: bool) -> bool { + match status { + Some(status) if status.is_closed() => true, + Some(status) if status.is_obsoleted() => is_stale, + None => is_stale, + Some(_) => false, + } +} + +pub(crate) async fn cleanup_babe_setup_states( + local_db: &LocalDB, + payload_store: &str, +) -> Result { + let root = babe_setup_state_root(local_db); + let instance_dirs = match std::fs::read_dir(&root) { + Ok(entries) => entries, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(0), + Err(err) => { + return Err(err) + .with_context(|| format!("read BABE setup state root {}", root.display())); + } + }; + let mut storage = local_db.acquire().await?; + let now = SystemTime::now(); + let mut deleted = 0; + for instance_dir in instance_dirs { + let instance_dir = instance_dir?; + if !instance_dir.file_type()?.is_dir() { + continue; + } + let Ok(instance_id) = Uuid::parse_str(&instance_dir.file_name().to_string_lossy()) else { + continue; + }; + for state_file in std::fs::read_dir(instance_dir.path())? { + let state_file = state_file?; + let path = state_file.path(); + if path.extension().and_then(|extension| extension.to_str()) != Some("bin") { + continue; + } + let Some(graph_id) = path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| Uuid::parse_str(stem).ok()) + else { + continue; + }; + let is_stale = babe_setup_state_is_stale(&path, now)?; + let status = match storage.find_graph(&graph_id).await? { + Some(graph) if graph.instance_id == instance_id => { + let Ok(status) = GraphStatus::from_str(&graph.status) else { + warn!( + "Skip BABE setup cleanup for {graph_id}: invalid graph status {}", + graph.status + ); + continue; + }; + Some(status) + } + Some(graph) => { + warn!( + "BABE setup state {graph_id} belongs to {instance_id}, but the stored graph belongs to {}", + graph.instance_id + ); + None + } + None => None, + }; + if !should_cleanup_babe_setup_state(status, is_stale) { + continue; + } + if status.is_none() { + warn!("Cleaning stale orphan BABE setup state for {instance_id}:{graph_id}"); + } + let Some(state) = load_babe_setup_state_from_path(&path)? else { + continue; + }; + let mut payloads = Vec::new(); + if let Some(ready) = state.verifier.and_then(|verifier| verifier.soldering_proof_ready) + { + payloads.push(ready); + } + if let Some(operator) = state.operator { + payloads.extend( + operator + .candidates + .into_iter() + .filter_map(|candidate| candidate.soldering_proof_ready), + ); + } + for ready in payloads { + if ready.instance_id != instance_id || ready.graph_id != graph_id { + bail!("BABE setup payload reference does not match state path"); + } + let payload_path = soldering_proof_payload_store_path( + payload_store, + instance_id, + graph_id, + ready.candidate_index, + &ready.payload_hash, + )?; + delete_soldering_proof_store_payload(&payload_path).await?; + } + std::fs::remove_file(&path) + .with_context(|| format!("delete BABE setup state {}", path.display()))?; + deleted += 1; + } + } + Ok(deleted) +} + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct BabeSetupState { + pub verifier: Option, + pub operator: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct VerifierBabeSetupState { + pub verifier_pubkey: PublicKey, + /// Authenticated operator identity that initiated this setup. + pub operator_pubkey: PublicKey, + /// Signed operator P2P identity used for directed setup replies and ACKs. + pub operator_peer_id: Vec, + pub setup_package: CACSetupPackage, + pub private_state: BabeVerifierPrivateState, + pub finalized_indices: Vec, + pub soldering_proof_ready: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct OperatorVerifierCandidate { + pub verifier_peer_id: Vec, + pub verifier_pubkey: PublicKey, + pub setup_package: CACSetupPackage, + /// Immutable slot within the candidate pool and proof payload store. + pub candidate_index: Option, + pub selected_circuit_indexes: Vec, + pub gc_data: Option, + pub soldering_proof_ready: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct OperatorBabeSetupState { + /// Candidate pool closed before CutCircuits. This is not the final graph + /// verifier set; `selected_verifier_pubkeys` is set after proof validation. + pub candidate_verifier_pubkeys: Option>, + pub candidates: Vec, + #[serde(default)] + pub candidate_collection_started_at: Option, + #[serde(default)] + pub proof_collection_started_at: Option, + #[serde(default)] + pub selected_verifier_pubkeys: Option>, + #[serde(default)] + pub asserted_operator_proof: Option>, +} diff --git a/node/tla/InstanceLifecycle.cfg b/node/tla/InstanceLifecycle.cfg new file mode 100644 index 000000000..34c7b25d9 --- /dev/null +++ b/node/tla/InstanceLifecycle.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec +CONSTANTS + T0 = 4 + B = 2 + KD = 3 + MaxHeight = 12 +CHECK_DEADLOCK FALSE +INVARIANTS + TypeOK + NoSignatureAtOrAfterT0 + KeyDeletedOnlyWhenSafe + TimeoutKeepsKey + BroadcastedMeansConfirmed +PROPERTIES + FailureAbsorbing + NoTimeoutWhileVisible + MintedIsFinal diff --git a/node/tla/InstanceLifecycle.tla b/node/tla/InstanceLifecycle.tla new file mode 100644 index 000000000..850104e37 --- /dev/null +++ b/node/tla/InstanceLifecycle.tla @@ -0,0 +1,236 @@ +---- MODULE InstanceLifecycle ---- +(***************************************************************************) +(* Bridge-in lifecycle model for Instance aggregate decisions. *) +(* Models chain heights, committee answers, transaction visibility, *) +(* cancellation, signing eligibility and committee key deletion. *) +(* Failure states are terminal except Timeout -> UserCanceled. *) +(* Signing closes at the cancellation height; Timeout keeps the key. *) +(***************************************************************************) +EXTENDS Naturals, FiniteSets + +CONSTANTS T0, B, KD, MaxHeight + +Statuses == {"UserInited", "CommitteesAnswered", "UserBroadcastPeginPrepare", + "Presigned", "RelayerL1Broadcasted", "RelayerL2Minted", + "NoEnoughCommitteesAnswered", "UserDiscarded", "UserCanceled", + "Timeout"} +Failed == {"NoEnoughCommitteesAnswered", "UserDiscarded", "UserCanceled", "Timeout"} +DeletesKey == {"NoEnoughCommitteesAnswered", "UserDiscarded", "UserCanceled"} + +VARIABLES + status, \* the instance row + keyDeleted, \* instance.key_deleted + h, \* Bitcoin tip height + answered, \* "none" | "short" | "quorum": the Goat response window + prepareH, \* H_p, 0 while Peg-In-Prepare is unconfirmed + inputsSpent, \* an input of Peg-In-Prepare was spent elsewhere + endorsed, \* a graph of the instance is finalized and endorsed + signed, \* a committee member produced a Peg-In-Confirm signature + signedAt, \* heights at which signatures were produced + confirm, \* Peg-In-Confirm: "none" | "mempool" | "confirmed" + confirmH, \* confirmation height of Peg-In-Confirm + cancelConfirmed, \* Peg-In-Cancel confirmed + minted \* Goat BridgeIn emitted + +vars == <> +env == <> +node == <> + +TypeOK == + /\ status \in Statuses + /\ keyDeleted \in BOOLEAN + /\ h \in 0..MaxHeight + /\ answered \in {"none", "short", "quorum"} + /\ prepareH \in 0..MaxHeight + /\ inputsSpent \in BOOLEAN + /\ endorsed \in BOOLEAN + /\ signed \in BOOLEAN + /\ signedAt \subseteq 0..MaxHeight + /\ confirm \in {"none", "mempool", "confirmed"} + /\ confirmH \in 0..MaxHeight + /\ cancelConfirmed \in BOOLEAN + /\ minted \in BOOLEAN + +Init == + /\ status = "UserInited" + /\ keyDeleted = FALSE + /\ h = 0 + /\ answered = "none" + /\ prepareH = 0 + /\ inputsSpent = FALSE + /\ endorsed = FALSE + /\ signed = FALSE + /\ signedAt = {} + /\ confirm = "none" + /\ confirmH = 0 + /\ cancelConfirmed = FALSE + /\ minted = FALSE + +CancelFromHeight == prepareH + T0 +AbandonHeight == prepareH + T0 + B +SigningOpen == status \notin Failed /\ (prepareH = 0 \/ h < CancelFromHeight) + +-------------------------------------------------------------------------- +(* Environment *) + +Tick == + /\ h < MaxHeight + /\ h' = h + 1 + /\ UNCHANGED <> + +WindowCloses == + /\ answered = "none" + /\ answered' \in {"short", "quorum"} + /\ UNCHANGED <> + +\* The user can only build Peg-In-Prepare from a quorum of answers. +PrepareConfirms == + /\ answered = "quorum" /\ prepareH = 0 /\ ~inputsSpent /\ h > 0 + /\ prepareH' = h + /\ UNCHANGED <> + +InputSpentElsewhere == + /\ prepareH = 0 /\ ~inputsSpent + /\ inputsSpent' = TRUE + /\ UNCHANGED <> + +GraphEndorsed == + /\ prepareH > 0 /\ ~endorsed + /\ endorsed' = TRUE + /\ UNCHANGED <> + +\* A committee member signs Peg-In-Confirm: only in the Confirm phase and +\* only through the T0 gate (instance not failed, h < H_p + T0). +Sign == + /\ status = "Presigned" /\ SigningOpen /\ ~signed + /\ signed' = TRUE + /\ signedAt' = signedAt \cup {h} + /\ UNCHANGED <> + +\* Anyone holding the signed transaction may broadcast it at any time, +\* including after this node abandoned the instance. +ConfirmEntersMempool == + /\ signed /\ confirm = "none" /\ ~cancelConfirmed + /\ confirm' = "mempool" + /\ UNCHANGED <> + +ConfirmConfirms == + /\ confirm = "mempool" /\ ~cancelConfirmed + /\ confirm' = "confirmed" /\ confirmH' = h + /\ UNCHANGED <> + +\* BIP68: Peg-In-Cancel is minable from H_p + T0; it conflicts with an +\* unconfirmed Peg-In-Confirm, which is then evicted. +CancelConfirms == + /\ prepareH > 0 /\ h >= CancelFromHeight /\ confirm # "confirmed" /\ ~cancelConfirmed + /\ cancelConfirmed' = TRUE /\ confirm' = "none" + /\ UNCHANGED <> + +Mint == + /\ confirm = "confirmed" /\ ~minted + /\ minted' = TRUE + /\ UNCHANGED <> + +Environment == + \/ Tick \/ WindowCloses \/ PrepareConfirms \/ InputSpentElsewhere + \/ GraphEndorsed \/ Sign \/ ConfirmEntersMempool \/ ConfirmConfirms + \/ CancelConfirms \/ Mint + +-------------------------------------------------------------------------- +(* The Instance aggregate *) + +\* decide_answer_window +ScanAnswers == + /\ status = "UserInited" /\ answered # "none" + /\ status' = IF answered = "quorum" THEN "CommitteesAnswered" + ELSE "NoEnoughCommitteesAnswered" + /\ keyDeleted' = (keyDeleted \/ answered = "short") + /\ UNCHANGED env + +\* decide_btc_snapshot on the facts as they are now +ScanBitcoin == + /\ status \notin {"UserCanceled", "UserDiscarded", "NoEnoughCommitteesAnswered"} + /\ LET s1 == IF status = "CommitteesAnswered" + THEN IF prepareH > 0 THEN "UserBroadcastPeginPrepare" + ELSE IF inputsSpent THEN "UserDiscarded" + ELSE status + ELSE status + s2 == IF s1 \in {"UserBroadcastPeginPrepare", "Presigned"} + THEN IF confirm = "confirmed" THEN "RelayerL1Broadcasted" + ELSE IF cancelConfirmed THEN "UserCanceled" + ELSE IF prepareH > 0 /\ h >= AbandonHeight /\ confirm = "none" + THEN "Timeout" + ELSE s1 + ELSE IF s1 = "Timeout" /\ cancelConfirmed THEN "UserCanceled" + ELSE s1 + failureDelete == s2 \in DeletesKey /\ s2 # status + successDelete == s2 \in {"RelayerL1Broadcasted", "RelayerL2Minted"} + /\ confirm = "confirmed" /\ confirmH + KD <= h + IN /\ status' = s2 + /\ keyDeleted' = (keyDeleted \/ failureDelete \/ successDelete) + /\ UNCHANGED env + +\* decide_graph_endorsed +ScanEndorsed == + /\ status = "UserBroadcastPeginPrepare" /\ endorsed + /\ status' = "Presigned" + /\ UNCHANGED <> /\ UNCHANGED env + +\* decide_l2_minted: success from any live state, only an alert for failed instances +ScanMint == + /\ minted /\ status \notin Failed /\ status # "RelayerL2Minted" + /\ status' = "RelayerL2Minted" + /\ UNCHANGED <> /\ UNCHANGED env + +Node == ScanAnswers \/ ScanBitcoin \/ ScanEndorsed \/ ScanMint + +Next == Environment \/ Node +Spec == Init /\ [][Next]_vars + +-------------------------------------------------------------------------- +(* Properties *) + +\* The T0 gate: no protocol signature at or after H_p + T0. +NoSignatureAtOrAfterT0 == + \A x \in signedAt : prepareH > 0 /\ x < CancelFromHeight + +\* The key is deleted only in the three failure states that delete it, or +\* on the success side once the Peg-In-Confirm is KD blocks deep. +KeyDeletedOnlyWhenSafe == + keyDeleted => + \/ status \in DeletesKey + \/ (status \in {"RelayerL1Broadcasted", "RelayerL2Minted"} + /\ confirm = "confirmed" /\ confirmH + KD <= h) + +TimeoutKeepsKey == status = "Timeout" => ~keyDeleted + +BroadcastedMeansConfirmed == + status \in {"RelayerL1Broadcasted", "RelayerL2Minted"} => confirm = "confirmed" + +\* Failure states are terminal except Timeout -> UserCanceled. +FailureAbsorbing == + [][(status \in Failed => + status' \in (IF status = "Timeout" THEN {"Timeout", "UserCanceled"} ELSE {status}))]_vars + +\* Timeout is only entered past the abandon height while the Peg-In-Confirm +\* is neither confirmed nor in the mempool. +NoTimeoutWhileVisible == + [][((status' = "Timeout" /\ status # "Timeout") => + (prepareH > 0 /\ h >= AbandonHeight /\ confirm = "none"))]_vars + +MintedIsFinal == [][(status = "RelayerL2Minted" => status' = "RelayerL2Minted")]_vars + +==== diff --git a/node/tla/InstancePresigned.tla b/node/tla/InstancePresigned.tla index b984c0453..5b6a753fe 100644 --- a/node/tla/InstancePresigned.tla +++ b/node/tla/InstancePresigned.tla @@ -13,7 +13,7 @@ (* `update_graph_status_guarded` (node/src/utils.rs, ~5195) both write *) (* `InstanceBridgeInStatus::Presigned` unconditionally as a side effect of *) (* a graph reaching `CommitteePresigned`, reachable from independent P2P- *) -(* message handlers (handle.rs) and chain-rescans (graph_maintenance_tasks) *) +(* message handlers (handle.rs) and chain-rescans (graph_scan) *) (* with no coordination - so a stale/replayed event could revert an *) (* instance that already progressed past Presigned (e.g. to *) (* RelayerL1Broadcasted or RelayerL2Minted, written by instance_btc_tx_monitor *) diff --git a/scripts/ops/README.md b/scripts/ops/README.md new file mode 100644 index 000000000..045b3f897 --- /dev/null +++ b/scripts/ops/README.md @@ -0,0 +1,64 @@ +# Node ops scripts + +Three read-only helpers for one running `bitvm-noded`. They need `bash`, `jq`, `curl`; `trace.sh` also needs +`sqlite3`. Defaults come from the environment, flags override them: + +| Variable | Meaning | Default | +|---|---|---| +| `NODE_RPC` | the node's RPC base URL (`--rpc-addr`) | `http://127.0.0.1:8080` | +| `NODE_METRICS` | the metrics listener base URL (`--metrics-addr`) | `http://127.0.0.1:9090` | +| `NODE_LOG` | the node's JSON log file (stdout of `bitvm-noded`) | none | +| `NODE_DB` | the node's SQLite file (the `--db-path` without `sqlite:`) | none | +| `NODE_PID` | the node's pid, for process stats in `triage.sh` | none | + +## `triage.sh` — one screen: is this node healthy, and if not, where + +``` +scripts/ops/triage.sh [-r RPC] [-m METRICS] [-f LOG] [-p PID] [-s 10m] +``` + +Prints `GET /v1/health` (readiness bits, each backend's availability with its last error, event watcher state), +the key gauges from `/metrics` (peers, lags, balances, backend probe counters, tasks whose last success is older +than 90 s, alert counters), `/v1/debug/status` and the parked / poisoned engine rows when the node was built with +`rpc-debug-endpoints`, and the WARN/ERROR lines of the last window grouped by event and outcome when a log file is +given. First thing to run when a node looks stuck. + +## `logq.sh` — query the JSON log + +``` +scripts/ops/logq.sh [-f LOG] [-s 10m|2h|1d|2026-10-15T08:00:00] [-l INFO|WARN|ERROR] [-e EVENT] [-g SUBSTRING] [-n N] [-j] +``` + +One compact line per record: `time level role event-or-target outcome message extra=fields`. `-l WARN` includes +ERROR; `-e` matches the structured `event` field exactly (`maintenance_tick_result`, `engine_jobs`, +`backend_availability`, ...); `-g` is a plain substring filter applied before parsing, so an instance id, graph id +or txid is cheap even on a large file; `-j` prints the raw JSON lines. Reads stdin when no file is given, so it +composes with `journalctl -o cat` or `docker logs`. + +Useful queries: + +``` +logq.sh -f node.log -s 30m -l WARN # everything that went wrong in the last half hour +logq.sh -f node.log -s 1h -e backend_availability # when did a backend go down / come back +logq.sh -f node.log -s 2h -e engine_jobs -l WARN # job attempts that failed on an error path +logq.sh -f node.log -s 1d -g 475a4574 # every line naming an instance / graph / tx +``` + +## `trace.sh` — timeline of one peg-in instance or graph + +``` +scripts/ops/trace.sh [-r RPC] [-d DB] [-f LOG] [-s 6h] [-n 300] +``` + +Resolves the related ids (the graphs of an instance, the instance of a graph), prints the RPC status, the +database rows that drive the flow (instance, graphs, `pending_graph_init`, the engine aggregate, every job with its +state / attempts / lease / last error, parked inbox events, poison rows) and the log lines naming any of the ids, +oldest first. Give it the same node's database and log; for several nodes run it once per node and compare the +`graph` and `job` lines. + +## Notes + +- All three only read: no RPC that changes state, the database is opened read-only. +- The log format is the node's JSON output (`tracing_subscriber` JSON with top-level `event` / `outcome` fields). +- `/v1/health` returns 503 while the node is not ready; `triage.sh` still prints its body. +- `logq.sh` skips lines that are not JSON (panic messages, stray stderr output) instead of stopping there. diff --git a/scripts/ops/logq.sh b/scripts/ops/logq.sh new file mode 100755 index 000000000..d5725df89 --- /dev/null +++ b/scripts/ops/logq.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Query a bitvm-noded JSON log (one node): time window, level, event, substring; compact one-line output. +# logq.sh [-f ] [-s 10m|2h|1d|] [-l INFO|WARN|ERROR] [-e ] [-g ] [-n N] [-j] +# -f log file (default: $NODE_LOG, else stdin) -s since (default 10m; WARN includes ERROR) +# -e exact `event` field -g substring anywhere in the line (ids, txids, ...) +# -n keep only the last N lines -j raw JSON lines instead of the compact form +# Output columns: time level role event-or-target outcome message extra-fields. +set -uo pipefail +file=${NODE_LOG:-} since=10m level="" event="" grep_="" n="" raw=0 +while getopts 'f:s:l:e:g:n:j' o; do case $o in f) file=$OPTARG;; s) since=$OPTARG;; l) level=${OPTARG^^};; e) event=$OPTARG;; g) grep_=$OPTARG;; n) n=$OPTARG;; j) raw=1;; *) exit 2;; esac; done +case $since in + *s) SINCE=$(date -u -d "-${since%s} sec" +%Y-%m-%dT%H:%M:%S) ;; *m) SINCE=$(date -u -d "-${since%m} min" +%Y-%m-%dT%H:%M:%S) ;; + *h) SINCE=$(date -u -d "-${since%h} hour" +%Y-%m-%dT%H:%M:%S) ;; *d) SINCE=$(date -u -d "-${since%d} day" +%Y-%m-%dT%H:%M:%S) ;; + *) SINCE=$since ;; +esac +src() { if [[ -n $file ]]; then cat -- "$file"; else cat; fi; } +{ if [[ -n $grep_ ]]; then src | grep -F -- "$grep_"; else src; fi; } \ +| jq -R -r --arg since "$SINCE" --arg lv "$level" --arg e "$event" --argjson raw "$raw" ' + (try fromjson catch empty) + | select(type=="object" and (.timestamp // "") >= $since) + | select($lv=="" or ($lv=="WARN" and (.level=="WARN" or .level=="ERROR")) or .level==$lv) + | select($e=="" or .event==$e) + | if $raw==1 then tostring else + [ .timestamp, .level, (.span.role // "-"), (.event // ((.target // "?")|split("::")|last)), (.outcome // "-"), (.message // ""), + ( to_entries | map(select(.key|IN("timestamp","level","message","target","span","spans","event","outcome")|not)) + | map("\(.key)=\(.value|tostring|.[0:80])") | join(" ") ) ] | @tsv end' 2>/dev/null \ +| { if [[ -n $n ]]; then tail -n "$n"; else cat; fi; } \ +| { if (( raw )); then cat; else awk -F'\t' '{printf "%s %-5s %-10s %-28s %-12s %s %s\n", substr($1,1,23), $2, $3, substr($4,1,28), $5, $6, $7}'; fi; } diff --git a/scripts/ops/trace.sh b/scripts/ops/trace.sh new file mode 100755 index 000000000..2c01061d7 --- /dev/null +++ b/scripts/ops/trace.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Timeline of one peg-in instance or one graph on a single bitvm-noded: RPC status, database rows +# (instance, graphs, engine aggregate / jobs / parked inbox events / poison) and the log lines naming the id. +# trace.sh [-r http://host:rpc] [-d ] [-f ] [-s 6h] [-n 300] +# Defaults: NODE_RPC (http://127.0.0.1:8080), NODE_DB (sqlite file path, e.g. /tmp/bitvm-node.db), NODE_LOG. +set -uo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +id=${1:?instance or graph id}; shift +rpc=${NODE_RPC:-http://127.0.0.1:8080} db=${NODE_DB:-} log=${NODE_LOG:-} since=6h n=300 +while getopts 'r:d:f:s:n:' o; do case $o in r) rpc=$OPTARG;; d) db=$OPTARG;; f) log=$OPTARG;; s) since=$OPTARG;; n) n=$OPTARG;; *) exit 2;; esac; done +q() { sqlite3 -readonly -noheader -list -cmd ".timeout 8000" "$db" "$1" 2>/dev/null | tr '|' ' '; } +# uuid text for a column stored either as TEXT or as a 16-byte BLOB +uf() { echo "CASE WHEN typeof($1)='blob' THEN lower(substr(hex($1),1,8)||'-'||substr(hex($1),9,4)||'-'||substr(hex($1),13,4)||'-'||substr(hex($1),17,4)||'-'||substr(hex($1),21)) ELSE $1 END"; } +hexid=$(tr -d - <<<"$id" | tr a-f A-F) +echo "### trace $id ($(date -u +%FT%TZ))" +related="" +if [[ -n $db && -r $db ]]; then + related=$(q "select $(uf graph_id) from graph where instance_id='$id' or hex(instance_id)='$hexid'; select $(uf instance_id) from graph where graph_id='$id' or hex(graph_id)='$hexid';" | sort -u | grep -av '^$') +fi +ids="$id"; [[ -n $related ]] && ids+=$'\n'"$related" +echo "related ids: $(tr '\n' ' ' <<<"$related")" +echo "### RPC" +is=$(curl -s --max-time 5 "$rpc/v1/instances/$id" | jq -r '.instance_wrap.instance.status // "-"' 2>/dev/null) +gs=$(curl -s --max-time 5 "$rpc/v1/graphs/$id" | jq -r '.graph | "\(.status // "-")/\(.sub_status // "")"' 2>/dev/null) +echo " instance=${is:-?} graph=${gs:-?}" +if [[ -n $db && -r $db ]]; then + echo "### database $db" + q "select ' instance', status, 'btc_h='||btc_height, 'status_updated_at='||status_updated_at, 'goat_tx='||substr(goat_tx_hash,1,10) from instance where instance_id='$id' or hex(instance_id)='$hexid';" + for g in $ids; do gh=$(tr -d - <<<"$g" | tr a-f A-F) + q "select ' graph', $(uf graph_id), 'slot='||kickoff_index, status, sub_status, 'status_updated_at='||status_updated_at, 'kickoff='||substr(kickoff_txid,1,10) from graph where graph_id='$g' or hex(graph_id)='$gh';" + done + q "select ' pending_graph_init', $(uf graph_id), operator_pubkey, updated_at from pending_graph_init where hex(instance_id)='$hexid' or instance_id='$id';" + for g in $ids; do gh=$(tr -d - <<<"$g" | tr a-f A-F) + q "select ' aggregate', aggregate_kind, status, phase, 'v='||version, 'updated='||updated_at from engine_aggregate where aggregate_id='$g';" + q "select ' job', aggregate_kind, phase||'/'||action, state, 'attempts='||attempts, 'next_at='||next_at, 'lease='||lease_until, 'cancel='||cancel_requested, substr(coalesce(last_error,''),1,70), 'updated='||updated_at from engine_job where aggregate_id='$g' order by updated_at;" + q "select ' inbox!', kind, state, 'attempts='||attempts, coalesce(waiting_ref,''), substr(coalesce(waiting_detail,last_error,''),1,70) from engine_inbox where aggregate_id='$g' and state!='Applied';" + q "select ' POISON', action, 'count='||poison_count, 'blocked_until='||blocked_until, substr(reason,1,70) from engine_aggregate_poison where aggregate_id='$g';" + done +else echo "### database: not given (-d) or unreadable; skipped"; fi +if [[ -n $log && -r $log ]]; then + echo "### log timeline (last $n lines, since $since)" + for g in $ids; do "$HERE/logq.sh" -f "$log" -s "$since" -g "${g:0:8}"; done | grep -av ' http_re[sq]' | sort -u | tail -n "$n" +else echo "### log: not given (-f) or unreadable; skipped"; fi diff --git a/scripts/ops/triage.sh b/scripts/ops/triage.sh new file mode 100755 index 000000000..190e83bd1 --- /dev/null +++ b/scripts/ops/triage.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# One-screen "why is this node unhealthy" view of a single bitvm-noded. +# triage.sh [-r http://host:rpc] [-m http://host:metrics] [-f ] [-p ] [-s 10m] +# Defaults come from NODE_RPC (http://127.0.0.1:8080), NODE_METRICS (http://127.0.0.1:9090), NODE_LOG, NODE_PID. +# Reads GET /v1/health, GET /metrics, GET /v1/debug/status + /v1/debug/engine when the debug feature is on, +# and the log's WARN/ERROR lines of the last window grouped by event/outcome. +set -uo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +rpc=${NODE_RPC:-http://127.0.0.1:8080} met=${NODE_METRICS:-http://127.0.0.1:9090} log=${NODE_LOG:-} pid=${NODE_PID:-} since=10m +while getopts 'r:m:f:p:s:' o; do case $o in r) rpc=$OPTARG;; m) met=$OPTARG;; f) log=$OPTARG;; p) pid=$OPTARG;; s) since=$OPTARG;; *) exit 2;; esac; done +now=$(date +%s) +metric() { awk -v k="$1" '$1==k{print $2; exit}' <<<"$2"; } +age() { local t=$1; [[ -z $t || $t == 0 ]] && echo "never" || echo "$(( now - t ))s"; } + +if [[ -n $pid && -d /proc/$pid ]]; then + echo "process: pid=$pid up=$(ps -o etimes= -p "$pid" | tr -d ' ')s rss=$(awk '/VmRSS/{printf "%dMB",$2/1024}' "/proc/$pid/status") fds=$(ls "/proc/$pid/fd" 2>/dev/null | wc -l)" +fi +h=$(curl -s --max-time 5 "$rpc/v1/health") +if [[ -n $h ]]; then + jq -r '"health: ready=\(.ready) startup=\(.components.startup) database=\(.components.database) backend=\(.components.backend) event_watcher=\(.components.event_watcher) role=\(.role) version=\(.version) uptime=\(.uptime_secs)s spv_lag=\(.spv_lag_blocks)", + (.backends[] | " backend \(.backend): up=\(.up)\(if .unavailable_since then " unavailable_since=\(.unavailable_since)" else "" end) consecutive_failures=\(.consecutive_failures)\(if .last_error then " last_error=\(.last_error[0:100])" else "" end)"), + " event_watch: state=\(.event_watch.state) lag_blocks=\(.event_watch.lag_blocks) last_success=\(.event_watch.last_success_timestamp)"' <<<"$h" 2>/dev/null || echo "health: unparseable: ${h:0:200}" +else echo "health: $rpc/v1/health UNREACHABLE"; fi +m=$(curl -s --max-time 5 "$met/metrics") +if [[ -n $m ]]; then + echo "metrics: ready=$(metric bitvm_node_ready "$m") p2p_peers=$(metric bitvm_node_p2p_connected_peers "$m") topics_healthy=$(metric bitvm_node_p2p_required_topics_healthy "$m") event_watch_lag=$(metric bitvm_node_event_watch_lag_blocks "$m") spv_lag=$(metric bitvm_node_spv_lag_blocks "$m") gas_wei=$(metric bitvm_node_goat_gas_balance_wei "$m") fee_sats=$(metric bitvm_node_fee_wallet_balance_sats "$m")" + echo " backend probes: $(grep -E '^bitvm_node_(btc|goat)_backend_requests_total' <<<"$m" | sed -E 's/bitvm_node_(btc|goat)_backend_requests_total\{outcome="([a-z]+)"\} ([0-9]+)/\1.\2=\3/' | tr '\n' ' ') btc_last_ok=$(age "$(metric bitvm_node_btc_backend_last_success_timestamp_seconds "$m")") goat_last_ok=$(age "$(metric bitvm_node_goat_backend_last_success_timestamp_seconds "$m")")" + echo " backend_up: $(grep '^bitvm_node_backend_up' <<<"$m" | sed -E 's/.*backend="([a-z]+)"\} ([0-9]+)/\1=\2/' | tr '\n' ' ')" + stale=$(grep '^bitvm_node_task_last_success_timestamp_seconds' <<<"$m" | awk -v now="$now" -F'[{}" ]+' '{a=now-$NF; if (a>90) printf "%s=%ds ", $3, a}') + echo " stale tasks (>90s): ${stale:-none}" + al=$(grep '^bitvm_node_instance_alerts_total' <<<"$m" | sed -E 's/.*alert="([a-z_]+)"\} ([0-9]+)/\1=\2/' | tr '\n' ' '); [[ -n $al ]] && echo " alerts: $al" +else echo "metrics: $met/metrics UNREACHABLE"; fi +d=$(curl -s --max-time 6 "$rpc/v1/debug/status") +if [[ -n $d ]] && jq -e .bitcoin_height >/dev/null 2>&1 <<<"$d"; then + echo "debug status: $(jq -r '"btc=\(.bitcoin_height // "null") goat=\(.goat_height // "null") spv=\(.goat_spv_height // "null")"' <<<"$d") engine=$(jq -c '.engine // {}' <<<"$d" | cut -c1-220)" + e=$(curl -s --max-time 6 "$rpc/v1/debug/engine") + [[ -n $e ]] && jq -r '(.waiting[]? | " WAITING \(.aggregate_kind) \(.aggregate_id[0:8]) \(.kind) \(.waiting_ref // "") \((.waiting_detail // "")[0:80])"), (.poisoned[]? | " POISONED \(.aggregate_kind) \(.aggregate_id[0:8]) \(.action) count=\(.poison_count) blocked_until=\(.blocked_until) \((.reason // "")[0:80])")' <<<"$e" 2>/dev/null +fi +if [[ -n $log && -r $log ]]; then + echo "log $log, WARN/ERROR last $since (top 10 by level/event/outcome):" + "$HERE/logq.sh" -f "$log" -s "$since" -l WARN | awk '{printf "%s %s %s\n", $2, $4, $5}' | sort | uniq -c | sort -rn | head -10 | sed 's/^/ /' + le=$("$HERE/logq.sh" -f "$log" -s "$since" -l ERROR -n 1); [[ -n $le ]] && echo " last ERROR: $le" +fi +exit 0