From 993c9a5130a2a2fec7e774362377e5b8fad4d768 Mon Sep 17 00:00:00 2001 From: Parithosh Jayanthi Date: Mon, 13 Jul 2026 15:47:49 +0200 Subject: [PATCH 1/2] feat(playbooks): add single-node blackout & recovery playbook Fully isolates one participant's beacon node via the disruptoor HTTP API (p2p + control traffic to all other participants and to its own EL/VC), holds the blackout for a configurable number of slots, heals, then verifies re-sync and fresh finality. Complements two-way-network-split-reorg-trigger: instead of a two-half split this reproduces total single-node network loss, e.g. for testing client stall/recovery behaviour after a fork. Co-Authored-By: Claude Fable 5 --- playbooks/_index.yaml | 73 ++++++- .../dev/single-node-blackout-recovery.yaml | 193 ++++++++++++++++++ 2 files changed, 264 insertions(+), 2 deletions(-) create mode 100644 playbooks/dev/single-node-blackout-recovery.yaml diff --git a/playbooks/_index.yaml b/playbooks/_index.yaml index 6d74d84f..d59a67d2 100644 --- a/playbooks/_index.yaml +++ b/playbooks/_index.yaml @@ -1,8 +1,8 @@ # Auto-generated playbook index -# Generated: 2026-07-07T07:50:04Z +# Generated: 2026-07-13T13:47:28Z # DO NOT EDIT MANUALLY - regenerate via `make generate-playbook-index`. -generated: 2026-07-07T07:50:04Z +generated: 2026-07-13T13:47:28Z folders: - path: api-compatibility name: API Compatibility @@ -219,6 +219,38 @@ playbooks: tags: - utility timeout: 4h + - file: dev/single-node-blackout-recovery.yaml + id: single-node-blackout-recovery + name: Single-Node Blackout & Recovery + description: |- + Fully isolates one participant's beacon node from the rest of a + Kurtosis-launched devnet through the disruptoor HTTP API, keeps the + blackout active for a configurable number of slots, heals it, then + verifies the node re-syncs and the network regains fresh finality. + + "Fully" means the target beacon node loses ALL traffic: p2p and control + traffic to every other participant AND to its own execution and validator + client (scope `[cl_p2p, el_p2p, include_control]` on both partitions). + This reproduces client behaviour under sudden total network loss — + e.g. a silent stall followed by a missed proposal during catch-up. + + The blackout is gated on `minStartEpoch` so fork-activation devnets can + schedule it safely after the fork (set it to fork epoch + 2 or later). + The target participant is selected by its 1-based `node-index` label via + `targetParticipantIndex`; the counterparty group is computed dynamically + from the assertoor client pool. Choose a topology where the remaining + nodes retain a 2/3 finality majority (e.g. 4 nodes, blackout 1). A + disruptoor service is expected at `disruptoorApiUrl` (default + `http://disruptoor:7700`). + version: 1.0.0 + tags: + - disruptoor + - kurtosis + - blackout + - node-isolation + - recovery + - consensus + timeout: 120m - file: dev/synchronized-check.yaml id: synchronized-check name: Check client pairs are in sync @@ -655,6 +687,43 @@ playbooks: - queue - eip-8282 timeout: 1h + - file: gloas-dev/deploy-eip8282-contracts.yaml + id: deploy-eip8282-contracts + name: 'GLOAS: Deploy EIP-8282 builder request contracts (pre-fork)' + description: |- + Deploys the EIP-8282 builder deposit (request type 0x03) and builder exit + (request type 0x04) system contracts onto a running devnet BEFORE the GLOAS + fork activates. + + glamsterdam-devnet-7 deliberately omits these predeploys from genesis + (DEPLOY_EIP8282_CONTRACTS=false in the genesis generator) to exercise the + mainnet-like deployment workflow: from GLOAS activation onward every block + is INVALID unless both contracts exist, so they must be deployed as regular + transactions pre-fork. Intended to be registered as a startup test so the + contracts land right after the devnet comes up. + + Deployment goes through the deterministic CREATE2 factory (predeployed in + devnet genesis as a well-known contract) using the vanity-mined salts from + ethereum/execution-specs#3091 — CREATE2(factory, salt, initcode) yields the + canonical ...8282 addresses the clients expect. The factory REVERTS when + CREATE2 fails (including the already-deployed collision case), so a + successful receipt is proof of deployment. + + Idempotent: an eth_call precheck against the builder deposit contract's fee + getter detects existing code and skips deployment, so restarts of a + startup-scheduled run stay green. The exit contract cannot be prechecked the + same way — its constructor arms the slot-0 excess inhibitor, which makes its + fee getter revert by design until the first post-GLOAS system call — so its + deployment is asserted via the deploy receipt (both contracts only ever + deploy as a pair). + version: 1.0.0 + tags: + - gloas + - builder + - eip-8282 + - deployment + - prefork + timeout: 30m - file: gloas-dev/exit-builders.yaml id: exit-builders name: 'GLOAS: Exit builders deposited by prefork-queue-fill-public' diff --git a/playbooks/dev/single-node-blackout-recovery.yaml b/playbooks/dev/single-node-blackout-recovery.yaml new file mode 100644 index 00000000..438b6cb8 --- /dev/null +++ b/playbooks/dev/single-node-blackout-recovery.yaml @@ -0,0 +1,193 @@ +id: single-node-blackout-recovery +name: "Single-Node Blackout & Recovery" +description: | + Fully isolates one participant's beacon node from the rest of a + Kurtosis-launched devnet through the disruptoor HTTP API, keeps the + blackout active for a configurable number of slots, heals it, then + verifies the node re-syncs and the network regains fresh finality. + + "Fully" means the target beacon node loses ALL traffic: p2p and control + traffic to every other participant AND to its own execution and validator + client (scope `[cl_p2p, el_p2p, include_control]` on both partitions). + This reproduces client behaviour under sudden total network loss — + e.g. a silent stall followed by a missed proposal during catch-up. + + The blackout is gated on `minStartEpoch` so fork-activation devnets can + schedule it safely after the fork (set it to fork epoch + 2 or later). + The target participant is selected by its 1-based `node-index` label via + `targetParticipantIndex`; the counterparty group is computed dynamically + from the assertoor client pool. Choose a topology where the remaining + nodes retain a 2/3 finality majority (e.g. 4 nodes, blackout 1). A + disruptoor service is expected at `disruptoorApiUrl` (default + `http://disruptoor:7700`). +version: 1.0.0 +tags: [disruptoor, kurtosis, blackout, node-isolation, recovery, consensus] +timeout: 120m +config: + disruptoorApiUrl: "http://disruptoor:7700" # Disruptoor HTTP API used to apply and clear the blackout. + targetParticipantIndex: 1 # 1-based participant index (node-index label) whose beacon node gets blacked out. + blackoutDurationSlots: 5 # Number of slots to keep the blackout active (5 slots = 30s on the 6s-slot minimal preset). + minStartEpoch: 3 # Do not start the blackout before this epoch (set to fork epoch + 2 or later on fork-activation devnets). + minimumClientCount: 2 # Minimum number of healthy clients required before starting the blackout. + minFinalizedEpochIncreaseAfterRecovery: 1 # Required finalized epoch increase after healing to prove fresh finality. + maxUnfinalizedEpochsAfterRecovery: 6 # Maximum finality lag allowed after healing. +tasks: +- name: sleep + title: "Wait for disruptoor API to come up" + config: + duration: 10s + +- name: run_shell + title: "Check disruptoor API health" + timeout: 1m + config: + envVars: + DISRUPTOOR_URL: disruptoorApiUrl + command: | + set -euo pipefail + disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .) + curl -fsS "${disruptoor_url}/v1/healthz" >/dev/null + +- name: check_clients_are_healthy + id: client_check + title: "Wait for all devnet clients to be healthy" + timeout: 20m + configVars: + minClientCount: "minimumClientCount" + config: + maxUnhealthyCount: 0 + +- name: check_consensus_finality + id: initial_finality + title: "Wait for initial finality" + timeout: 40m + config: + minFinalizedEpochs: 2 + maxUnfinalizedEpochs: 3 + +- name: check_consensus_slot_range + title: "Wait for the configured blackout start epoch" + timeout: 30m + configVars: + minEpochNumber: "minStartEpoch" + +- name: check_consensus_slot_range + id: blackout_start + title: "Capture blackout start slot" + timeout: 1m + config: {} + +- name: run_shell + title: "Black out the target beacon node" + timeout: 1m + config: + envVars: + DISRUPTOOR_URL: disruptoorApiUrl + TARGET_INDEX: targetParticipantIndex + OTHER_PARTICIPANTS: "| [range(1; .tasks.client_check.outputs.totalCount + 1)] | map(select(. != (.targetParticipantIndex | tonumber)))" + command: | + set -euo pipefail + disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .) + target_index=$(echo "$TARGET_INDEX" | jq -r .) + other_participants=$(echo "$OTHER_PARTICIPANTS" | jq -c .) + other_count=$(echo "$other_participants" | jq 'length') + echo "Blacking out participant ${target_index} (beacon) vs ${other_count} other participants (${other_participants})" + if [ "$other_count" -lt 1 ]; then + echo "Refusing to black out: no other participants to partition against" >&2 + exit 1 + fi + state_file=$(mktemp) + trap 'rm -f "$state_file"' EXIT + + # Two partitions that only ever pit the target beacon node against + # others (disruptoor rejects overlapping groups, so a single + # "target vs all" partition is not expressible): + # 1. target beacon node vs every other participant (all components) + # 2. target beacon node vs its own execution + validator client + # include_control extends the cut beyond p2p to RPC/engine/API traffic. + jq -n \ + --argjson target "$target_index" \ + --argjson others "$other_participants" \ + '{ + partitions: [ + { + name: "assertoor-blackout-cl-vs-network", + groups: [ + {"node-index": [$target], "client-type": ["beacon"]}, + {"node-index": $others} + ], + scope: ["cl_p2p", "el_p2p", "include_control"], + symmetric: true + }, + { + name: "assertoor-blackout-cl-vs-own-stack", + groups: [ + {"node-index": [$target], "client-type": ["beacon"]}, + {"node-index": [$target], "client-type": ["execution", "validator"]} + ], + scope: ["cl_p2p", "el_p2p", "include_control"], + symmetric: true + } + ] + }' >"$state_file" + + curl -fsS -X PUT "${disruptoor_url}/v1/state" \ + -H "Content-Type: application/json" \ + --data @"$state_file" + curl -fsS "${disruptoor_url}/v1/state" | jq -e '.partitions | length == 2' >/dev/null + +- name: check_consensus_slot_range + title: "Keep the blackout active for the configured duration" + timeout: 15m + configVars: + minSlotNumber: "| (.tasks.blackout_start.outputs.currentSlot | tonumber) + (.blackoutDurationSlots | tonumber)" + +- name: run_shell + title: "Heal the blackout" + timeout: 1m + config: + envVars: + DISRUPTOOR_URL: disruptoorApiUrl + command: | + set -euo pipefail + disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .) + curl -fsS -X POST "${disruptoor_url}/v1/state/clear" + curl -fsS "${disruptoor_url}/v1/state" | jq -e '(.partitions | length) == 0 and (.shaping | length) == 0' >/dev/null + +- name: check_clients_are_healthy + title: "Wait for all clients to report healthy after healing" + timeout: 10m + configVars: + minClientCount: "minimumClientCount" + config: + maxUnhealthyCount: 0 + +- name: check_consensus_sync_status + title: "Wait for all clients to report synced after healing" + timeout: 10m + config: + expectSyncing: false + expectMinPercent: 100 + +- name: check_consensus_finality + title: "Poll for fresh finality after recovery" + timeout: 25m + configVars: + # Require a fresh finalized checkpoint after the blackout. maxUnfinalizedEpochs + # alone can pass on stale pre-blackout finality when the current epoch is close. + minFinalizedEpochs: "| (.tasks.initial_finality.outputs.finalizedEpoch | tonumber) + (.minFinalizedEpochIncreaseAfterRecovery | tonumber)" + maxUnfinalizedEpochs: "maxUnfinalizedEpochsAfterRecovery" + +cleanupTasks: +- name: run_shell + title: "Clear disruptoor state" + timeout: 1m + config: + envVars: + DISRUPTOOR_URL: disruptoorApiUrl + command: | + set -euo pipefail + disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .) + if ! curl -fsS -X POST "${disruptoor_url}/v1/state/clear"; then + echo "WARNING: failed to clear disruptoor state during cleanup; the target node may still be blacked out" >&2 + fi From f12e4061a592768c35ed670a31bbc14cbe0355f9 Mon Sep 17 00:00:00 2001 From: Parithosh Jayanthi Date: Mon, 13 Jul 2026 16:21:57 +0200 Subject: [PATCH 2/2] fix(playbooks): bind target index before map() in blackout participant expression Inside map(select(...)) jq rebinds . to the array element, so .targetParticipantIndex resolved against a number and errored (assertoor marshals gojq errors to {}). Bind it to $target at the root first. Co-Authored-By: Claude Fable 5 --- playbooks/dev/single-node-blackout-recovery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/dev/single-node-blackout-recovery.yaml b/playbooks/dev/single-node-blackout-recovery.yaml index 438b6cb8..9360a918 100644 --- a/playbooks/dev/single-node-blackout-recovery.yaml +++ b/playbooks/dev/single-node-blackout-recovery.yaml @@ -84,7 +84,7 @@ tasks: envVars: DISRUPTOOR_URL: disruptoorApiUrl TARGET_INDEX: targetParticipantIndex - OTHER_PARTICIPANTS: "| [range(1; .tasks.client_check.outputs.totalCount + 1)] | map(select(. != (.targetParticipantIndex | tonumber)))" + OTHER_PARTICIPANTS: "| (.targetParticipantIndex | tonumber) as $target | [range(1; .tasks.client_check.outputs.totalCount + 1)] | map(select(. != $target))" command: | set -euo pipefail disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .)