feat(playbooks): add single-node blackout & recovery playbook

playbooks/_index.yaml

@@ -1,8 +1,8 @@
 # Auto-generated playbook index
-# Generated: 2026-07-07T07:50:04Z
+# Generated: 2026-07-13T13:47:28Z
 # DO NOT EDIT MANUALLY - regenerate via `make generate-playbook-index`.
 
-generated: 2026-07-07T07:50:04Z
+generated: 2026-07-13T13:47:28Z
 folders:
   - path: api-compatibility
     name: API Compatibility
@@ -219,6 +219,38 @@ playbooks:
     tags:
       - utility
     timeout: 4h
+  - file: dev/single-node-blackout-recovery.yaml
+    id: single-node-blackout-recovery
+    name: Single-Node Blackout & Recovery
+    description: |-
+      Fully isolates one participant's beacon node from the rest of a
+      Kurtosis-launched devnet through the disruptoor HTTP API, keeps the
+      blackout active for a configurable number of slots, heals it, then
+      verifies the node re-syncs and the network regains fresh finality.
+
+      "Fully" means the target beacon node loses ALL traffic: p2p and control
+      traffic to every other participant AND to its own execution and validator
+      client (scope `[cl_p2p, el_p2p, include_control]` on both partitions).
+      This reproduces client behaviour under sudden total network loss —
+      e.g. a silent stall followed by a missed proposal during catch-up.
+
+      The blackout is gated on `minStartEpoch` so fork-activation devnets can
+      schedule it safely after the fork (set it to fork epoch + 2 or later).
+      The target participant is selected by its 1-based `node-index` label via
+      `targetParticipantIndex`; the counterparty group is computed dynamically
+      from the assertoor client pool. Choose a topology where the remaining
+      nodes retain a 2/3 finality majority (e.g. 4 nodes, blackout 1). A
+      disruptoor service is expected at `disruptoorApiUrl` (default
+      `http://disruptoor:7700`).
+    version: 1.0.0
+    tags:
+      - disruptoor
+      - kurtosis
+      - blackout
+      - node-isolation
+      - recovery
+      - consensus
+    timeout: 120m
   - file: dev/synchronized-check.yaml
     id: synchronized-check
     name: Check client pairs are in sync
@@ -655,6 +687,43 @@ playbooks:
       - queue
       - eip-8282
     timeout: 1h
+  - file: gloas-dev/deploy-eip8282-contracts.yaml
+    id: deploy-eip8282-contracts
+    name: 'GLOAS: Deploy EIP-8282 builder request contracts (pre-fork)'
+    description: |-
+      Deploys the EIP-8282 builder deposit (request type 0x03) and builder exit
+      (request type 0x04) system contracts onto a running devnet BEFORE the GLOAS
+      fork activates.
+
+      glamsterdam-devnet-7 deliberately omits these predeploys from genesis
+      (DEPLOY_EIP8282_CONTRACTS=false in the genesis generator) to exercise the
+      mainnet-like deployment workflow: from GLOAS activation onward every block
+      is INVALID unless both contracts exist, so they must be deployed as regular
+      transactions pre-fork. Intended to be registered as a startup test so the
+      contracts land right after the devnet comes up.
+
+      Deployment goes through the deterministic CREATE2 factory (predeployed in
+      devnet genesis as a well-known contract) using the vanity-mined salts from
+      ethereum/execution-specs#3091 — CREATE2(factory, salt, initcode) yields the
+      canonical ...8282 addresses the clients expect. The factory REVERTS when
+      CREATE2 fails (including the already-deployed collision case), so a
+      successful receipt is proof of deployment.
+
+      Idempotent: an eth_call precheck against the builder deposit contract's fee
+      getter detects existing code and skips deployment, so restarts of a
+      startup-scheduled run stay green. The exit contract cannot be prechecked the
+      same way — its constructor arms the slot-0 excess inhibitor, which makes its
+      fee getter revert by design until the first post-GLOAS system call — so its
+      deployment is asserted via the deploy receipt (both contracts only ever
+      deploy as a pair).
+    version: 1.0.0
+    tags:
+      - gloas
+      - builder
+      - eip-8282
+      - deployment
+      - prefork
+    timeout: 30m
   - file: gloas-dev/exit-builders.yaml
     id: exit-builders
     name: 'GLOAS: Exit builders deposited by prefork-queue-fill-public'

playbooks/dev/single-node-blackout-recovery.yaml

@@ -0,0 +1,193 @@
+id: single-node-blackout-recovery
+name: "Single-Node Blackout & Recovery"
+description: |
+  Fully isolates one participant's beacon node from the rest of a
+  Kurtosis-launched devnet through the disruptoor HTTP API, keeps the
+  blackout active for a configurable number of slots, heals it, then
+  verifies the node re-syncs and the network regains fresh finality.
+
+  "Fully" means the target beacon node loses ALL traffic: p2p and control
+  traffic to every other participant AND to its own execution and validator
+  client (scope `[cl_p2p, el_p2p, include_control]` on both partitions).
+  This reproduces client behaviour under sudden total network loss —
+  e.g. a silent stall followed by a missed proposal during catch-up.
+
+  The blackout is gated on `minStartEpoch` so fork-activation devnets can
+  schedule it safely after the fork (set it to fork epoch + 2 or later).
+  The target participant is selected by its 1-based `node-index` label via
+  `targetParticipantIndex`; the counterparty group is computed dynamically
+  from the assertoor client pool. Choose a topology where the remaining
+  nodes retain a 2/3 finality majority (e.g. 4 nodes, blackout 1). A
+  disruptoor service is expected at `disruptoorApiUrl` (default
+  `http://disruptoor:7700`).
+version: 1.0.0
+tags: [disruptoor, kurtosis, blackout, node-isolation, recovery, consensus]
+timeout: 120m
+config:
+  disruptoorApiUrl: "http://disruptoor:7700" # Disruptoor HTTP API used to apply and clear the blackout.
+  targetParticipantIndex: 1 # 1-based participant index (node-index label) whose beacon node gets blacked out.
+  blackoutDurationSlots: 5 # Number of slots to keep the blackout active (5 slots = 30s on the 6s-slot minimal preset).
+  minStartEpoch: 3 # Do not start the blackout before this epoch (set to fork epoch + 2 or later on fork-activation devnets).
+  minimumClientCount: 2 # Minimum number of healthy clients required before starting the blackout.
+  minFinalizedEpochIncreaseAfterRecovery: 1 # Required finalized epoch increase after healing to prove fresh finality.
+  maxUnfinalizedEpochsAfterRecovery: 6 # Maximum finality lag allowed after healing.
+tasks:
+- name: sleep
+  title: "Wait for disruptoor API to come up"
+  config:
+    duration: 10s
+
+- name: run_shell
+  title: "Check disruptoor API health"
+  timeout: 1m
+  config:
+    envVars:
+      DISRUPTOOR_URL: disruptoorApiUrl
+    command: |
+      set -euo pipefail
+      disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .)
+      curl -fsS "${disruptoor_url}/v1/healthz" >/dev/null
+
+- name: check_clients_are_healthy
+  id: client_check
+  title: "Wait for all devnet clients to be healthy"
+  timeout: 20m
+  configVars:
+    minClientCount: "minimumClientCount"
+  config:
+    maxUnhealthyCount: 0
+
+- name: check_consensus_finality
+  id: initial_finality
+  title: "Wait for initial finality"
+  timeout: 40m
+  config:
+    minFinalizedEpochs: 2
+    maxUnfinalizedEpochs: 3
+
+- name: check_consensus_slot_range
+  title: "Wait for the configured blackout start epoch"
+  timeout: 30m
+  configVars:
+    minEpochNumber: "minStartEpoch"
+
+- name: check_consensus_slot_range
+  id: blackout_start
+  title: "Capture blackout start slot"
+  timeout: 1m
+  config: {}
+
+- name: run_shell
+  title: "Black out the target beacon node"
+  timeout: 1m
+  config:
+    envVars:
+      DISRUPTOOR_URL: disruptoorApiUrl
+      TARGET_INDEX: targetParticipantIndex
+      OTHER_PARTICIPANTS: "| (.targetParticipantIndex | tonumber) as $target | [range(1; .tasks.client_check.outputs.totalCount + 1)] | map(select(. != $target))"
+    command: |
+      set -euo pipefail
+      disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .)
+      target_index=$(echo "$TARGET_INDEX" | jq -r .)
+      other_participants=$(echo "$OTHER_PARTICIPANTS" | jq -c .)
+      other_count=$(echo "$other_participants" | jq 'length')
+      echo "Blacking out participant ${target_index} (beacon) vs ${other_count} other participants (${other_participants})"
+      if [ "$other_count" -lt 1 ]; then
+        echo "Refusing to black out: no other participants to partition against" >&2
+        exit 1
+      fi
+      state_file=$(mktemp)
+      trap 'rm -f "$state_file"' EXIT
+
+      # Two partitions that only ever pit the target beacon node against
+      # others (disruptoor rejects overlapping groups, so a single
+      # "target vs all" partition is not expressible):
+      #   1. target beacon node vs every other participant (all components)
+      #   2. target beacon node vs its own execution + validator client
+      # include_control extends the cut beyond p2p to RPC/engine/API traffic.
+      jq -n \
+        --argjson target "$target_index" \
+        --argjson others "$other_participants" \
+        '{
+          partitions: [
+            {
+              name: "assertoor-blackout-cl-vs-network",
+              groups: [
+                {"node-index": [$target], "client-type": ["beacon"]},
+                {"node-index": $others}
+              ],
+              scope: ["cl_p2p", "el_p2p", "include_control"],
+              symmetric: true
+            },
+            {
+              name: "assertoor-blackout-cl-vs-own-stack",
+              groups: [
+                {"node-index": [$target], "client-type": ["beacon"]},
+                {"node-index": [$target], "client-type": ["execution", "validator"]}
+              ],
+              scope: ["cl_p2p", "el_p2p", "include_control"],
+              symmetric: true
+            }
+          ]
+        }' >"$state_file"
+
+      curl -fsS -X PUT "${disruptoor_url}/v1/state" \
+        -H "Content-Type: application/json" \
+        --data @"$state_file"
+      curl -fsS "${disruptoor_url}/v1/state" | jq -e '.partitions | length == 2' >/dev/null
+
+- name: check_consensus_slot_range
+  title: "Keep the blackout active for the configured duration"
+  timeout: 15m
+  configVars:
+    minSlotNumber: "| (.tasks.blackout_start.outputs.currentSlot | tonumber) + (.blackoutDurationSlots | tonumber)"
+
+- name: run_shell
+  title: "Heal the blackout"
+  timeout: 1m
+  config:
+    envVars:
+      DISRUPTOOR_URL: disruptoorApiUrl
+    command: |
+      set -euo pipefail
+      disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .)
+      curl -fsS -X POST "${disruptoor_url}/v1/state/clear"
+      curl -fsS "${disruptoor_url}/v1/state" | jq -e '(.partitions | length) == 0 and (.shaping | length) == 0' >/dev/null
+
+- name: check_clients_are_healthy
+  title: "Wait for all clients to report healthy after healing"
+  timeout: 10m
+  configVars:
+    minClientCount: "minimumClientCount"
+  config:
+    maxUnhealthyCount: 0
+
+- name: check_consensus_sync_status
+  title: "Wait for all clients to report synced after healing"
+  timeout: 10m
+  config:
+    expectSyncing: false
+    expectMinPercent: 100
+
+- name: check_consensus_finality
+  title: "Poll for fresh finality after recovery"
+  timeout: 25m
+  configVars:
+    # Require a fresh finalized checkpoint after the blackout. maxUnfinalizedEpochs
+    # alone can pass on stale pre-blackout finality when the current epoch is close.
+    minFinalizedEpochs: "| (.tasks.initial_finality.outputs.finalizedEpoch | tonumber) + (.minFinalizedEpochIncreaseAfterRecovery | tonumber)"
+    maxUnfinalizedEpochs: "maxUnfinalizedEpochsAfterRecovery"
+
+cleanupTasks:
+- name: run_shell
+  title: "Clear disruptoor state"
+  timeout: 1m
+  config:
+    envVars:
+      DISRUPTOOR_URL: disruptoorApiUrl
+    command: |
+      set -euo pipefail
+      disruptoor_url=$(echo "$DISRUPTOOR_URL" | jq -r .)
+      if ! curl -fsS -X POST "${disruptoor_url}/v1/state/clear"; then
+        echo "WARNING: failed to clear disruptoor state during cleanup; the target node may still be blacked out" >&2
+      fi