From 010c7c6636eeb73c398c5027429e2356f53c6c24 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 17 Jun 2026 10:52:22 +0200 Subject: [PATCH 01/37] chore: add radius check plan --- docs/radius-check-plan.md | 246 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 docs/radius-check-plan.md diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md new file mode 100644 index 00000000..fe193cf9 --- /dev/null +++ b/docs/radius-check-plan.md @@ -0,0 +1,246 @@ +# Reserve-radius change check — plan + +Status: **draft / not started**. Owner: TBD. Last updated: 2026-06-16. + +> **Working agreement** +> - Build the check on a **new feature branch off `master`** (e.g. +> `feat/reserve-radius-check`), never directly on `master`. The patched-local +> experiments and the bee-side patch may also need a coordinated bee branch. +> - The check must follow the **standardized behaviour of existing checks** — same +> shape, helpers, and idioms as `smoke` / `feed` / `load` (random full-node +> selection via `cluster.ShuffledFullNodeClients`, seeded RNG, `PostageTTL`-based +> batches, `metrics.Reporter`). See "Standard check conventions" below; do not +> invent a new pattern. + +A repeatable, automated way to stage a **storage-radius change** across a Bee +cluster and capture what happens to the reserve and to pull-sync afterwards. The +end product is a beekeeper check (`pkg/check/reserveradius`) plus the metrics it +emits. Operating know-how lives in the `radius-testing` skill; this doc is the +design and the build checklist. + +## Why this is hard (and what prior tryouts taught us) + +- A stock node's reserve is far too large to overflow in CI-time, so a radius + transition won't happen on a small local cluster without help. +- After a radius change, the node **resyncs its reserve over pull-sync**, and the + redistribution-game "fully synced" status takes a long time to settle. So the + test is **stage → monitor over time**, not a one-shot assertion. +- **PR #581** (`radiusdecrease`, open, unmerged) — deliberately forces a radius + *decrease* (overflow a tiny reserve to push `storageRadius` 0→1, let it fall + 1→0) and polls `/status` for `pullsyncRate > 0` to confirm puller recovery. + Reusable `waitForRadius` / `waitForRecovery` poll loops. **Requires a patched + bee image** (`master-scenario-b`) + a coordinated bee PR; no review; carries + dirty local config. The real radius tryout. +- **PR #591** (`stampexpiry`, draft) — patch-free, observes radius only as a side + effect of stamp expiry. Review feedback (unactioned): use `PostageTTL`, pick a + **random** node not `sortedNodes[0]`, account for the ~10-block batch-usable + delay, and run as a **periodic / public-testnet RC** check, not standard CI. +- **GC check** (`pkg/check/gc/reserve.go`) — the precedent for a check that + depends on a bee patch: patches live in `bee/.github/patches/`, applied by + bee's beekeeper workflow before the check runs. + +## Signals (see the `radius-testing` skill for the full table) + +Per node, HTTP API (`http://.localhost`): +`/status` → `storageRadius`, `pullsyncRate`, `reserveSize`, +`reserveSizeWithinRadius`, `committedDepth`; `/reservestate` → `radius`, +`commitment`; `/redistributionstate` (full-mode only) → `isFullySynced`, `phase`. +Prometheus: `bee_localstore_storage_radius`, `bee_localstore_reserve_size*`, +`bee_pullsync_chunks_*`, `bee_postage_radius`, `bee_storageincentives_*`. + +## Approach: patched-local first, then real ephemeral + +**Decision:** start with a **patched local cluster** for deterministic, fast radius +transitions; graduate to an **unpatched ephemeral k8s cluster** for realistic +timing (periodic/RC cadence). Both phases drive the same check; only the +environment and timeouts differ. + +--- + +## Phase 0 — Local loop scaffolding (mostly done) + +- [x] `/cluster-verify` command — verify substrate + Bee nodes + capture a + reserve/radius baseline. +- [x] `/cluster-up` / `/cluster-down` commands — wrap beelocal + docker + beekeeper. +- [x] `radius-testing` skill + `scripts/radius-poll.sh` — the monitoring poller. +- [ ] Run `/cluster-up local-dns`, `/cluster-verify local-dns`, confirm a clean + baseline (`storageRadius==0`, full nodes connected). + +## Staging the change: driving the radius via uploads (reuse the `load` check) + +You do **not** pre-compute "how many bytes" to upload — you upload in a loop, watch the +radius signal, and stop at a target. The `load` check already does exactly this: + +- `load` (`pkg/check/load/load.go`) has `MaxCommittedDepth` and gates every upload on + `checkCommittedDepth(client, max, wait)`, which reads `client.Status(ctx).CommittedDepth` + and keeps uploading while `CommittedDepth < max` (and *waits* once the cap is reached). + Since `committedDepth = storageRadius + capacityDoubling`, capping committedDepth caps radius. + `ci-load` in `config/local.yaml` already wires `max-committed-depth: 2`. + +**Increase and decrease are NOT symmetric — this is the crux:** + +- **Increase** is upload-driven: fill the reserve past capacity ⇒ eviction ⇒ `storageRadius` + rises. `load` already produces this. +- **Decrease** is *not* upload-driven. Per `bee/pkg/storer/reserve.go`, radius decreases only + when the reserve is **under-utilized** AND the node is **fully synced** (`SyncRate()==0`) AND + above `minimumRadius`. Trigger it by first inflating radius, then letting the patched + (tiny-reserve, threshold=capacity, fast wake-up) reserve worker re-evaluate — PR #581's + "overflow to push 0→1, then watch it fall 1→0" cascade. **The resync-over-decrease is the + mechanism that is not yet understood, so it must be observed (spike) before it is designed.** + +**Decisions:** +- The final `reserveradius` check **drives its own upload** (it needs tight + stage→poll→record→assert control), **reusing load's committedDepth-gated upload primitive** + — factor it into a shared helper rather than copy/paste. Smaller batches = finer radius control. +- For the **initial spike**, drive uploads with the existing `ci-load` check (set + `max-committed-depth` to the target) while `radius-poll.sh` records the timeline — no new Go. + +## Phase 1 — Empirical spike: force a change with a patched bee, observe decrease + resync + +- [x] **Patch created** (in `bee/.github/patches/`, GC-check pattern). All three symbols + are unreachable by config — `ReserveCapacity` = `(1< +patch pkg/storer/reserve.go .github/patches/radius_threshold.patch +patch pkg/storer/storer.go .github/patches/radius_reserve.patch +make docker-build PLATFORM=linux/arm64 BEE_IMAGE=k3d-registry.localhost:5000/ethersphere/bee:latest \ + REACHABILITY_OVERRIDE_PUBLIC=true BATCHFACTOR_OVERRIDE_PUBLIC=2 +docker push k3d-registry.localhost:5000/ethersphere/bee:latest +git checkout pkg/storer/reserve.go pkg/storer/storer.go +``` + +Then redeploy so nodes pull the patched `:latest` (imagePullPolicy: Always): +`/cluster-down local-dns` → `/cluster-up local-dns` (or `kubectl rollout restart statefulset -n local`). +To wire into CI later, add the two `patch` lines to bee's `.github/workflows/beekeeper.yml` +"Apply patches and build" step (next to `postage_api`/`retrieval`). + +- [ ] **Drive the increase** with the existing `ci-load` check (no new Go): set + `max-committed-depth` to the target so it uploads until `storageRadius`/`committedDepth` + reaches it. Run `radius-poll.sh` alongside to record the 0→target timeline. +- [ ] **Observe the decrease + resync** — stop uploads and watch whether `storageRadius` + falls back and how pull-sync behaves (`pullsyncRate`, `reserveSize`, + `reserveSizeWithinRadius` over time). Capture the full CSV timeline; use `/loop` for the + long watch. This answers the open question of *how* resync-over-radius-decrease works. +- [x] **Gate met** — spike reproduced a real increase, decrease, and resync (findings below). + +### Spike findings (2026-06-17, local-dns, patched `:200/10s/100%` image) + +Sequence observed (3 full nodes, uploading 1 MB blobs to `bee-0`): + +1. **Increase is immediate** — `storageRadius` 0→1 after ~2 uploads (~2 MB), 0→2 within ~14 s. + On stock capacity it never moves; the patch is what makes it observable. +2. **Overshoot after uploads stop** — radius kept climbing (1→2) *after* uploads ceased, + because in-flight **pushsync** keeps filling reserves. "Uploads stopped" ≠ "radius settled". +3. **Decrease lags ~10 min and is gated on stabilization** — the 2→1 decrease fired at + ~10.5 min after node start (`node "Sync status check evaluated" stabilized=true`), ~30 s + after a 10-min watch window ended. The reserve worker's decrease loop sits behind the + startup-stabilizer gate (`reserve.go` `startReserveWorkers` waits on `startupStabilizer`), + so a fresh node won't decrease for several minutes regardless of reserve state. +4. **Puller drives the reconfiguration** — `node/puller "radius decrease" old=2 new=1`, preceded + by a storm of `syncWorker context cancelled` (disconnect/reconnect per bin). This is the + PR #581 `manage()`/`disconnectPeer()` path — the liveness risk lives here. +5. **Decrease ↔ resync interlock (staircase)** — decrease requires `SyncRate()==0`, but a decrease + triggers pull-sync (`pullsyncRate>0`), which then blocks the next decrease until it drains. + Radius settles in steps, not one jump. + +**Implications for the Phase 2 check (bake these in):** + +- Use a **random full node** but expect the transition to show on whichever node's neighborhood + fills — assert on the cluster, not one node. +- **Long, staged timeouts**: increase ~minutes; decrease/settle **≥15 min** (cf. PR #581's 20-min + recovery). Don't fail fast. +- "Radius reached target" must wait for **pushsync to drain** (overshoot) before treating a value + as settled — poll until `storageRadius` is stable for K ticks AND `pullsyncRate==0`, not a single read. +- Gate the decrease assertion on **`isWarmingUp==false`/stabilized** first; before that, no decrease can occur. +- The key liveness signal is `pullsyncRate>0` returning after the decrease (puller workers + recovered) — the same assertion PR #581 makes. A stuck `manage()` shows as `pullsyncRate` flat at 0. +- `radius-poll.sh` with `-i 15` + auto-stop-on-stable is the right monitor; the in-repo check + formalizes this loop. + +## Standard check conventions (match existing checks — don't invent a pattern) + +Copy the shape of `smoke` (`pkg/check/smoke/smoke.go`), `feed`, `load`. Concretely: + +- **Boilerplate:** `Options` struct + `NewDefaultOptions() Options`; compile check + `var _ beekeeper.Action = (*Check)(nil)`; `Check{ metrics, logger }`; + `NewCheck(log logging.Logger) beekeeper.Action` → `&Check{metrics: newMetrics("check_reserve_radius"), logger: log}`. +- **Run signature:** `Run(ctx context.Context, cluster orchestration.Cluster, opts any) error`, + first line `o, ok := opts.(Options); if !ok { return errors.New("invalid options type") }`. +- **Seeded RNG + random node selection** (the part the user called out — `smoke.go:116-131`). + Take `[0]` of the shuffled list; never hard-index an unshuffled list / always-node-0 (the + exact thing flagged in PR #591 review): + + ```go + rnd := random.PseudoGenerator(o.RndSeed) // pkg/random + fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) // (ctx, *rand.Rand) (ClientList, error) + if err != nil { return fmt.Errorf("get shuffled full node clients: %w", err) } + if len(fullNodeClients) < 1 { return fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(fullNodeClients)) } + node := fullNodeClients[0] // a random node, since the list is shuffled + c.logger.Infof("random seed: %d", o.RndSeed) + ``` + +- **Postage:** use `node.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel)` + with `PostageTTL time.Duration` (default `24h`), **not** a raw amount. +- **Options idioms:** `RndSeed int64` defaulting to `time.Now().UnixNano()`; durations as + `time.Duration`; a `Duration` + `scheduler.NewDurationExecutor(...)` if it's a long/repeating run. +- **Loops:** `select { case <-ctx.Done(): return nil; default: }` guard each iteration. +- **Errors:** wrap with context (`fmt.Errorf("...: %w", err)`); failure messages name the + suspected cause + the timeout hit. +- **Tests:** external `package reserveradius_test`; race detector clean. + +## Phase 2 — The beekeeper check (`pkg/check/reserveradius`) + +On a **new branch** (`feat/reserve-radius-check`). Mirror the existing check structure +(`Action` interface, `NewCheck(logger)`, `metrics.Reporter`, the conventions above); +register in `pkg/config/check.go` and add a `ci-reserve-radius` entry to `config/local.yaml`. + +- [ ] `pkg/check/reserveradius/reserveradius.go` + - `Options`: target direction (increase/decrease), upload size, postage + (`PostageTTL` not raw amount), poll interval, overflow/recovery timeouts, seed. + - `Run`: (1) pick a random full node via `cluster.ShuffledFullNodeClients`; + (2) record baseline `/status` + `/reservestate`; (3) **stage** the change + (buy batch, upload to overflow / drive decrease); (4) **monitor**: poll + until `storageRadius` reaches target and then `pullsyncRate` returns to 0 + (reuse #581's `waitForRadius` / `waitForRecovery`); (5) assert + emit metrics. + - Fail messages must name the suspected cause (e.g. puller `manage()` stuck, + cf. #581) and the timeout hit. +- [ ] `pkg/check/reserveradius/metrics.go` — emit: `storage_radius` gauge per node + over time, `time_to_radius_change_seconds`, `time_to_resync_seconds` + (pullsync→0), `reserve_size` / `reserve_size_within_radius` gauges, + `pullsync_rate` gauge. Implement `Report()`. +- [ ] Register in `pkg/config/check.go` `Checks` map + `NewOptions` decoder. +- [ ] `config/local.yaml`: `ci-reserve-radius` check + (if needed) a patched + node-group/bee-config. Generous timeout. +- [ ] Gate: `make build && make vet && make lint && make test`. + +## Phase 3 — Real ephemeral cluster, no patch + +- [ ] Run the same check against an unpatched ephemeral k8s cluster with realistic + reserve sizes and timing; expect long durations. +- [ ] Wire it as a **periodic / public-testnet RC** check (per #591 review), not + standard PR CI. A/B candidate bee versions by comparing the emitted metrics. + +## Open questions + +- Increase, decrease, or both? Decrease exercises the known puller bug (#581) and + is the higher-value regression target; increase is the common steady-state path. +- Patch vs config: how much of the small-reserve setup can come from `bee-config` + knobs alone, avoiding a source patch entirely? +- Does `/redistributionstate` need full-mode + incentives enabled on the local + cluster, or do we assert purely on `/status` + `/reservestate`? +- Metrics sink for local runs: pushgateway vs scrape vs the CSV from `radius-poll.sh`. + +## References + +- Skill: `.claude/skills/radius-testing/SKILL.md` (+ `scripts/radius-poll.sh`) +- Prior art: PR #581, PR #591, `pkg/check/gc/reserve.go`, `pkg/check/pingpong/`, + `pkg/check/smoke/metrics.go` +- Bee internals: `bee/pkg/storer/reserve.go`, `bee/pkg/postage/batchstore/store.go`, + `bee/pkg/api/{status,postage,redistribution}.go` From 13a8a77b32389fe7dbeebcccee413565c6ddb65a Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 17 Jun 2026 12:32:21 +0200 Subject: [PATCH 02/37] feat(check): add reserve-radius check --- config/local.yaml | 17 ++ docs/radius-check-plan.md | 49 ++-- pkg/bee/api/status.go | 1 + pkg/check/reserveradius/metrics.go | 67 +++++ pkg/check/reserveradius/reserveradius.go | 296 +++++++++++++++++++++++ pkg/config/check.go | 29 +++ 6 files changed, 438 insertions(+), 21 deletions(-) create mode 100644 pkg/check/reserveradius/metrics.go create mode 100644 pkg/check/reserveradius/reserveradius.go diff --git a/config/local.yaml b/config/local.yaml index 4d2e81f8..7d28bb61 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -373,6 +373,23 @@ checks: - light timeout: 30m type: load + ci-reserve-radius: + options: + postage-ttl: 24h + postage-depth: 22 + postage-label: reserve-radius + blob-size: 1048576 + max-uploads: 60 + target-radius: 1 + warmup-wait: 15m + increase-timeout: 5m + settle-wait: 1m + decrease-timeout: 20m + poll-interval: 15s + upload-groups: + - bee + timeout: 45m + type: reserve-radius ci-soc: options: postage-ttl: 24h diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index fe193cf9..2ebb0a1e 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -149,6 +149,13 @@ Sequence observed (3 full nodes, uploading 1 MB blobs to `bee-0`): 5. **Decrease ↔ resync interlock (staircase)** — decrease requires `SyncRate()==0`, but a decrease triggers pull-sync (`pullsyncRate>0`), which then blocks the next decrease until it drains. Radius settles in steps, not one jump. +6. **It does not return to a fixed floor** — after `2→1`, the radius held at **1 for 30+ min and never + reached 0** (still 1 at re-check, even with `pullsyncRate==0` on two of three nodes by then; + `reserveSizeWithinRadius` rose to equal `reserveSize` as the reserve reorganized within radius 1). + The `1→0` step simply did not occur in-window — the exact gate is secondary (candidate: residual + sync noise and/or how `countWithinRadius` evaluates at the boundary). Design takeaway: the check + must assert *"a decrease occurred AND pullsync recovered"*, NOT *"radius returned to N"* — the + equilibrium depends on data volume and sync state. **Implications for the Phase 2 check (bake these in):** @@ -197,28 +204,28 @@ Copy the shape of `smoke` (`pkg/check/smoke/smoke.go`), `feed`, `load`. Concrete ## Phase 2 — The beekeeper check (`pkg/check/reserveradius`) -On a **new branch** (`feat/reserve-radius-check`). Mirror the existing check structure +On branch **`ljubisa-radius-check`**. Mirrors the existing check structure (`Action` interface, `NewCheck(logger)`, `metrics.Reporter`, the conventions above); -register in `pkg/config/check.go` and add a `ci-reserve-radius` entry to `config/local.yaml`. - -- [ ] `pkg/check/reserveradius/reserveradius.go` - - `Options`: target direction (increase/decrease), upload size, postage - (`PostageTTL` not raw amount), poll interval, overflow/recovery timeouts, seed. - - `Run`: (1) pick a random full node via `cluster.ShuffledFullNodeClients`; - (2) record baseline `/status` + `/reservestate`; (3) **stage** the change - (buy batch, upload to overflow / drive decrease); (4) **monitor**: poll - until `storageRadius` reaches target and then `pullsyncRate` returns to 0 - (reuse #581's `waitForRadius` / `waitForRecovery`); (5) assert + emit metrics. - - Fail messages must name the suspected cause (e.g. puller `manage()` stuck, - cf. #581) and the timeout hit. -- [ ] `pkg/check/reserveradius/metrics.go` — emit: `storage_radius` gauge per node - over time, `time_to_radius_change_seconds`, `time_to_resync_seconds` - (pullsync→0), `reserve_size` / `reserve_size_within_radius` gauges, - `pullsync_rate` gauge. Implement `Report()`. -- [ ] Register in `pkg/config/check.go` `Checks` map + `NewOptions` decoder. -- [ ] `config/local.yaml`: `ci-reserve-radius` check + (if needed) a patched - node-group/bee-config. Generous timeout. -- [ ] Gate: `make build && make vet && make lint && make test`. +registered in `pkg/config/check.go`, `ci-reserve-radius` entry in `config/local.yaml`. + +- [x] `pkg/check/reserveradius/reserveradius.go` — `Run` = `waitForWarmupDone` → + baseline → `driveIncrease` (upload blobs until `storageRadius≥TargetRadius`, reuses + `test.Upload`) → settle (pushsync drain) → `observeDecrease` (assert a decrease vs the + per-node peak; watch `pullsyncRate` recovery; timeout message names the PR #581 puller stall). +- [x] `pkg/check/reserveradius/metrics.go` — `storage_radius` / `reserve_size` / + `pullsync_rate` gauges per node + `time_to_increase_seconds` / `time_to_decrease_seconds`. +- [x] Registered in `pkg/config/check.go` (type `reserve-radius`); added `IsWarmingUp` to + `pkg/bee/api/status.go` `StatusResponse` (node returns `isWarmingUp`; struct lacked it). +- [x] `config/local.yaml`: `ci-reserve-radius` (timeout 45m). Runs against the patched image. +- [x] Gate: `make build`, `go vet`, `golangci-lint` (0 issues) all green. (Unit tests deferred.) +- [x] **Validated end-to-end (2026-06-17)** on a fresh local-dns: increase 0→1 in 3s (1 MiB), + 1-min settle, **decrease observed on bee-2 after 13m16s with `pullsyncRate>0` recovery=true**, + `check completed successfully` (total 14m36s). Confirms the ~13-min stabilization-gated + decrease and that the 20-min `DecreaseTimeout` is correctly sized. + +Open follow-ups (not blockers): make `pullsyncRate>0` recovery a **hard** gate once its rate floor is +characterised on a data-heavy cluster (Phase 3); add external `_test` package; optional `increase`-only +and `both` directions. ## Phase 3 — Real ephemeral cluster, no patch diff --git a/pkg/bee/api/status.go b/pkg/bee/api/status.go index 32e98a16..de0028f0 100644 --- a/pkg/bee/api/status.go +++ b/pkg/bee/api/status.go @@ -22,6 +22,7 @@ type StatusResponse struct { IsReachable bool `json:"isReachable"` LastSyncedBlock uint64 `json:"lastSyncedBlock"` CommittedDepth uint8 `json:"committedDepth"` + IsWarmingUp bool `json:"isWarmingUp"` } // Ping pings given node diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go new file mode 100644 index 00000000..0d69df40 --- /dev/null +++ b/pkg/check/reserveradius/metrics.go @@ -0,0 +1,67 @@ +package reserveradius + +import ( + m "github.com/ethersphere/beekeeper/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type metrics struct { + StorageRadius *prometheus.GaugeVec + ReserveSize *prometheus.GaugeVec + PullsyncRate *prometheus.GaugeVec + TimeToIncrease prometheus.Gauge + TimeToDecrease prometheus.Gauge +} + +func newMetrics(subsystem string) metrics { + return metrics{ + StorageRadius: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "storage_radius", + Help: "Storage radius reported by /status, per node.", + }, + []string{"node"}, + ), + ReserveSize: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "reserve_size", + Help: "Reserve size (chunks) reported by /status, per node.", + }, + []string{"node"}, + ), + PullsyncRate: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "pullsync_rate", + Help: "Pull-sync rate reported by /status, per node.", + }, + []string{"node"}, + ), + TimeToIncrease: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "time_to_increase_seconds", + Help: "Seconds to reach the target storage radius from the first upload.", + }, + ), + TimeToDecrease: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "time_to_decrease_seconds", + Help: "Seconds from uploads-stopped to the first observed radius decrease.", + }, + ), + } +} + +// Report implements the metrics.Reporter interface. +func (c *Check) Report() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(c.metrics) +} diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go new file mode 100644 index 00000000..5e1d6de0 --- /dev/null +++ b/pkg/check/reserveradius/reserveradius.go @@ -0,0 +1,296 @@ +// Package reserveradius stages a storage-radius change across a cluster and +// verifies the node recovers (pull-sync resumes) afterwards. +// +// It follows the empirical sequence found by the Phase-1 spike (see +// docs/radius-check-plan.md "Spike findings"): +// +// 1. wait for warmup/stabilization — the reserve worker's decrease loop is +// gated on it, so a fresh node will not decrease for several minutes; +// 2. drive the radius UP by uploading until a node reaches the target; +// 3. let pushsync overshoot drain (uploads stopping != radius settling); +// 4. stop uploading and watch for a DECREASE and for pull-sync to recover +// (pullsyncRate > 0) — a stuck puller manage()/disconnectPeer() shows as +// no decrease / no recovery within the timeout (cf. beekeeper PR #581). +// +// This check requires a bee node patched for a small reserve (see +// bee/.github/patches/radius_*.patch); on stock capacity the radius never moves. +package reserveradius + +import ( + "context" + crand "crypto/rand" + "errors" + "fmt" + "time" + + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/bee/api" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + "github.com/ethersphere/beekeeper/pkg/random" + "github.com/ethersphere/beekeeper/pkg/test" +) + +// Options represents reserve-radius check options. +type Options struct { + RndSeed int64 + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string + UploadGroups []string // node groups to upload to / observe (empty = all full nodes) + BlobSize int64 // bytes per upload + MaxUploads int // cap on uploads during the increase phase + TargetRadius uint8 // storageRadius to reach before stopping uploads + WarmupWait time.Duration // max wait for nodes to leave warmup before staging + IncreaseTimeout time.Duration // max time to reach TargetRadius + SettleWait time.Duration // wait after uploads for pushsync overshoot to drain + DecreaseTimeout time.Duration // max time to observe a decrease after uploads stop + PollInterval time.Duration +} + +// NewDefaultOptions returns new default options. +func NewDefaultOptions() Options { + return Options{ + RndSeed: time.Now().UnixNano(), + PostageTTL: 24 * time.Hour, + PostageDepth: 22, + PostageLabel: "reserve-radius", + UploadGroups: []string{"bee"}, + BlobSize: 1 << 20, // 1 MiB + MaxUploads: 60, + TargetRadius: 1, + WarmupWait: 15 * time.Minute, + IncreaseTimeout: 5 * time.Minute, + SettleWait: time.Minute, + DecreaseTimeout: 20 * time.Minute, + PollInterval: 15 * time.Second, + } +} + +// compile check whether Check implements interface +var _ beekeeper.Action = (*Check)(nil) + +// Check instance. +type Check struct { + metrics metrics + logger logging.Logger +} + +// NewCheck returns a new reserve-radius check. +func NewCheck(log logging.Logger) beekeeper.Action { + return &Check{ + metrics: newMetrics("check_reserve_radius"), + logger: log, + } +} + +// Run stages a radius increase, then observes the decrease + pull-sync recovery. +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return errors.New("invalid options type") + } + if o.TargetRadius == 0 { + return errors.New("target-radius must be > 0") + } + + c.logger.Infof("random seed: %d", o.RndSeed) + rnd := random.PseudoGenerator(o.RndSeed) + fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) + if err != nil { + return fmt.Errorf("get shuffled full node clients: %w", err) + } + nodes := fullNodeClients + if len(o.UploadGroups) > 0 { + nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) + } + if len(nodes) < 1 { + return fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(nodes)) + } + uploader := nodes[0] // a random node, since the list is shuffled + c.logger.Infof("uploader: %s, observing %d node(s)", uploader.Name(), len(nodes)) + + // 1. Wait for warmup/stabilization — the decrease loop is gated on it. + if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { + return err + } + c.snapshot(ctx, nodes, "baseline") + + // 2. Drive the radius up by uploading. + batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + return fmt.Errorf("create batch on %s: %w", uploader.Name(), err) + } + c.logger.WithField("batch_id", batchID).Infof("node %s: using batch", uploader.Name()) + if err := c.driveIncrease(ctx, uploader, nodes, batchID, o); err != nil { + return err + } + + // 3. Let pushsync overshoot drain before snapshotting the peak. + c.logger.Infof("uploads stopped; waiting %s for pushsync to settle", o.SettleWait) + if err := sleepCtx(ctx, o.SettleWait); err != nil { + return err + } + peak := c.radiusByNode(ctx, nodes) + c.logger.Infof("peak storageRadius per node after settle: %v", peak) + + // 4. Observe the decrease + pull-sync recovery. + return c.observeDecrease(ctx, nodes, peak, o) +} + +// waitForWarmupDone blocks until every observed node reports isWarmingUp=false. +func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.ClientList, o Options) error { + c.logger.Infof("waiting up to %s for nodes to finish warmup/stabilization", o.WarmupWait) + deadline := time.Now().Add(o.WarmupWait) + for { + if err := ctx.Err(); err != nil { + return err + } + ready := true + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil || s.IsWarmingUp { + ready = false + } + } + if ready { + c.logger.Info("all observed nodes finished warmup") + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("nodes still warming up after %s", o.WarmupWait) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + +// driveIncrease uploads blobs to the uploader until any observed node reaches TargetRadius. +func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes orchestration.ClientList, batchID string, o Options) error { + c.logger.Infof("driving increase: %d-byte blobs to %s until storageRadius>=%d (max %d uploads, timeout %s)", + o.BlobSize, uploader.Name(), o.TargetRadius, o.MaxUploads, o.IncreaseTimeout) + start := time.Now() + deadline := start.Add(o.IncreaseTimeout) + t := test.NewTest(c.logger) + data := make([]byte, o.BlobSize) + + for i := 1; i <= o.MaxUploads; i++ { + if err := ctx.Err(); err != nil { + return err + } + if time.Now().After(deadline) { + return fmt.Errorf("storageRadius did not reach %d within %s (%d uploads) — is the bee reserve patch active and is there enough data?", o.TargetRadius, o.IncreaseTimeout, i-1) + } + if _, err := crand.Read(data); err != nil { + return fmt.Errorf("generate random data: %w", err) + } + if _, _, err := t.Upload(ctx, uploader, data, batchID, nil); err != nil { + c.logger.Errorf("upload #%d failed: %v", i, err) + continue + } + mx := c.snapshot(ctx, nodes, "increase") + c.logger.Infof("increase: upload #%d, max storageRadius=%d (target %d)", i, mx, o.TargetRadius) + if mx >= o.TargetRadius { + c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) + c.logger.Infof("reached storageRadius %d after %d uploads (~%.1f MiB) in %s", + mx, i, float64(int64(i)*o.BlobSize)/(1<<20), time.Since(start).Round(time.Second)) + return nil + } + } + return fmt.Errorf("storageRadius did not reach %d after %d uploads", o.TargetRadius, o.MaxUploads) +} + +// observeDecrease watches for any node's radius to fall below its peak, and for +// pull-sync to recover (pullsyncRate>0). No decrease within the timeout is the +// failure signal (cf. PR #581 puller manage()/disconnectPeer() stall). +func (c *Check) observeDecrease(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8, o Options) error { + c.logger.Infof("watching for radius decrease + pull-sync recovery (timeout %s)", o.DecreaseTimeout) + start := time.Now() + deadline := start.Add(o.DecreaseTimeout) + recovered := false + + for { + if err := ctx.Err(); err != nil { + return err + } + decreasedOn := "" + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + continue + } + c.emit(n.Name(), s) + if s.PullsyncRate > 0 { + recovered = true + } + if p, ok := peak[n.Name()]; ok && s.StorageRadius < p { + decreasedOn = n.Name() + } + } + if decreasedOn != "" { + c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) + c.logger.Infof("radius decrease observed on %s after %s (pull-sync recovery seen: %t)", + decreasedOn, time.Since(start).Round(time.Second), recovered) + // TODO(reserve-radius): make pull-sync recovery a hard gate once the + // expected rate floor is characterised on a data-heavy cluster (Phase 3). + if !recovered { + c.logger.Warning("decrease observed but pullsyncRate stayed 0 — puller may not have resumed; investigate (cf. PR #581)") + } + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("no radius decrease within %s after uploads stopped — puller manage()/disconnectPeer() may be stalled (cf. PR #581), or the decrease gate (synced + count mx { + mx = s.StorageRadius + } + c.logger.Infof("[%s] %s: storageRadius=%d reserveSize=%d withinR=%d pullsyncRate=%.4f warmingUp=%t", + phase, n.Name(), s.StorageRadius, s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate, s.IsWarmingUp) + } + return mx +} + +func (c *Check) radiusByNode(ctx context.Context, nodes orchestration.ClientList) map[string]uint8 { + out := make(map[string]uint8, len(nodes)) + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + continue + } + out[n.Name()] = s.StorageRadius + } + return out +} + +func (c *Check) emit(node string, s *api.StatusResponse) { + c.metrics.StorageRadius.WithLabelValues(node).Set(float64(s.StorageRadius)) + c.metrics.ReserveSize.WithLabelValues(node).Set(float64(s.ReserveSize)) + c.metrics.PullsyncRate.WithLabelValues(node).Set(s.PullsyncRate) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} diff --git a/pkg/config/check.go b/pkg/config/check.go index 06332fb1..e676bcc5 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -30,6 +30,7 @@ import ( "github.com/ethersphere/beekeeper/pkg/check/pullsync" "github.com/ethersphere/beekeeper/pkg/check/pushsync" "github.com/ethersphere/beekeeper/pkg/check/redundancy" + "github.com/ethersphere/beekeeper/pkg/check/reserveradius" "github.com/ethersphere/beekeeper/pkg/check/retrieval" "github.com/ethersphere/beekeeper/pkg/check/settlements" "github.com/ethersphere/beekeeper/pkg/check/smoke" @@ -495,6 +496,34 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "reserve-radius": { + NewAction: reserveradius.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + RndSeed *int64 `yaml:"rnd-seed"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + UploadGroups *[]string `yaml:"upload-groups"` + BlobSize *int64 `yaml:"blob-size"` + MaxUploads *int `yaml:"max-uploads"` + TargetRadius *uint8 `yaml:"target-radius"` + WarmupWait *time.Duration `yaml:"warmup-wait"` + IncreaseTimeout *time.Duration `yaml:"increase-timeout"` + SettleWait *time.Duration `yaml:"settle-wait"` + DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + PollInterval *time.Duration `yaml:"poll-interval"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := reserveradius.NewDefaultOptions() + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, "soc": { NewAction: soc.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { From 7e42c73d6be438f4ed7122ccb1edc1409f4fa4d3 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 18 Jun 2026 11:09:39 +0200 Subject: [PATCH 03/37] fix(check): track high-water peak in reserve-radius check --- pkg/check/reserveradius/README.md | 114 +++++++++++++++++++++++ pkg/check/reserveradius/reserveradius.go | 44 ++++++--- 2 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 pkg/check/reserveradius/README.md diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md new file mode 100644 index 00000000..eecbf3b8 --- /dev/null +++ b/pkg/check/reserveradius/README.md @@ -0,0 +1,114 @@ +# reserve-radius check + +Stages a **storage-radius change** across a Bee cluster and verifies the node +recovers afterwards. It drives the radius up by uploading, then watches for the +radius to come back down and for pull-sync to react — catching regressions in the +puller's reaction to a radius change (the `manage()` / `disconnectPeer()` path that +caused the liveness bug in beekeeper PR #581). + +> Keep this file in sync with the code. If you change `Options`, the `Run` flow, the +> emitted metrics, the registration, or the bee-patch requirement, update the matching +> section below in the same change. + +## What it does (`Run` flow) + +`reserveradius.go`, in order: + +1. **`waitForWarmupDone`** — block until every observed node reports + `isWarmingUp == false`. The reserve worker's *decrease* loop is gated on + stabilization, so this is a precondition for the radius ever dropping. +2. **baseline** — snapshot `/status` for all observed nodes. +3. **`driveIncrease`** — buy a mutable batch on a random full node, then upload + `BlobSize` random blobs to it until any observed node's `storageRadius` reaches + `TargetRadius` (or `MaxUploads` / `IncreaseTimeout`). Tracks a **per-node + high-water `peak`**. +4. **settle** — keep polling for `SettleWait`, still raising `peak`. This matters: + on an already-stabilized node the decrease can begin *during* settle, so `peak` + must be the max seen, not a single post-settle read (otherwise it reads back at + baseline and no decrease is ever detectable). +5. **`observeDecrease`** — poll until any node's `storageRadius` falls below its + `peak` (success), watching `pullsyncRate` for recovery along the way. If no + decrease occurs within `DecreaseTimeout`, fail with a message pointing at the + PR #581 puller stall. + +## Options + +Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in +`pkg/config/check.go` under the `reserve-radius` entry. + +| field | yaml | default | purpose | +| --- | --- | --- | --- | +| `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | seed for `random.PseudoGenerator` → shuffled node pick | +| `PostageTTL` | `postage-ttl` | `24h` | batch TTL (use TTL, not a raw amount) | +| `PostageDepth` | `postage-depth` | `22` | batch depth | +| `PostageLabel` | `postage-label` | `reserve-radius` | batch label | +| `UploadGroups` | `upload-groups` | `[bee]` | node groups to upload to / observe (empty = all full nodes) | +| `BlobSize` | `blob-size` | `1048576` (1 MiB) | bytes per upload | +| `MaxUploads` | `max-uploads` | `60` | cap on uploads in the increase phase | +| `TargetRadius` | `target-radius` | `1` | storageRadius to reach before stopping uploads | +| `WarmupWait` | `warmup-wait` | `15m` | max wait for nodes to finish warmup | +| `IncreaseTimeout` | `increase-timeout` | `5m` | max time to reach `TargetRadius` | +| `SettleWait` | `settle-wait` | `1m` | post-upload window (pushsync drain + peak tracking) | +| `DecreaseTimeout` | `decrease-timeout` | `20m` | max time to observe a decrease | +| `PollInterval` | `poll-interval` | `15s` | poll cadence | + +## Metrics (`metrics.go`) + +Emitted via the `metrics.Reporter` interface (pushed to the pushgateway when the +check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem +`check_reserve_radius`: + +- `…_storage_radius{node}` — gauge, storageRadius per node +- `…_reserve_size{node}` — gauge, reserve size (chunks) per node +- `…_pullsync_rate{node}` — gauge, `/status` pullsyncRate per node +- `…_time_to_increase_seconds` — gauge, first-upload → `TargetRadius` +- `…_time_to_decrease_seconds` — gauge, uploads-stopped → first observed decrease + +## Requirements + +- **Patched bee image.** On stock capacity the reserve is far too large to move. + The check needs a node built with `bee/.github/patches/radius_reserve.patch` + (`DefaultReserveCapacity`→200, `ReserveWakeUpDuration`→10s) and + `radius_threshold.patch` (decrease `threshold`→100%). See + `docs/radius-check-plan.md` and the `radius-testing` skill. +- **`isWarmingUp` in `StatusResponse`** (`pkg/bee/api/status.go`) — the node returns + it; the struct field was added for this check's warmup gate. + +## Running it + +Against a patched local cluster (`local-dns`), via the `ci-reserve-radius` config +entry (timeout 45m): + +```sh +./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius --log-verbosity=info +# add --metrics-enabled=true --metrics-pusher-address=localhost:9091 to push metrics +``` + +## Observed behavior (local, patched cluster) + +- **Increase is fast** — ~1 MiB moves `storageRadius` 0→1 in seconds. +- **Decrease lags ~13 min** and is stabilization-gated; sized by `DecreaseTimeout` (20m). +- **No fixed floor** — the equilibrium radius depends on data volume (a data-heavy + node can stay elevated and never decrease in-window). The check asserts *a decrease + occurred*, not that the radius returns to a specific value. +- **Pull-sync on a radius change is a puller reconfiguration**: `bee_puller_worker` + contracts on increase / expands on decrease, with a `bee_puller_worker_errors` + cancel-storm. Actual resync volume (`bee_pullsync_chunks_delivered`) is ~0 on a + near-empty cluster — meaningful resync needs data (Phase 3 / real cluster). + +## Known limitations / follow-ups + +- Pull-sync recovery is **observed and logged, not asserted** (`observeDecrease` has a + `TODO`). `/status` `pullsyncRate` is a poor signal on light clusters (stays ~0); a + robust assertion would scrape node `/metrics` (`bee_puller_worker`, + `bee_pullsync_chunks_delivered`) and check the worker set reconfigures and recovers. +- Direction is fixed to increase-then-decrease; `increase`-only / `both` not yet supported. +- No unit tests yet (external `_test` package TBD). + +## Related + +- `docs/radius-check-plan.md` — design, phases, spike findings. +- `.claude/skills/radius-testing/` — operating know-how, `radius-poll.sh`, the + Pushgateway/Prometheus/Grafana metrics stack (`metrics/`). +- Prior art: PR #581 (`radiusdecrease`), PR #591 (`stampexpiry`), `pkg/check/gc`, + `pkg/check/load` (the committedDepth-gated upload primitive), `pkg/check/smoke`. diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 5e1d6de0..28d25c32 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -123,17 +123,24 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return fmt.Errorf("create batch on %s: %w", uploader.Name(), err) } c.logger.WithField("batch_id", batchID).Infof("node %s: using batch", uploader.Name()) - if err := c.driveIncrease(ctx, uploader, nodes, batchID, o); err != nil { + peak := make(map[string]uint8, len(nodes)) + if err := c.driveIncrease(ctx, uploader, nodes, batchID, o, peak); err != nil { return err } - // 3. Let pushsync overshoot drain before snapshotting the peak. - c.logger.Infof("uploads stopped; waiting %s for pushsync to settle", o.SettleWait) - if err := sleepCtx(ctx, o.SettleWait); err != nil { - return err + // 3. Keep raising the per-node high-water `peak` through the settle window. + // The decrease can begin during settle (on an already-stabilised node it is + // near-immediate), so peak must be the max seen — a single post-settle read + // would come back at baseline and make a decrease impossible to detect. + c.logger.Infof("uploads stopped; tracking peak for %s (pushsync settle)", o.SettleWait) + settleDeadline := time.Now().Add(o.SettleWait) + for time.Now().Before(settleDeadline) { + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + c.updatePeak(ctx, nodes, peak) } - peak := c.radiusByNode(ctx, nodes) - c.logger.Infof("peak storageRadius per node after settle: %v", peak) + c.logger.Infof("peak storageRadius per node: %v", peak) // 4. Observe the decrease + pull-sync recovery. return c.observeDecrease(ctx, nodes, peak, o) @@ -168,7 +175,7 @@ func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.Clien } // driveIncrease uploads blobs to the uploader until any observed node reaches TargetRadius. -func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes orchestration.ClientList, batchID string, o Options) error { +func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes orchestration.ClientList, batchID string, o Options, peak map[string]uint8) error { c.logger.Infof("driving increase: %d-byte blobs to %s until storageRadius>=%d (max %d uploads, timeout %s)", o.BlobSize, uploader.Name(), o.TargetRadius, o.MaxUploads, o.IncreaseTimeout) start := time.Now() @@ -190,7 +197,7 @@ func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes o c.logger.Errorf("upload #%d failed: %v", i, err) continue } - mx := c.snapshot(ctx, nodes, "increase") + mx := c.updatePeak(ctx, nodes, peak) c.logger.Infof("increase: upload #%d, max storageRadius=%d (target %d)", i, mx, o.TargetRadius) if mx >= o.TargetRadius { c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) @@ -268,16 +275,27 @@ func (c *Check) snapshot(ctx context.Context, nodes orchestration.ClientList, ph return mx } -func (c *Check) radiusByNode(ctx context.Context, nodes orchestration.ClientList) map[string]uint8 { - out := make(map[string]uint8, len(nodes)) +// updatePeak polls each node, emits metrics, raises the per-node high-water peak, +// logs a line, and returns the current max storageRadius across nodes. +func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8) uint8 { + var mx uint8 for _, n := range nodes { s, err := n.Status(ctx) if err != nil { + c.logger.Debugf("%s: status error: %v", n.Name(), err) continue } - out[n.Name()] = s.StorageRadius + c.emit(n.Name(), s) + if s.StorageRadius > peak[n.Name()] { + peak[n.Name()] = s.StorageRadius + } + if s.StorageRadius > mx { + mx = s.StorageRadius + } + c.logger.Infof("%s: storageRadius=%d (peak %d) reserveSize=%d withinR=%d pullsyncRate=%.4f", + n.Name(), s.StorageRadius, peak[n.Name()], s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate) } - return out + return mx } func (c *Check) emit(node string, s *api.StatusResponse) { From 5b5898d390b1b05cce9dfb3ac802f1fc149886fd Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 18 Jun 2026 13:53:56 +0200 Subject: [PATCH 04/37] feat(check): add --parallel-checks and load/reserve-radius soak modes --- cmd/beekeeper/cmd/check.go | 4 +- config/local.yaml | 26 +++++ docs/radius-check-plan.md | 27 +++++ pkg/check/load/load.go | 40 +++++++- pkg/check/reserveradius/README.md | 109 ++++++++++++-------- pkg/check/reserveradius/metrics.go | 30 +++++- pkg/check/reserveradius/reserveradius.go | 124 +++++++++++++++++++++-- pkg/check/runner.go | 74 +++++++++----- pkg/config/check.go | 4 + 9 files changed, 357 insertions(+), 81 deletions(-) diff --git a/cmd/beekeeper/cmd/check.go b/cmd/beekeeper/cmd/check.go index 6cc818f5..b1636666 100644 --- a/cmd/beekeeper/cmd/check.go +++ b/cmd/beekeeper/cmd/check.go @@ -22,6 +22,7 @@ func (c *command) initCheckCmd() error { optionNameSeed = "seed" optionNameTimeout = "timeout" optionNameMetricsPusherAddress = "metrics-pusher-address" + optionNameParallelChecks = "parallel-checks" ) cmd := &cobra.Command{ @@ -103,7 +104,7 @@ Use --metrics-enabled to collect and push metrics to Prometheus.`, checkRunner := check.NewCheckRunner(checkGlobalConfig, c.config.Checks, cluster, metricsPusher, tracer, c.log) - return checkRunner.Run(ctx, checks) + return checkRunner.Run(ctx, checks, c.globalConfig.GetBool(optionNameParallelChecks)) }) }, PreRunE: c.preRunE, @@ -113,6 +114,7 @@ Use --metrics-enabled to collect and push metrics to Prometheus.`, cmd.Flags().String(optionNameMetricsPusherAddress, "pushgateway.staging.internal", "prometheus metrics pusher address") cmd.Flags().Bool(optionNameCreateCluster, false, "creates cluster before executing checks") cmd.Flags().StringSlice(optionNameChecks, []string{"pingpong"}, "list of checks to execute") + cmd.Flags().Bool(optionNameParallelChecks, false, "run the listed checks concurrently instead of sequentially") cmd.Flags().Bool(optionNameMetricsEnabled, true, "enable metrics") cmd.Flags().Int64(optionNameSeed, -1, "seed, -1 for random") cmd.Flags().Duration(optionNameTimeout, 30*time.Minute, "timeout") diff --git a/config/local.yaml b/config/local.yaml index 7d28bb61..85b8fd0a 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -390,6 +390,32 @@ checks: - bee timeout: 45m type: reserve-radius + ci-reserve-radius-observe: + options: + mode: observe + duration: 30m + recovery-wait: 10m + poll-interval: 15s + upload-groups: + - bee + timeout: 35m + type: reserve-radius + ci-load-soak: + options: + content-size: 1048576 + postage-ttl: 24h + postage-depth: 22 + postage-label: load-soak + duration: 30m + uploader-count: 1 + downloader-count: 0 + max-committed-depth: 1 + committed-depth-check-wait: 15s + decrease-hold: 2m + upload-groups: + - bee + timeout: 35m + type: load ci-soc: options: postage-ttl: 24h diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index 2ebb0a1e..3a8209b7 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -227,6 +227,33 @@ Open follow-ups (not blockers): make `pullsyncRate>0` recovery a **hard** gate o characterised on a data-heavy cluster (Phase 3); add external `_test` package; optional `increase`-only and `both` directions. +## Phase 2.5 — driver/observer split + parallel checks (soak) + +To test radius behavior over long runs (24h–3 days), split the **uploader** (load) from the +**observer** (reserve-radius) and run them concurrently, so the cluster oscillates repeatedly and +we get statistical coverage of resync/recovery rather than a single transition. + +- [x] **`--parallel-checks` flag** (`cmd/beekeeper/cmd/check.go` + `pkg/check/runner.go`) — opt-in; + default stays sequential. When set, `runner.Run` runs the listed checks in goroutines via a + `WaitGroup`; each carries its own timeout and an error does not cancel the others (independent + failures — a monitor blip won't kill a 24h load run). +- [x] **`load` re-arming `decrease-hold`** (`pkg/check/load/load.go`) — new `DecreaseHold` duration + (yaml `decrease-hold`, default 0 = today's behavior). Once `committedDepth` reaches + `max-committed-depth` and then drops, pause uploads for `decrease-hold` before re-filling, then + re-arm → repeated fill→decrease→hold→re-fill cycles. State guarded by a mutex (single-uploader soak). +- [x] **`reserve-radius` `mode: drive | observe`** (`pkg/check/reserveradius/`) — `drive` = the existing + one-shot flow; `observe` = a `Duration`-bounded monitor (no uploads) that records up/down + transitions and asserts recovery (`RecoveryWait`) after each decrease. New metrics + `radius_transitions_total{node,direction}` and `recovery_observed_total{node,result}`. +- [x] Config: `ci-reserve-radius-observe` (mode observe) + `ci-load-soak` (decrease-hold) in `config/local.yaml`. + +Run the soak: `beekeeper check --checks=ci-load-soak,ci-reserve-radius-observe --parallel-checks ...`. +Watch the Grafana `radius-cluster-behavior` / `pullsync-behavior` dashboards for repeated cycles. + +Note: observe-mode recovery is asserted via `/status` `pullsyncRate` (weak on light clusters — can +false-`timeout`); the stronger node-`/metrics` signal (`bee_puller_worker`, `chunks_delivered`) is the +Phase-3 follow-up. + ## Phase 3 — Real ephemeral cluster, no patch - [ ] Run the same check against an unpatched ephemeral k8s cluster with realistic diff --git a/pkg/check/load/load.go b/pkg/check/load/load.go index 71e29b2c..3228f7f3 100644 --- a/pkg/check/load/load.go +++ b/pkg/check/load/load.go @@ -37,6 +37,7 @@ type Options struct { DownloadGroups []string MaxCommittedDepth uint8 CommittedDepthCheckWait time.Duration + DecreaseHold time.Duration // once target reached and committedDepth drops, pause uploads this long before re-filling (0 = disabled) IterationWait time.Duration } @@ -57,6 +58,7 @@ func NewDefaultOptions() Options { DownloadGroups: []string{}, MaxCommittedDepth: 2, CommittedDepthCheckWait: 5 * time.Minute, + DecreaseHold: 0, IterationWait: 5 * time.Minute, } } @@ -70,6 +72,9 @@ var _ beekeeper.Action = (*Check)(nil) type Check struct { metrics metrics logger logging.Logger + + mu sync.Mutex + reachedTarget bool // set once committedDepth reaches max; consumed on the following decrease } func NewCheck(log logging.Logger) beekeeper.Action { @@ -185,7 +190,7 @@ func (c *Check) run(ctx context.Context, cluster orchestration.Cluster, o Option default: } - if !c.checkCommittedDepth(ctx, uploader, o.MaxCommittedDepth, o.CommittedDepthCheckWait) { + if !c.checkCommittedDepth(ctx, uploader, o.MaxCommittedDepth, o.CommittedDepthCheckWait, o.DecreaseHold) { return } @@ -299,7 +304,7 @@ func (c *Check) run(ctx context.Context, cluster orchestration.Cluster, o Option return nil } -func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, maxDepth uint8, wait time.Duration) bool { +func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, maxDepth uint8, wait, decreaseHold time.Duration) bool { for { statusResp, err := client.Status(ctx) if err != nil { @@ -308,8 +313,21 @@ func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, max } if statusResp.CommittedDepth < maxDepth { + // Below target. If committedDepth just dropped from the target and a + // decrease-hold is set, pause uploads for that window before re-filling + // so the radius decrease + resync can play out without interference. + if decreaseHold > 0 && c.consumeReachedTarget() { + c.logger.Infof("committedDepth dropped to %d (max %d); holding uploads for %v before re-filling", statusResp.CommittedDepth, maxDepth, decreaseHold) + select { + case <-ctx.Done(): + return false + case <-time.After(decreaseHold): + } + } return true } + + c.markReachedTarget() c.logger.Infof("waiting %v for CommittedDepth to decrease. Current: %d, Max: %d", wait, statusResp.CommittedDepth, maxDepth) select { @@ -321,6 +339,24 @@ func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, max } } +func (c *Check) markReachedTarget() { + c.mu.Lock() + c.reachedTarget = true + c.mu.Unlock() +} + +// consumeReachedTarget reports whether the target was reached since the last call, +// resetting the flag so the decrease-hold fires once per fill→drop cycle (re-arming). +func (c *Check) consumeReachedTarget() bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.reachedTarget { + c.reachedTarget = false + return true + } + return false +} + func (c *Check) Report() []prometheus.Collector { return c.metrics.Report() } diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index eecbf3b8..3eaebda7 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -1,10 +1,18 @@ # reserve-radius check -Stages a **storage-radius change** across a Bee cluster and verifies the node -recovers afterwards. It drives the radius up by uploading, then watches for the -radius to come back down and for pull-sync to react — catching regressions in the -puller's reaction to a radius change (the `manage()` / `disconnectPeer()` path that -caused the liveness bug in beekeeper PR #581). +Exercises a **storage-radius change** across a Bee cluster and verifies the node +recovers afterwards — catching regressions in the puller's reaction to a radius +change (the `manage()` / `disconnectPeer()` path that caused the liveness bug in +beekeeper PR #581). + +It has two modes (`Options.Mode`): + +- **`drive`** (default) — a one-shot run: upload to force the radius up, then watch + it come back down and assert pull-sync recovers. +- **`observe`** — a long-running monitor: does **not** upload; watches radius + transitions for `Duration` while something else drives the cluster (typically the + `load` check running in parallel via `--parallel-checks`), recording every up/down + transition and asserting recovery after each decrease. This is the soak mode. > Keep this file in sync with the code. If you change `Options`, the `Run` flow, the > emitted metrics, the registration, or the bee-patch requirement, update the matching @@ -12,7 +20,9 @@ caused the liveness bug in beekeeper PR #581). ## What it does (`Run` flow) -`reserveradius.go`, in order: +`Run` dispatches on `Mode`. + +**`drive`** (`runDrive`): 1. **`waitForWarmupDone`** — block until every observed node reports `isWarmingUp == false`. The reserve worker's *decrease* loop is gated on @@ -22,35 +32,42 @@ caused the liveness bug in beekeeper PR #581). `BlobSize` random blobs to it until any observed node's `storageRadius` reaches `TargetRadius` (or `MaxUploads` / `IncreaseTimeout`). Tracks a **per-node high-water `peak`**. -4. **settle** — keep polling for `SettleWait`, still raising `peak`. This matters: - on an already-stabilized node the decrease can begin *during* settle, so `peak` - must be the max seen, not a single post-settle read (otherwise it reads back at - baseline and no decrease is ever detectable). +4. **settle** — keep polling for `SettleWait`, still raising `peak` (on an + already-stabilized node the decrease can begin *during* settle, so `peak` must be + the max seen, not a single post-settle read). 5. **`observeDecrease`** — poll until any node's `storageRadius` falls below its - `peak` (success), watching `pullsyncRate` for recovery along the way. If no - decrease occurs within `DecreaseTimeout`, fail with a message pointing at the - PR #581 puller stall. + `peak`, watching `pullsyncRate` for recovery. No decrease within `DecreaseTimeout` + fails with a message pointing at the PR #581 puller stall. + +**`observe`** (`runObserve`): `waitForWarmupDone`, snapshot a baseline, then for +`Duration` poll every `PollInterval`, comparing each node's `storageRadius` to its +last value. Every change is recorded as an up/down transition (metric + log); after a +**down** transition, wait up to `RecoveryWait` for `pullsyncRate > 0`. A decrease that +never recovers is counted and the check fails at the end listing the count. Never uploads. ## Options Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in `pkg/config/check.go` under the `reserve-radius` entry. -| field | yaml | default | purpose | -| --- | --- | --- | --- | -| `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | seed for `random.PseudoGenerator` → shuffled node pick | -| `PostageTTL` | `postage-ttl` | `24h` | batch TTL (use TTL, not a raw amount) | -| `PostageDepth` | `postage-depth` | `22` | batch depth | -| `PostageLabel` | `postage-label` | `reserve-radius` | batch label | -| `UploadGroups` | `upload-groups` | `[bee]` | node groups to upload to / observe (empty = all full nodes) | -| `BlobSize` | `blob-size` | `1048576` (1 MiB) | bytes per upload | -| `MaxUploads` | `max-uploads` | `60` | cap on uploads in the increase phase | -| `TargetRadius` | `target-radius` | `1` | storageRadius to reach before stopping uploads | -| `WarmupWait` | `warmup-wait` | `15m` | max wait for nodes to finish warmup | -| `IncreaseTimeout` | `increase-timeout` | `5m` | max time to reach `TargetRadius` | -| `SettleWait` | `settle-wait` | `1m` | post-upload window (pushsync drain + peak tracking) | -| `DecreaseTimeout` | `decrease-timeout` | `20m` | max time to observe a decrease | -| `PollInterval` | `poll-interval` | `15s` | poll cadence | +| field | yaml | default | mode | purpose | +| --- | --- | --- | --- | --- | +| `Mode` | `mode` | `drive` | both | `drive` (upload to force a change) or `observe` (monitor only) | +| `Duration` | `duration` | `12h` | observe | total monitor run length | +| `RecoveryWait` | `recovery-wait` | `5m` | observe | max wait for pull-sync recovery after each decrease | +| `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | both | seed for `random.PseudoGenerator` → shuffled node pick | +| `UploadGroups` | `upload-groups` | `[bee]` | both | node groups to observe (and, in drive, upload to) | +| `PollInterval` | `poll-interval` | `15s` | both | poll cadence | +| `PostageTTL` | `postage-ttl` | `24h` | drive | batch TTL (use TTL, not a raw amount) | +| `PostageDepth` | `postage-depth` | `22` | drive | batch depth | +| `PostageLabel` | `postage-label` | `reserve-radius` | drive | batch label | +| `BlobSize` | `blob-size` | `1048576` (1 MiB) | drive | bytes per upload | +| `MaxUploads` | `max-uploads` | `60` | drive | cap on uploads in the increase phase | +| `TargetRadius` | `target-radius` | `1` | drive | storageRadius to reach before stopping uploads | +| `WarmupWait` | `warmup-wait` | `15m` | both | max wait for nodes to finish warmup | +| `IncreaseTimeout` | `increase-timeout` | `5m` | drive | max time to reach `TargetRadius` | +| `SettleWait` | `settle-wait` | `1m` | drive | post-upload window (pushsync drain + peak tracking) | +| `DecreaseTimeout` | `decrease-timeout` | `20m` | drive | max time to observe a decrease | ## Metrics (`metrics.go`) @@ -61,8 +78,9 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - `…_storage_radius{node}` — gauge, storageRadius per node - `…_reserve_size{node}` — gauge, reserve size (chunks) per node - `…_pullsync_rate{node}` — gauge, `/status` pullsyncRate per node -- `…_time_to_increase_seconds` — gauge, first-upload → `TargetRadius` -- `…_time_to_decrease_seconds` — gauge, uploads-stopped → first observed decrease +- `…_time_to_increase_seconds` / `…_time_to_decrease_seconds` — gauges (drive mode) +- `…_radius_transitions_total{node,direction}` — counter, observed up/down transitions (observe mode) +- `…_recovery_observed_total{node,result}` — counter, recovery outcome `recovered`/`timeout` (observe mode) ## Requirements @@ -76,14 +94,21 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem ## Running it -Against a patched local cluster (`local-dns`), via the `ci-reserve-radius` config -entry (timeout 45m): +Against a patched local cluster (`local-dns`): ```sh +# drive (one-shot): force a change and assert recovery ./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius --log-verbosity=info -# add --metrics-enabled=true --metrics-pusher-address=localhost:9091 to push metrics + +# soak: load oscillates the radius, reserve-radius observes it, both run concurrently +./dist/beekeeper check --cluster-name=local-dns \ + --checks=ci-load-soak,ci-reserve-radius-observe --parallel-checks \ + --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info ``` +`--parallel-checks` runs the listed checks in goroutines instead of sequentially; +checks fail independently (a monitor failure does not stop the load run). + ## Observed behavior (local, patched cluster) - **Increase is fast** — ~1 MiB moves `storageRadius` 0→1 in seconds. @@ -94,21 +119,23 @@ entry (timeout 45m): - **Pull-sync on a radius change is a puller reconfiguration**: `bee_puller_worker` contracts on increase / expands on decrease, with a `bee_puller_worker_errors` cancel-storm. Actual resync volume (`bee_pullsync_chunks_delivered`) is ~0 on a - near-empty cluster — meaningful resync needs data (Phase 3 / real cluster). + near-empty cluster — meaningful resync needs data (real cluster). ## Known limitations / follow-ups -- Pull-sync recovery is **observed and logged, not asserted** (`observeDecrease` has a - `TODO`). `/status` `pullsyncRate` is a poor signal on light clusters (stays ~0); a - robust assertion would scrape node `/metrics` (`bee_puller_worker`, - `bee_pullsync_chunks_delivered`) and check the worker set reconfigures and recovers. -- Direction is fixed to increase-then-decrease; `increase`-only / `both` not yet supported. +- Recovery is asserted via `/status` `pullsyncRate`, which is a poor signal on light + clusters (stays ~0, so observe mode can false-`timeout`). A stronger signal is + scraping node `/metrics` (`bee_puller_worker`, `bee_pullsync_chunks_delivered`) and + checking the worker set reconfigures and recovers — not yet wired in. +- Direction is fixed to increase-then-decrease (drive) / observe-both (observe); no + `increase`-only assertion mode yet. - No unit tests yet (external `_test` package TBD). ## Related -- `docs/radius-check-plan.md` — design, phases, spike findings. +- `docs/radius-check-plan.md` — design, phases, spike findings, the driver/observer split. - `.claude/skills/radius-testing/` — operating know-how, `radius-poll.sh`, the Pushgateway/Prometheus/Grafana metrics stack (`metrics/`). - Prior art: PR #581 (`radiusdecrease`), PR #591 (`stampexpiry`), `pkg/check/gc`, - `pkg/check/load` (the committedDepth-gated upload primitive), `pkg/check/smoke`. + `pkg/check/load` (the committedDepth-gated upload primitive, reused as the soak driver), + `pkg/check/smoke`. diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go index 0d69df40..5f27821a 100644 --- a/pkg/check/reserveradius/metrics.go +++ b/pkg/check/reserveradius/metrics.go @@ -6,11 +6,13 @@ import ( ) type metrics struct { - StorageRadius *prometheus.GaugeVec - ReserveSize *prometheus.GaugeVec - PullsyncRate *prometheus.GaugeVec - TimeToIncrease prometheus.Gauge - TimeToDecrease prometheus.Gauge + StorageRadius *prometheus.GaugeVec + ReserveSize *prometheus.GaugeVec + PullsyncRate *prometheus.GaugeVec + TimeToIncrease prometheus.Gauge + TimeToDecrease prometheus.Gauge + RadiusTransitions *prometheus.CounterVec + RecoveryObserved *prometheus.CounterVec } func newMetrics(subsystem string) metrics { @@ -58,6 +60,24 @@ func newMetrics(subsystem string) metrics { Help: "Seconds from uploads-stopped to the first observed radius decrease.", }, ), + RadiusTransitions: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "radius_transitions_total", + Help: "Observed storage-radius transitions, by node and direction (up/down).", + }, + []string{"node", "direction"}, + ), + RecoveryObserved: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "recovery_observed_total", + Help: "Pull-sync recovery outcome after a radius decrease, by node and result (recovered/timeout).", + }, + []string{"node", "result"}, + ), } } diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 28d25c32..d533c2e4 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -34,6 +34,8 @@ import ( // Options represents reserve-radius check options. type Options struct { + Mode string // "drive" (upload to force a change) or "observe" (monitor a change driven externally) + Duration time.Duration // observe mode: total monitor run length RndSeed int64 PostageTTL time.Duration PostageDepth uint64 @@ -46,12 +48,15 @@ type Options struct { IncreaseTimeout time.Duration // max time to reach TargetRadius SettleWait time.Duration // wait after uploads for pushsync overshoot to drain DecreaseTimeout time.Duration // max time to observe a decrease after uploads stop + RecoveryWait time.Duration // observe mode: max wait for pull-sync recovery after each decrease PollInterval time.Duration } // NewDefaultOptions returns new default options. func NewDefaultOptions() Options { return Options{ + Mode: ModeDrive, + Duration: 12 * time.Hour, RndSeed: time.Now().UnixNano(), PostageTTL: 24 * time.Hour, PostageDepth: 22, @@ -64,6 +69,7 @@ func NewDefaultOptions() Options { IncreaseTimeout: 5 * time.Minute, SettleWait: time.Minute, DecreaseTimeout: 20 * time.Minute, + RecoveryWait: 5 * time.Minute, PollInterval: 15 * time.Second, } } @@ -85,31 +91,58 @@ func NewCheck(log logging.Logger) beekeeper.Action { } } -// Run stages a radius increase, then observes the decrease + pull-sync recovery. +// Mode values for Options.Mode. +const ( + ModeDrive = "drive" // upload to force a radius change, then observe the decrease + ModeObserve = "observe" // monitor radius changes driven externally (e.g. by the load check) +) + +// Run dispatches on Mode: drive (force a change and observe it) or observe +// (monitor changes driven externally, e.g. by a parallel load check). func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { o, ok := opts.(Options) if !ok { return errors.New("invalid options type") } - if o.TargetRadius == 0 { - return errors.New("target-radius must be > 0") + switch o.Mode { + case "", ModeDrive: + return c.runDrive(ctx, cluster, o) + case ModeObserve: + return c.runObserve(ctx, cluster, o) + default: + return fmt.Errorf("invalid mode %q (want %q or %q)", o.Mode, ModeDrive, ModeObserve) } +} +// selectNodes returns the observed full-node clients (shuffled, optionally filtered to UploadGroups). +func (c *Check) selectNodes(ctx context.Context, cluster orchestration.Cluster, o Options) (orchestration.ClientList, error) { c.logger.Infof("random seed: %d", o.RndSeed) rnd := random.PseudoGenerator(o.RndSeed) fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) if err != nil { - return fmt.Errorf("get shuffled full node clients: %w", err) + return nil, fmt.Errorf("get shuffled full node clients: %w", err) } nodes := fullNodeClients if len(o.UploadGroups) > 0 { nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) } if len(nodes) < 1 { - return fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(nodes)) + return nil, fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(nodes)) + } + return nodes, nil +} + +// runDrive uploads to force a radius increase, then observes the decrease + recovery. +func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o Options) error { + if o.TargetRadius == 0 { + return errors.New("target-radius must be > 0") + } + nodes, err := c.selectNodes(ctx, cluster, o) + if err != nil { + return err } uploader := nodes[0] // a random node, since the list is shuffled - c.logger.Infof("uploader: %s, observing %d node(s)", uploader.Name(), len(nodes)) + c.logger.Infof("mode=drive uploader: %s, observing %d node(s)", uploader.Name(), len(nodes)) // 1. Wait for warmup/stabilization — the decrease loop is gated on it. if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { @@ -146,6 +179,85 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return c.observeDecrease(ctx, nodes, peak, o) } +// runObserve monitors radius transitions for Duration without uploading; the radius +// is expected to be driven externally (e.g. by a parallel load check). It records +// every up/down transition, and after each decrease waits up to RecoveryWait for +// pull-sync to resume — decreases that never recover are reported at the end. +func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o Options) error { + nodes, err := c.selectNodes(ctx, cluster, o) + if err != nil { + return err + } + c.logger.Infof("mode=observe monitoring %d node(s) for %s (no uploads; drive the radius externally, e.g. the load check)", len(nodes), o.Duration) + + if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { + return err + } + + last := make(map[string]uint8, len(nodes)) // last seen storageRadius per node + recoverBy := make(map[string]time.Time, len(nodes)) // node -> deadline to see recovery after a decrease + unrecovered := 0 + + // seed baseline radii + for _, n := range nodes { + if s, err := n.Status(ctx); err == nil { + last[n.Name()] = s.StorageRadius + c.emit(n.Name(), s) + } + } + c.logger.Info("observe: baseline captured; watching for radius transitions") + + deadline := time.Now().Add(o.Duration) + for time.Now().Before(deadline) { + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err // parent context cancelled (e.g. check timeout) + } + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + continue + } + name := n.Name() + c.emit(name, s) + cur := s.StorageRadius + + if prev, seen := last[name]; seen && cur != prev { + dir := "up" + if cur < prev { + dir = "down" + } + c.metrics.RadiusTransitions.WithLabelValues(name, dir).Inc() + c.logger.Infof("observe: %s storageRadius %d -> %d (%s) reserveSize=%d pullsyncRate=%.4f", name, prev, cur, dir, s.ReserveSize, s.PullsyncRate) + if dir == "down" { + recoverBy[name] = time.Now().Add(o.RecoveryWait) + } + } + last[name] = cur + + // recovery tracking after a decrease + if rd, waiting := recoverBy[name]; waiting { + switch { + case s.PullsyncRate > 0: + delete(recoverBy, name) + c.metrics.RecoveryObserved.WithLabelValues(name, "recovered").Inc() + c.logger.Infof("observe: %s pull-sync recovered (pullsyncRate=%.4f)", name, s.PullsyncRate) + case time.Now().After(rd): + delete(recoverBy, name) + unrecovered++ + c.metrics.RecoveryObserved.WithLabelValues(name, "timeout").Inc() + c.logger.Warningf("observe: %s no pull-sync recovery within %s after decrease (cf. PR #581)", name, o.RecoveryWait) + } + } + } + } + + if unrecovered > 0 { + return fmt.Errorf("observe: %d radius decrease(s) showed no pull-sync recovery within %s", unrecovered, o.RecoveryWait) + } + c.logger.Infof("observe: completed %s monitor with all decreases recovered", o.Duration) + return nil +} + // waitForWarmupDone blocks until every observed node reports isWarmingUp=false. func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.ClientList, o Options) error { c.logger.Infof("waiting up to %s for nodes to finish warmup/stabilization", o.WarmupWait) diff --git a/pkg/check/runner.go b/pkg/check/runner.go index 71c89e2b..33a92865 100644 --- a/pkg/check/runner.go +++ b/pkg/check/runner.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "strings" + "sync" "time" "github.com/ethersphere/beekeeper/pkg/beekeeper" @@ -46,7 +47,7 @@ func NewCheckRunner( } } -func (c *CheckRunner) Run(ctx context.Context, checks []string) error { +func (c *CheckRunner) Run(ctx context.Context, checks []string, parallel bool) error { if len(checks) == 0 { return nil } @@ -91,35 +92,30 @@ func (c *CheckRunner) Run(ctx context.Context, checks []string) error { }) } - checkResults := make([]checkResult, 0, len(validatedChecks)) - hasFailures := false - - // run checks - for _, check := range validatedChecks { - c.logger.WithFields(map[string]any{ - "type": check.typeName, - "options": fmt.Sprintf("%+v", check.options), - }).Infof("running check: %s", check.name) + // run checks — each writes its own index, so the slice needs no locking + checkResults := make([]checkResult, len(validatedChecks)) + if parallel { + c.logger.WithField("count", len(validatedChecks)).Info("running checks concurrently") + var wg sync.WaitGroup + for i, check := range validatedChecks { + wg.Go(func() { + checkResults[i] = c.runOne(ctx, check) + }) + } + wg.Wait() + } else { + for i, check := range validatedChecks { + checkResults[i] = c.runOne(ctx, check) + } + } - err := check.Run(ctx, c.cluster) - if err != nil { + hasFailures := false + for _, r := range checkResults { + if r.err != nil { hasFailures = true - c.logger.WithFields(map[string]any{ - "type": check.typeName, - "error": err, - }).Errorf("'%s' check failed", check.name) - } else { - c.logger.WithField("type", check.typeName).Infof("'%s' check completed successfully", check.name) + break } - - // append check result - checkResults = append(checkResults, checkResult{ - check: check.name, - err: err, - timestamp: time.Now(), - }) } - if hasFailures { return formatErrorReport(checkResults) } @@ -128,6 +124,32 @@ func (c *CheckRunner) Run(ctx context.Context, checks []string) error { return nil } +// runOne runs a single prepared check, logs the outcome, and returns its result. +// Each check carries its own timeout (checkRun.Run), and an error does not cancel +// the shared parent context, so concurrent checks fail independently. +func (c *CheckRunner) runOne(ctx context.Context, check checkRun) checkResult { + c.logger.WithFields(map[string]any{ + "type": check.typeName, + "options": fmt.Sprintf("%+v", check.options), + }).Infof("running check: %s", check.name) + + err := check.Run(ctx, c.cluster) + if err != nil { + c.logger.WithFields(map[string]any{ + "type": check.typeName, + "error": err, + }).Errorf("'%s' check failed", check.name) + } else { + c.logger.WithField("type", check.typeName).Infof("'%s' check completed successfully", check.name) + } + + return checkResult{ + check: check.name, + err: err, + timestamp: time.Now(), + } +} + type checkRun struct { name string typeName string diff --git a/pkg/config/check.go b/pkg/config/check.go index e676bcc5..be7d7445 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -482,6 +482,7 @@ var Checks = map[string]CheckType{ DownloadGroups *[]string `yaml:"download-groups"` MaxCommittedDepth *uint8 `yaml:"max-committed-depth"` CommittedDepthCheckWait *time.Duration `yaml:"committed-depth-check-wait"` + DecreaseHold *time.Duration `yaml:"decrease-hold"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -500,6 +501,8 @@ var Checks = map[string]CheckType{ NewAction: reserveradius.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { + Mode *string `yaml:"mode"` + Duration *time.Duration `yaml:"duration"` RndSeed *int64 `yaml:"rnd-seed"` PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` @@ -512,6 +515,7 @@ var Checks = map[string]CheckType{ IncreaseTimeout *time.Duration `yaml:"increase-timeout"` SettleWait *time.Duration `yaml:"settle-wait"` DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + RecoveryWait *time.Duration `yaml:"recovery-wait"` PollInterval *time.Duration `yaml:"poll-interval"` }) if err := check.Options.Decode(checkOpts); err != nil { From cb5d276d0aa120dea04da8f701e60297dd372129 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 18 Jun 2026 14:06:57 +0200 Subject: [PATCH 05/37] feat(check): add --parallel for concurrent check execution --- cmd/beekeeper/cmd/check.go | 6 +++--- docs/radius-check-plan.md | 4 ++-- pkg/check/reserveradius/README.md | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/beekeeper/cmd/check.go b/cmd/beekeeper/cmd/check.go index b1636666..d91df1aa 100644 --- a/cmd/beekeeper/cmd/check.go +++ b/cmd/beekeeper/cmd/check.go @@ -22,7 +22,7 @@ func (c *command) initCheckCmd() error { optionNameSeed = "seed" optionNameTimeout = "timeout" optionNameMetricsPusherAddress = "metrics-pusher-address" - optionNameParallelChecks = "parallel-checks" + optionNameParallel = "parallel" ) cmd := &cobra.Command{ @@ -104,7 +104,7 @@ Use --metrics-enabled to collect and push metrics to Prometheus.`, checkRunner := check.NewCheckRunner(checkGlobalConfig, c.config.Checks, cluster, metricsPusher, tracer, c.log) - return checkRunner.Run(ctx, checks, c.globalConfig.GetBool(optionNameParallelChecks)) + return checkRunner.Run(ctx, checks, c.globalConfig.GetBool(optionNameParallel)) }) }, PreRunE: c.preRunE, @@ -114,7 +114,7 @@ Use --metrics-enabled to collect and push metrics to Prometheus.`, cmd.Flags().String(optionNameMetricsPusherAddress, "pushgateway.staging.internal", "prometheus metrics pusher address") cmd.Flags().Bool(optionNameCreateCluster, false, "creates cluster before executing checks") cmd.Flags().StringSlice(optionNameChecks, []string{"pingpong"}, "list of checks to execute") - cmd.Flags().Bool(optionNameParallelChecks, false, "run the listed checks concurrently instead of sequentially") + cmd.Flags().Bool(optionNameParallel, false, "run the --checks list concurrently instead of sequentially (default sequential)") cmd.Flags().Bool(optionNameMetricsEnabled, true, "enable metrics") cmd.Flags().Int64(optionNameSeed, -1, "seed, -1 for random") cmd.Flags().Duration(optionNameTimeout, 30*time.Minute, "timeout") diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index 3a8209b7..7a0085be 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -233,7 +233,7 @@ To test radius behavior over long runs (24h–3 days), split the **uploader** (l **observer** (reserve-radius) and run them concurrently, so the cluster oscillates repeatedly and we get statistical coverage of resync/recovery rather than a single transition. -- [x] **`--parallel-checks` flag** (`cmd/beekeeper/cmd/check.go` + `pkg/check/runner.go`) — opt-in; +- [x] **`--parallel` flag** (`cmd/beekeeper/cmd/check.go` + `pkg/check/runner.go`) — opt-in; default stays sequential. When set, `runner.Run` runs the listed checks in goroutines via a `WaitGroup`; each carries its own timeout and an error does not cancel the others (independent failures — a monitor blip won't kill a 24h load run). @@ -247,7 +247,7 @@ we get statistical coverage of resync/recovery rather than a single transition. `radius_transitions_total{node,direction}` and `recovery_observed_total{node,result}`. - [x] Config: `ci-reserve-radius-observe` (mode observe) + `ci-load-soak` (decrease-hold) in `config/local.yaml`. -Run the soak: `beekeeper check --checks=ci-load-soak,ci-reserve-radius-observe --parallel-checks ...`. +Run the soak: `beekeeper check --checks=ci-load-soak,ci-reserve-radius-observe --parallel ...`. Watch the Grafana `radius-cluster-behavior` / `pullsync-behavior` dashboards for repeated cycles. Note: observe-mode recovery is asserted via `/status` `pullsyncRate` (weak on light clusters — can diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 3eaebda7..ec3dd5e4 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -11,7 +11,7 @@ It has two modes (`Options.Mode`): it come back down and assert pull-sync recovers. - **`observe`** — a long-running monitor: does **not** upload; watches radius transitions for `Duration` while something else drives the cluster (typically the - `load` check running in parallel via `--parallel-checks`), recording every up/down + `load` check running in parallel via `--parallel`), recording every up/down transition and asserting recovery after each decrease. This is the soak mode. > Keep this file in sync with the code. If you change `Options`, the `Run` flow, the @@ -102,11 +102,11 @@ Against a patched local cluster (`local-dns`): # soak: load oscillates the radius, reserve-radius observes it, both run concurrently ./dist/beekeeper check --cluster-name=local-dns \ - --checks=ci-load-soak,ci-reserve-radius-observe --parallel-checks \ + --checks=ci-load-soak,ci-reserve-radius-observe --parallel \ --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info ``` -`--parallel-checks` runs the listed checks in goroutines instead of sequentially; +`--parallel` runs the listed checks in goroutines instead of sequentially; checks fail independently (a monitor failure does not stop the load run). ## Observed behavior (local, patched cluster) From fe88ae9f45487f5cc73d3ece0dcd8f871df67da7 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 25 Jun 2026 10:36:28 +0200 Subject: [PATCH 06/37] feat(check): assert pull-sync halt indicators in reserve-radius observe mode --- docs/radius-check-plan.md | 44 +++++++++ pkg/bee/api/api.go | 34 ++++--- pkg/bee/api/redistribution.go | 35 +++++++ pkg/bee/client.go | 5 + pkg/check/reserveradius/README.md | 36 +++++-- pkg/check/reserveradius/metrics.go | 38 +++++-- pkg/check/reserveradius/reserveradius.go | 121 +++++++++++++++++------ 7 files changed, 250 insertions(+), 63 deletions(-) create mode 100644 pkg/bee/api/redistribution.go diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index 7a0085be..93929022 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -18,6 +18,50 @@ end product is a beekeeper check (`pkg/check/reserveradius`) plus the metrics it emits. Operating know-how lives in the `radius-testing` skill; this doc is the design and the build checklist. +## Pull-sync validation (the real target) + +The narrow goal is "force a radius decrease, confirm the puller doesn't deadlock" +(PR #581). The **broader goal** — why @sig/@marios want this harness — is to make a +**reproducible test of the October network halt** and to validate the pull-sync +redesign (SWIP-25 / `pullsync-optimal-design`). + +**The October mechanism.** A large operator went offline → network commitment dropped +→ **storage radius decreased**, which is a neighbourhood **merge**: neighbourhood size +`k` transiently jumps to ~8, so every node must suddenly re-sync a much larger reserve +from many more peers. Today's pull-sync is "correctness through redundancy" — each of +`k` peers runs an independent session with **up to k× redundant deliveries** per chunk, +no shared dedup, no explicit failover-with-exclude; on a depth change bins get +"cancelled and restarted repeatedly, causing unbounded delay," and stalls livelock. +Result: incomplete reserves → wrong `ReserveSample` → nodes **freeze, skip redistribution +rounds, go unresponsive** (→ slashing risk). The avalanche. + +**What the test must show** on a staged radius **decrease**: + +- **Completeness** — each node's reserve catches up (`reserveSizeWithinRadius` recovers). +- **Liveness / no halt** — nodes don't freeze, redistribution `round` keeps advancing, + `isFullySynced` returns within a bound. +- **Bounded convergence** — resync finishes in bounded time; no stuck `STALLED` bins. +- **(SWIP-25 efficiency)** — redundant deliveries drop ~k×→1× (`offered/delivered` ratio). +- Ideally **A/B**: current bee reproduces the halt; fixed bee doesn't. + +**What the check now measures toward this** (implemented): observe mode reads +`/status` **and `/redistributionstate`** and asserts the halt indicators — un-recovered +decreases (prefers `isFullySynced`, falls back to `pullsyncRate`) and **freeze episodes** +(`isFrozen`). New metrics: `fully_synced`, `frozen`, `redistribution_round`, +`reserve_within_radius`, `last_sample_duration_seconds`, `time_to_fully_synced_seconds`. +The `radius-cluster-behavior` dashboard has a **Halt indicators** row (fullySynced/frozen, +round-advance, `is_playing_errors` rate, the offered/delivered redundancy ratio). + +**Still needs a sync with @marios + scale** (not solo-buildable): + +- Exact pass/fail thresholds (the `isFullySynced` bound after a decrease, acceptable + convergence time, the redundancy-ratio target). +- The per-`(peer,bin)` `STALLED` / `MaxStallsPerBin` metrics — only the **fixed** + pull-sync (@sig's PR) exposes those; the check would then scrape them. +- A **~20-node ephemeral cluster** — the merge (`k→8`) needs scale; 3 local nodes can't + form a realistic neighbourhood. This is ljubisa's stated end goal. +- An **A/B harness** (same scenario, current vs fixed bee, diff the signals). + ## Why this is hard (and what prior tryouts taught us) - A stock node's reserve is far too large to overflow in CI-time, so a radius diff --git a/pkg/bee/api/api.go b/pkg/bee/api/api.go index f943fc33..5450b17f 100644 --- a/pkg/bee/api/api.go +++ b/pkg/bee/api/api.go @@ -42,22 +42,23 @@ type Client struct { service service // Reuse a single struct instead of allocating one for each service on the heap. // Services that API provides. - Act *ActService - Bytes *BytesService - Chunks *ChunksService - Dirs *DirsService - Feed *FeedService - Files *FilesService - Node *NodeService - PingPong *PingPongService - Pinning *PinningService - Postage *PostageService - PSS *PSSService - SOC *SOCService - Stake *StakingService - Status *StatusService - Stewardship *StewardshipService - Tags *TagsService + Act *ActService + Bytes *BytesService + Chunks *ChunksService + Dirs *DirsService + Feed *FeedService + Files *FilesService + Node *NodeService + PingPong *PingPongService + Pinning *PinningService + Postage *PostageService + PSS *PSSService + Redistribution *RedistributionService + SOC *SOCService + Stake *StakingService + Status *StatusService + Stewardship *StewardshipService + Tags *TagsService } // NewClient constructs a new Client. @@ -103,6 +104,7 @@ func newClient(apiURL *url.URL, httpClient *http.Client) (c *Client) { c.Pinning = (*PinningService)(&c.service) c.Postage = (*PostageService)(&c.service) c.PSS = (*PSSService)(&c.service) + c.Redistribution = (*RedistributionService)(&c.service) c.SOC = (*SOCService)(&c.service) c.Stake = (*StakingService)(&c.service) c.Status = (*StatusService)(&c.service) diff --git a/pkg/bee/api/redistribution.go b/pkg/bee/api/redistribution.go new file mode 100644 index 00000000..de7f6417 --- /dev/null +++ b/pkg/bee/api/redistribution.go @@ -0,0 +1,35 @@ +package api + +import ( + "context" + "net/http" +) + +type RedistributionService service + +// RedistributionState is the subset of the node's /redistributionstate response +// relevant to radius / pull-sync validation: whether the node is frozen, fully +// synced, healthy, and how far the redistribution round has progressed. These are +// the signals that distinguish a recovering node from a halted one after a radius +// change (a node that cannot reach fullySynced or that freezes skips rounds). +type RedistributionState struct { + IsFrozen bool `json:"isFrozen"` + IsFullySynced bool `json:"isFullySynced"` + IsHealthy bool `json:"isHealthy"` + HasSufficientFunds bool `json:"hasSufficientFunds"` + Phase string `json:"phase"` + Round uint64 `json:"round"` + LastWonRound uint64 `json:"lastWonRound"` + LastPlayedRound uint64 `json:"lastPlayedRound"` + LastFrozenRound uint64 `json:"lastFrozenRound"` + LastSelectedRound uint64 `json:"lastSelectedRound"` + LastSampleDurationSeconds float64 `json:"lastSampleDurationSeconds"` + Block uint64 `json:"block"` +} + +// RedistributionState returns the node's redistribution-game state. This endpoint +// is full-mode only; it errors on nodes without the redistribution agent enabled. +func (s *RedistributionService) RedistributionState(ctx context.Context) (resp *RedistributionState, err error) { + err = s.client.requestJSON(ctx, http.MethodGet, "/redistributionstate", nil, &resp) + return resp, err +} diff --git a/pkg/bee/client.go b/pkg/bee/client.go index 711ffeb6..a39b1121 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -1089,3 +1089,8 @@ func (c *Client) FindFeedUpdate(ctx context.Context, signer crypto.Signer, topic func (c *Client) Status(ctx context.Context) (*api.StatusResponse, error) { return c.api.Status.Status(ctx) } + +// RedistributionState returns the node's redistribution-game state (full-mode only). +func (c *Client) RedistributionState(ctx context.Context) (*api.RedistributionState, error) { + return c.api.Redistribution.RedistributionState(ctx) +} diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index ec3dd5e4..b2a925e4 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -39,11 +39,20 @@ It has two modes (`Options.Mode`): `peak`, watching `pullsyncRate` for recovery. No decrease within `DecreaseTimeout` fails with a message pointing at the PR #581 puller stall. -**`observe`** (`runObserve`): `waitForWarmupDone`, snapshot a baseline, then for -`Duration` poll every `PollInterval`, comparing each node's `storageRadius` to its -last value. Every change is recorded as an up/down transition (metric + log); after a -**down** transition, wait up to `RecoveryWait` for `pullsyncRate > 0`. A decrease that -never recovers is counted and the check fails at the end listing the count. Never uploads. +**`observe`** (`runObserve`): `waitForWarmupDone`, then for `Duration` poll every +`PollInterval`, reading both `/status` and **`/redistributionstate`** per node. It records +up/down `storageRadius` transitions, and asserts the **halt indicators** that define the +October failure: + +- **recovery after a decrease** — after a **down** transition, the node must recover within + `RecoveryWait`. Recovery prefers the redistribution signal **`isFullySynced && !isFrozen`** + (the real "can play the game" signal); if `/redistributionstate` is unavailable (not + full-mode), it falls back to `pullsyncRate > 0`. +- **freeze episodes** — a node reporting `isFrozen` is skipping redistribution rounds (a halt + symptom); each freeze episode is counted. + +Un-recovered decreases or freezes fail the check at the end. Never uploads (the radius is +driven externally, e.g. by a parallel `load` check). ## Options @@ -79,8 +88,12 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - `…_reserve_size{node}` — gauge, reserve size (chunks) per node - `…_pullsync_rate{node}` — gauge, `/status` pullsyncRate per node - `…_time_to_increase_seconds` / `…_time_to_decrease_seconds` — gauges (drive mode) +- `…_reserve_within_radius{node}` — gauge, reserve chunks within radius (completeness signal) - `…_radius_transitions_total{node,direction}` — counter, observed up/down transitions (observe mode) - `…_recovery_observed_total{node,result}` — counter, recovery outcome `recovered`/`timeout` (observe mode) +- `…_time_to_fully_synced_seconds` — gauge, decrease → isFullySynced again (observe mode) +- from `/redistributionstate` (observe mode): `…_fully_synced{node}`, `…_frozen{node}`, + `…_redistribution_round{node}`, `…_last_sample_duration_seconds{node}` — the halt indicators ## Requirements @@ -123,10 +136,15 @@ checks fail independently (a monitor failure does not stop the load run). ## Known limitations / follow-ups -- Recovery is asserted via `/status` `pullsyncRate`, which is a poor signal on light - clusters (stays ~0, so observe mode can false-`timeout`). A stronger signal is - scraping node `/metrics` (`bee_puller_worker`, `bee_pullsync_chunks_delivered`) and - checking the worker set reconfigures and recovers — not yet wired in. +- Observe-mode recovery uses `/redistributionstate` (`isFullySynced`/`isFrozen`) where + available, falling back to `/status` `pullsyncRate` (weak on light clusters — can + false-`timeout`). The **exact thresholds** (the `isFullySynced` bound, acceptable + convergence time, the `offered/delivered` redundancy target) need to come from the + pull-sync design owner (@marios) — see `docs/radius-check-plan.md` "Pull-sync validation". +- Per-`(peer,bin)` `STALLED` / `MaxStallsPerBin` signals from the **fixed** pull-sync + (@sig's PR) are not yet scraped (they don't exist on stock bee). +- The neighbourhood **merge** (`k→8` on a decrease) needs a ~20-node cluster to reproduce; + the local 3-node cluster proves the mechanism and pipeline only. - Direction is fixed to increase-then-decrease (drive) / observe-both (observe); no `increase`-only assertion mode yet. - No unit tests yet (external `_test` package TBD). diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go index 5f27821a..f5c817eb 100644 --- a/pkg/check/reserveradius/metrics.go +++ b/pkg/check/reserveradius/metrics.go @@ -6,13 +6,20 @@ import ( ) type metrics struct { - StorageRadius *prometheus.GaugeVec - ReserveSize *prometheus.GaugeVec - PullsyncRate *prometheus.GaugeVec - TimeToIncrease prometheus.Gauge - TimeToDecrease prometheus.Gauge - RadiusTransitions *prometheus.CounterVec - RecoveryObserved *prometheus.CounterVec + StorageRadius *prometheus.GaugeVec + ReserveSize *prometheus.GaugeVec + ReserveWithinRadius *prometheus.GaugeVec + PullsyncRate *prometheus.GaugeVec + TimeToIncrease prometheus.Gauge + TimeToDecrease prometheus.Gauge + TimeToFullySynced prometheus.Gauge + RadiusTransitions *prometheus.CounterVec + RecoveryObserved *prometheus.CounterVec + // redistribution-game liveness (observe mode), from /redistributionstate + FullySynced *prometheus.GaugeVec + Frozen *prometheus.GaugeVec + RedistRound *prometheus.GaugeVec + LastSampleDuration *prometheus.GaugeVec } func newMetrics(subsystem string) metrics { @@ -78,9 +85,26 @@ func newMetrics(subsystem string) metrics { }, []string{"node", "result"}, ), + ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node (completeness signal)."), + TimeToFullySynced: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "time_to_fully_synced_seconds", Help: "Seconds from a radius decrease to the node reporting isFullySynced again.", + }), + FullySynced: nodeGauge(subsystem, "fully_synced", "Redistribution isFullySynced per node (1=yes, 0=no) — can the node play the game."), + Frozen: nodeGauge(subsystem, "frozen", "Redistribution isFrozen per node (1=frozen) — a halt symptom (skips rounds)."), + RedistRound: nodeGauge(subsystem, "redistribution_round", "Current redistribution round per node (stuck round = halt symptom)."), + LastSampleDuration: nodeGauge(subsystem, "last_sample_duration_seconds", "Last reserve-sample duration per node, from /redistributionstate."), } } +// nodeGauge builds a per-node GaugeVec in the check namespace/subsystem. +func nodeGauge(subsystem, name, help string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{Namespace: m.Namespace, Subsystem: subsystem, Name: name, Help: help}, + []string{"node"}, + ) +} + // Report implements the metrics.Reporter interface. func (c *Check) Report() []prometheus.Collector { return m.PrometheusCollectorsFromFields(c.metrics) diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index d533c2e4..74ac0901 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -179,10 +179,22 @@ func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o O return c.observeDecrease(ctx, nodes, peak, o) } -// runObserve monitors radius transitions for Duration without uploading; the radius -// is expected to be driven externally (e.g. by a parallel load check). It records -// every up/down transition, and after each decrease waits up to RecoveryWait for -// pull-sync to resume — decreases that never recover are reported at the end. +// obsNode is the per-node state the observe monitor tracks across polls. +type obsNode struct { + lastRadius uint8 + haveRadius bool + recoverBy time.Time // non-zero => awaiting recovery after a decrease + downAt time.Time // when the pending decrease started (for time-to-recovery) + frozen bool // currently inside a frozen episode +} + +// runObserve monitors a cluster for Duration without uploading; the radius is driven +// externally (e.g. by a parallel load check). It records every up/down radius +// transition and, after each decrease, asserts the node recovers — preferring the +// redistribution-game signal (isFullySynced, not frozen) over the weaker pullsyncRate +// when /redistributionstate is available. It also flags freeze episodes directly +// (a frozen node skips rounds — the October halt symptom). Halt indicators +// (un-recovered decreases or freezes) fail the check at the end. func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o Options) error { nodes, err := c.selectNodes(ctx, cluster, o) if err != nil { @@ -194,18 +206,13 @@ func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o return err } - last := make(map[string]uint8, len(nodes)) // last seen storageRadius per node - recoverBy := make(map[string]time.Time, len(nodes)) // node -> deadline to see recovery after a decrease - unrecovered := 0 - - // seed baseline radii + st := make(map[string]*obsNode, len(nodes)) for _, n := range nodes { - if s, err := n.Status(ctx); err == nil { - last[n.Name()] = s.StorageRadius - c.emit(n.Name(), s) - } + st[n.Name()] = &obsNode{} } - c.logger.Info("observe: baseline captured; watching for radius transitions") + redistAvailable := true // flips false on first error; selects which recovery signal to use + unrecovered, freezes := 0, 0 + c.logger.Info("observe: watching radius transitions, freezes, and redistribution liveness") deadline := time.Now().Add(o.Duration) for time.Now().Before(deadline) { @@ -213,48 +220,85 @@ func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o return err // parent context cancelled (e.g. check timeout) } for _, n := range nodes { + name := n.Name() + ns := st[name] + s, err := n.Status(ctx) if err != nil { continue } - name := n.Name() c.emit(name, s) - cur := s.StorageRadius - if prev, seen := last[name]; seen && cur != prev { + // redistribution-game state (full-mode only; best-effort) + var rs *api.RedistributionState + if redistAvailable { + if r, rerr := n.RedistributionState(ctx); rerr == nil { + rs = r + c.emitRedist(name, r) + } else { + redistAvailable = false + c.logger.Warningf("observe: /redistributionstate unavailable (%v); using pullsyncRate for recovery, no freeze detection", rerr) + } + } + + // radius transition + cur := s.StorageRadius + if ns.haveRadius && cur != ns.lastRadius { dir := "up" - if cur < prev { + if cur < ns.lastRadius { dir = "down" } c.metrics.RadiusTransitions.WithLabelValues(name, dir).Inc() - c.logger.Infof("observe: %s storageRadius %d -> %d (%s) reserveSize=%d pullsyncRate=%.4f", name, prev, cur, dir, s.ReserveSize, s.PullsyncRate) + c.logger.Infof("observe: %s storageRadius %d -> %d (%s) within=%d pullsyncRate=%.4f", name, ns.lastRadius, cur, dir, s.ReserveSizeWithinRadius, s.PullsyncRate) if dir == "down" { - recoverBy[name] = time.Now().Add(o.RecoveryWait) + ns.recoverBy = time.Now().Add(o.RecoveryWait) + ns.downAt = time.Now() + } + } + ns.lastRadius = cur + ns.haveRadius = true + + // freeze detection — a frozen node skips redistribution rounds (halt symptom) + if rs != nil { + switch { + case rs.IsFrozen && !ns.frozen: + ns.frozen = true + freezes++ + c.logger.Warningf("observe: %s is FROZEN at round %d (halt symptom; lastFrozenRound=%d)", name, rs.Round, rs.LastFrozenRound) + case !rs.IsFrozen: + ns.frozen = false } } - last[name] = cur // recovery tracking after a decrease - if rd, waiting := recoverBy[name]; waiting { + if !ns.recoverBy.IsZero() { + recovered, reason := false, "" + switch { + case rs != nil && rs.IsFullySynced && !rs.IsFrozen: + recovered, reason = true, "fullySynced" + case rs == nil && s.PullsyncRate > 0: + recovered, reason = true, "pullsyncRate(fallback)" + } switch { - case s.PullsyncRate > 0: - delete(recoverBy, name) + case recovered: c.metrics.RecoveryObserved.WithLabelValues(name, "recovered").Inc() - c.logger.Infof("observe: %s pull-sync recovered (pullsyncRate=%.4f)", name, s.PullsyncRate) - case time.Now().After(rd): - delete(recoverBy, name) + c.metrics.TimeToFullySynced.Set(time.Since(ns.downAt).Seconds()) + c.logger.Infof("observe: %s recovered after decrease in %s (%s)", name, time.Since(ns.downAt).Round(time.Second), reason) + ns.recoverBy = time.Time{} + case time.Now().After(ns.recoverBy): unrecovered++ c.metrics.RecoveryObserved.WithLabelValues(name, "timeout").Inc() - c.logger.Warningf("observe: %s no pull-sync recovery within %s after decrease (cf. PR #581)", name, o.RecoveryWait) + c.logger.Warningf("observe: %s did NOT recover within %s after decrease (not fullySynced / no pull-sync) — pull-sync halt symptom (cf. PR #581)", name, o.RecoveryWait) + ns.recoverBy = time.Time{} } } } } - if unrecovered > 0 { - return fmt.Errorf("observe: %d radius decrease(s) showed no pull-sync recovery within %s", unrecovered, o.RecoveryWait) + if unrecovered > 0 || freezes > 0 { + return fmt.Errorf("observe: halt indicators over %s — %d decrease(s) without recovery, %d freeze episode(s)", o.Duration, unrecovered, freezes) } - c.logger.Infof("observe: completed %s monitor with all decreases recovered", o.Duration) + c.logger.Infof("observe: completed %s monitor — no halt indicators (all decreases recovered, no freezes)", o.Duration) return nil } @@ -413,9 +457,24 @@ func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, func (c *Check) emit(node string, s *api.StatusResponse) { c.metrics.StorageRadius.WithLabelValues(node).Set(float64(s.StorageRadius)) c.metrics.ReserveSize.WithLabelValues(node).Set(float64(s.ReserveSize)) + c.metrics.ReserveWithinRadius.WithLabelValues(node).Set(float64(s.ReserveSizeWithinRadius)) c.metrics.PullsyncRate.WithLabelValues(node).Set(s.PullsyncRate) } +func (c *Check) emitRedist(node string, r *api.RedistributionState) { + c.metrics.FullySynced.WithLabelValues(node).Set(b2f(r.IsFullySynced)) + c.metrics.Frozen.WithLabelValues(node).Set(b2f(r.IsFrozen)) + c.metrics.RedistRound.WithLabelValues(node).Set(float64(r.Round)) + c.metrics.LastSampleDuration.WithLabelValues(node).Set(r.LastSampleDurationSeconds) +} + +func b2f(b bool) float64 { + if b { + return 1 + } + return 0 +} + func sleepCtx(ctx context.Context, d time.Duration) error { select { case <-ctx.Done(): From ebd3e2fb9005e5e38a6310f930e3177accc4aa42 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 25 Jun 2026 10:46:03 +0200 Subject: [PATCH 07/37] docs: add A/B testing procedure to radius-check plan --- docs/radius-check-plan.md | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index 93929022..47848922 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -305,6 +305,50 @@ Phase-3 follow-up. - [ ] Wire it as a **periodic / public-testnet RC** check (per #591 review), not standard PR CI. A/B candidate bee versions by comparing the emitted metrics. +## A/B testing procedure (current vs fixed bee) + +The single-build check only says "this build halted / didn't halt." To **prove the pull-sync +fix works**, run the *identical* radius-decrease scenario against two builds and diff them: + +- **A = current bee** (stock `master`) — the baseline; expected to **reproduce** the halt. +- **B = fixed bee** (@sig's SWIP-25 pull-sync PR) — expected to **survive** the same scenario. + +**Why both:** if B alone doesn't halt, that may just mean the scenario was too mild. Showing **A +halts first calibrates the test** — it proves the scenario reproduces the failure, so B surviving +is real evidence. The **A→B delta is the proof**. (It's the empirical complement to the design +doc's analytical + TLA+ proof — the implementation behaving as the design predicts on a real net.) + +**Procedure:** + +1. **Scenario fixed and deterministic** — same `RndSeed`, same `load`/decrease config, same cluster + size, keys, and data volume for both runs. Any difference confounds the comparison. +2. **Build two images** — `…/bee:current` (stock) and `…/bee:fixed` (the PR branch); push both + (the existing `docker-build`/patch mechanism). For A, no pull-sync fix; for B, the PR. +3. **Run the same scenario against each** (two clusters, or redeploy the same one per image) with + `--metrics-enabled`. **Label each run** so both land in one Prometheus — e.g. distinct pushgateway + job names or an external `build=current|fixed` label per deployment. +4. **Diff the halt indicators** (Grafana side-by-side, or a small report): + + | signal | A (current) expected | B (fixed) expected | + |---|---|---| + | freeze episodes / `is_playing_errors` rate | fire | none | + | un-recovered decreases (`fully_synced` timeout) | yes | no | + | `time_to_fully_synced_seconds` | huge / never | bounded | + | redistribution `round` | stuck / skipped | keeps advancing | + | `reserve_within_radius` (completeness) | stalls low | catches up | + | `offered/delivered` redundancy ratio | ~k | ~1 (SWIP-25 target) | + +**Gating / dependencies:** + +- **Scale** — only meaningful on the **~20-node cluster**: the halt is a neighbourhood *merge* + (k→8), so on 3 local nodes **neither** A nor B halts and there is nothing to diff. A/B is a + Phase-3 activity. +- **B must exist** — depends on @sig's PR being buildable; we don't control its timing. +- **Reproducing the halt in A** is itself part of the research — the scenario must be severe enough + (operator-offline → radius decrease) to trigger it reliably. +- Likely a **documented runbook or a thin wrapper command** (build A/B → run scenario twice → label + → compare), not new check logic — the check + metrics + dashboards already exist. + ## Open questions - Increase, decrease, or both? Decrease exercises the known puller bug (#581) and From 785533997ba473d8a53518e64f21c8efb0dda9aa Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 25 Jun 2026 11:45:16 +0200 Subject: [PATCH 08/37] docs: condense and reframe radius-check plan --- docs/radius-check-plan.md | 477 +++++++++++--------------------------- 1 file changed, 136 insertions(+), 341 deletions(-) diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index 47848922..296e6443 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -1,154 +1,90 @@ # Reserve-radius change check — plan -Status: **draft / not started**. Owner: TBD. Last updated: 2026-06-16. - -> **Working agreement** -> - Build the check on a **new feature branch off `master`** (e.g. -> `feat/reserve-radius-check`), never directly on `master`. The patched-local -> experiments and the bee-side patch may also need a coordinated bee branch. -> - The check must follow the **standardized behaviour of existing checks** — same -> shape, helpers, and idioms as `smoke` / `feed` / `load` (random full-node -> selection via `cluster.ShuffledFullNodeClients`, seeded RNG, `PostageTTL`-based -> batches, `metrics.Reporter`). See "Standard check conventions" below; do not -> invent a new pattern. - -A repeatable, automated way to stage a **storage-radius change** across a Bee -cluster and capture what happens to the reserve and to pull-sync afterwards. The -end product is a beekeeper check (`pkg/check/reserveradius`) plus the metrics it -emits. Operating know-how lives in the `radius-testing` skill; this doc is the -design and the build checklist. +Status: **Phases 0–2.5 done; Phase 3 + A/B pending.** Branch `ljubisa-radius-soak`. + +A repeatable, automated way to stage a **storage-radius change** across a Bee cluster and +verify pull-sync recovers afterwards. End product: a beekeeper check +(`pkg/check/reserveradius`) plus the metrics it emits. Operating know-how lives in the +`radius-testing` skill; this doc is the design and build checklist. ## Pull-sync validation (the real target) -The narrow goal is "force a radius decrease, confirm the puller doesn't deadlock" -(PR #581). The **broader goal** — why @sig/@marios want this harness — is to make a -**reproducible test of the October network halt** and to validate the pull-sync -redesign (SWIP-25 / `pullsync-optimal-design`). - -**The October mechanism.** A large operator went offline → network commitment dropped -→ **storage radius decreased**, which is a neighbourhood **merge**: neighbourhood size -`k` transiently jumps to ~8, so every node must suddenly re-sync a much larger reserve -from many more peers. Today's pull-sync is "correctness through redundancy" — each of -`k` peers runs an independent session with **up to k× redundant deliveries** per chunk, -no shared dedup, no explicit failover-with-exclude; on a depth change bins get -"cancelled and restarted repeatedly, causing unbounded delay," and stalls livelock. -Result: incomplete reserves → wrong `ReserveSample` → nodes **freeze, skip redistribution -rounds, go unresponsive** (→ slashing risk). The avalanche. - -**What the test must show** on a staged radius **decrease**: - -- **Completeness** — each node's reserve catches up (`reserveSizeWithinRadius` recovers). -- **Liveness / no halt** — nodes don't freeze, redistribution `round` keeps advancing, - `isFullySynced` returns within a bound. +The narrow goal is "force a radius decrease, confirm the puller doesn't deadlock" (PR #581). +The broader goal — why @sig/@marios want this — is a **reproducible test of the October +network halt** and validation of the pull-sync redesign (SWIP-25 / `pullsync-optimal-design`). + +**The October mechanism.** A large operator went offline → commitment dropped → **storage +radius decreased**, a neighbourhood **merge**: `k` jumps to ~8, so every node must re-sync a +much larger reserve from many more peers. Today's pull-sync is "correctness through +redundancy" — up to **k× redundant deliveries** per chunk, no shared dedup, no +failover-with-exclude; on a depth change bins are cancelled/restarted repeatedly (unbounded +delay) and stalls livelock. Result: incomplete reserves → wrong `ReserveSample` → nodes +**freeze, skip rounds, go unresponsive** (slashing risk). + +**What the test must show** on a staged decrease: + +- **Completeness** — reserves catch up (`reserveSizeWithinRadius` recovers). +- **Liveness** — no freezes, `round` keeps advancing, `isFullySynced` returns within a bound. - **Bounded convergence** — resync finishes in bounded time; no stuck `STALLED` bins. -- **(SWIP-25 efficiency)** — redundant deliveries drop ~k×→1× (`offered/delivered` ratio). -- Ideally **A/B**: current bee reproduces the halt; fixed bee doesn't. - -**What the check now measures toward this** (implemented): observe mode reads -`/status` **and `/redistributionstate`** and asserts the halt indicators — un-recovered -decreases (prefers `isFullySynced`, falls back to `pullsyncRate`) and **freeze episodes** -(`isFrozen`). New metrics: `fully_synced`, `frozen`, `redistribution_round`, -`reserve_within_radius`, `last_sample_duration_seconds`, `time_to_fully_synced_seconds`. -The `radius-cluster-behavior` dashboard has a **Halt indicators** row (fullySynced/frozen, -round-advance, `is_playing_errors` rate, the offered/delivered redundancy ratio). - -**Still needs a sync with @marios + scale** (not solo-buildable): - -- Exact pass/fail thresholds (the `isFullySynced` bound after a decrease, acceptable - convergence time, the redundancy-ratio target). -- The per-`(peer,bin)` `STALLED` / `MaxStallsPerBin` metrics — only the **fixed** - pull-sync (@sig's PR) exposes those; the check would then scrape them. -- A **~20-node ephemeral cluster** — the merge (`k→8`) needs scale; 3 local nodes can't - form a realistic neighbourhood. This is ljubisa's stated end goal. -- An **A/B harness** (same scenario, current vs fixed bee, diff the signals). - -## Why this is hard (and what prior tryouts taught us) - -- A stock node's reserve is far too large to overflow in CI-time, so a radius - transition won't happen on a small local cluster without help. -- After a radius change, the node **resyncs its reserve over pull-sync**, and the - redistribution-game "fully synced" status takes a long time to settle. So the - test is **stage → monitor over time**, not a one-shot assertion. -- **PR #581** (`radiusdecrease`, open, unmerged) — deliberately forces a radius - *decrease* (overflow a tiny reserve to push `storageRadius` 0→1, let it fall - 1→0) and polls `/status` for `pullsyncRate > 0` to confirm puller recovery. - Reusable `waitForRadius` / `waitForRecovery` poll loops. **Requires a patched - bee image** (`master-scenario-b`) + a coordinated bee PR; no review; carries - dirty local config. The real radius tryout. -- **PR #591** (`stampexpiry`, draft) — patch-free, observes radius only as a side - effect of stamp expiry. Review feedback (unactioned): use `PostageTTL`, pick a - **random** node not `sortedNodes[0]`, account for the ~10-block batch-usable - delay, and run as a **periodic / public-testnet RC** check, not standard CI. -- **GC check** (`pkg/check/gc/reserve.go`) — the precedent for a check that - depends on a bee patch: patches live in `bee/.github/patches/`, applied by - bee's beekeeper workflow before the check runs. - -## Signals (see the `radius-testing` skill for the full table) - -Per node, HTTP API (`http://.localhost`): -`/status` → `storageRadius`, `pullsyncRate`, `reserveSize`, -`reserveSizeWithinRadius`, `committedDepth`; `/reservestate` → `radius`, -`commitment`; `/redistributionstate` (full-mode only) → `isFullySynced`, `phase`. -Prometheus: `bee_localstore_storage_radius`, `bee_localstore_reserve_size*`, -`bee_pullsync_chunks_*`, `bee_postage_radius`, `bee_storageincentives_*`. - -## Approach: patched-local first, then real ephemeral - -**Decision:** start with a **patched local cluster** for deterministic, fast radius -transitions; graduate to an **unpatched ephemeral k8s cluster** for realistic -timing (periodic/RC cadence). Both phases drive the same check; only the -environment and timeouts differ. - ---- - -## Phase 0 — Local loop scaffolding (mostly done) - -- [x] `/cluster-verify` command — verify substrate + Bee nodes + capture a - reserve/radius baseline. -- [x] `/cluster-up` / `/cluster-down` commands — wrap beelocal + docker + beekeeper. -- [x] `radius-testing` skill + `scripts/radius-poll.sh` — the monitoring poller. -- [ ] Run `/cluster-up local-dns`, `/cluster-verify local-dns`, confirm a clean - baseline (`storageRadius==0`, full nodes connected). - -## Staging the change: driving the radius via uploads (reuse the `load` check) - -You do **not** pre-compute "how many bytes" to upload — you upload in a loop, watch the -radius signal, and stop at a target. The `load` check already does exactly this: - -- `load` (`pkg/check/load/load.go`) has `MaxCommittedDepth` and gates every upload on - `checkCommittedDepth(client, max, wait)`, which reads `client.Status(ctx).CommittedDepth` - and keeps uploading while `CommittedDepth < max` (and *waits* once the cap is reached). - Since `committedDepth = storageRadius + capacityDoubling`, capping committedDepth caps radius. - `ci-load` in `config/local.yaml` already wires `max-committed-depth: 2`. - -**Increase and decrease are NOT symmetric — this is the crux:** - -- **Increase** is upload-driven: fill the reserve past capacity ⇒ eviction ⇒ `storageRadius` - rises. `load` already produces this. -- **Decrease** is *not* upload-driven. Per `bee/pkg/storer/reserve.go`, radius decreases only - when the reserve is **under-utilized** AND the node is **fully synced** (`SyncRate()==0`) AND - above `minimumRadius`. Trigger it by first inflating radius, then letting the patched - (tiny-reserve, threshold=capacity, fast wake-up) reserve worker re-evaluate — PR #581's - "overflow to push 0→1, then watch it fall 1→0" cascade. **The resync-over-decrease is the - mechanism that is not yet understood, so it must be observed (spike) before it is designed.** - -**Decisions:** -- The final `reserveradius` check **drives its own upload** (it needs tight - stage→poll→record→assert control), **reusing load's committedDepth-gated upload primitive** - — factor it into a shared helper rather than copy/paste. Smaller batches = finer radius control. -- For the **initial spike**, drive uploads with the existing `ci-load` check (set - `max-committed-depth` to the target) while `radius-poll.sh` records the timeline — no new Go. - -## Phase 1 — Empirical spike: force a change with a patched bee, observe decrease + resync - -- [x] **Patch created** (in `bee/.github/patches/`, GC-check pattern). All three symbols - are unreachable by config — `ReserveCapacity` = `(1<0` recovery. Needs a patched bee image. The real radius tryout. +- **PR #591** (`stampexpiry`) — patch-free, observes radius via stamp expiry. Review feedback: + use `PostageTTL`, random node, periodic/RC cadence, not standard CI. +- **GC check** (`pkg/check/gc/reserve.go`) — precedent for a patch-dependent check; patches live + in `bee/.github/patches/`, applied by bee's beekeeper workflow. + +## Signals + +Per node: `/status` (`storageRadius`, `pullsyncRate`, `reserveSize`, `reserveSizeWithinRadius`, +`committedDepth`, `isWarmingUp`), `/reservestate` (network `radius`), `/redistributionstate` +(`isFullySynced`, `isFrozen`, `round`). Prometheus: `bee_localstore_*`, `bee_pullsync_*`, +`bee_storageincentives_*`. Full table in the `radius-testing` skill. + +## Approach + +Patched-local first (deterministic, fast transitions) → unpatched ephemeral k8s (realistic +timing, periodic/RC cadence). Same check, different environment and timeouts. + +## Phase 0 — Local scaffolding (done) + +- [x] `/cluster-verify`, `/cluster-up`, `/cluster-down` commands. +- [x] `radius-testing` skill + `scripts/radius-poll.sh` monitor. + +## Staging the change (reuse the `load` check) + +Don't pre-compute byte counts — upload in a loop, watch the signal, stop at a target. `load` +(`pkg/check/load/load.go`) already does this: `MaxCommittedDepth` gates uploads via +`checkCommittedDepth` (`committedDepth = storageRadius + capacityDoubling`). + +**Increase and decrease are not symmetric.** Increase is upload-driven (fill past capacity → +eviction → radius up). Decrease is not — per `bee/pkg/storer/reserve.go`, radius drops only when +the reserve is under-utilized AND fully synced (`SyncRate()==0`) above `minimumRadius`. Trigger +it by inflating first, then letting the patched reserve worker re-evaluate (0→1→0 cascade). + +## Phase 1 — Spike: force a change with a patched bee (done) + +- [x] **Patch created** in `bee/.github/patches/` (GC-check pattern; all three symbols are + unreachable by config) — what and why: + - `radius_reserve.patch`: `DefaultReserveCapacity` 1<<22→**200** (so ~2 MB moves the radius — + stock is too large to budge in CI-time) and `ReserveWakeUpDuration` 30m→**10s** (the reserve + worker re-evaluates in seconds, not 30 min). + - `radius_threshold.patch`: decrease `threshold` 50%→**100%** (makes the 1→0 decrease reachable). + gofmt-clean, applies cleanly. +- [ ] **Apply, build, push, redeploy** (revert source after build): ```sh cd @@ -160,209 +96,68 @@ docker push k3d-registry.localhost:5000/ethersphere/bee:latest git checkout pkg/storer/reserve.go pkg/storer/storer.go ``` -Then redeploy so nodes pull the patched `:latest` (imagePullPolicy: Always): -`/cluster-down local-dns` → `/cluster-up local-dns` (or `kubectl rollout restart statefulset -n local`). -To wire into CI later, add the two `patch` lines to bee's `.github/workflows/beekeeper.yml` -"Apply patches and build" step (next to `postage_api`/`retrieval`). - -- [ ] **Drive the increase** with the existing `ci-load` check (no new Go): set - `max-committed-depth` to the target so it uploads until `storageRadius`/`committedDepth` - reaches it. Run `radius-poll.sh` alongside to record the 0→target timeline. -- [ ] **Observe the decrease + resync** — stop uploads and watch whether `storageRadius` - falls back and how pull-sync behaves (`pullsyncRate`, `reserveSize`, - `reserveSizeWithinRadius` over time). Capture the full CSV timeline; use `/loop` for the - long watch. This answers the open question of *how* resync-over-radius-decrease works. -- [x] **Gate met** — spike reproduced a real increase, decrease, and resync (findings below). - -### Spike findings (2026-06-17, local-dns, patched `:200/10s/100%` image) - -Sequence observed (3 full nodes, uploading 1 MB blobs to `bee-0`): - -1. **Increase is immediate** — `storageRadius` 0→1 after ~2 uploads (~2 MB), 0→2 within ~14 s. - On stock capacity it never moves; the patch is what makes it observable. -2. **Overshoot after uploads stop** — radius kept climbing (1→2) *after* uploads ceased, - because in-flight **pushsync** keeps filling reserves. "Uploads stopped" ≠ "radius settled". -3. **Decrease lags ~10 min and is gated on stabilization** — the 2→1 decrease fired at - ~10.5 min after node start (`node "Sync status check evaluated" stabilized=true`), ~30 s - after a 10-min watch window ended. The reserve worker's decrease loop sits behind the - startup-stabilizer gate (`reserve.go` `startReserveWorkers` waits on `startupStabilizer`), - so a fresh node won't decrease for several minutes regardless of reserve state. -4. **Puller drives the reconfiguration** — `node/puller "radius decrease" old=2 new=1`, preceded - by a storm of `syncWorker context cancelled` (disconnect/reconnect per bin). This is the - PR #581 `manage()`/`disconnectPeer()` path — the liveness risk lives here. -5. **Decrease ↔ resync interlock (staircase)** — decrease requires `SyncRate()==0`, but a decrease - triggers pull-sync (`pullsyncRate>0`), which then blocks the next decrease until it drains. - Radius settles in steps, not one jump. -6. **It does not return to a fixed floor** — after `2→1`, the radius held at **1 for 30+ min and never - reached 0** (still 1 at re-check, even with `pullsyncRate==0` on two of three nodes by then; - `reserveSizeWithinRadius` rose to equal `reserveSize` as the reserve reorganized within radius 1). - The `1→0` step simply did not occur in-window — the exact gate is secondary (candidate: residual - sync noise and/or how `countWithinRadius` evaluates at the boundary). Design takeaway: the check - must assert *"a decrease occurred AND pullsync recovered"*, NOT *"radius returned to N"* — the - equilibrium depends on data volume and sync state. - -**Implications for the Phase 2 check (bake these in):** - -- Use a **random full node** but expect the transition to show on whichever node's neighborhood - fills — assert on the cluster, not one node. -- **Long, staged timeouts**: increase ~minutes; decrease/settle **≥15 min** (cf. PR #581's 20-min - recovery). Don't fail fast. -- "Radius reached target" must wait for **pushsync to drain** (overshoot) before treating a value - as settled — poll until `storageRadius` is stable for K ticks AND `pullsyncRate==0`, not a single read. -- Gate the decrease assertion on **`isWarmingUp==false`/stabilized** first; before that, no decrease can occur. -- The key liveness signal is `pullsyncRate>0` returning after the decrease (puller workers - recovered) — the same assertion PR #581 makes. A stuck `manage()` shows as `pullsyncRate` flat at 0. -- `radius-poll.sh` with `-i 15` + auto-stop-on-stable is the right monitor; the in-repo check - formalizes this loop. - -## Standard check conventions (match existing checks — don't invent a pattern) - -Copy the shape of `smoke` (`pkg/check/smoke/smoke.go`), `feed`, `load`. Concretely: - -- **Boilerplate:** `Options` struct + `NewDefaultOptions() Options`; compile check - `var _ beekeeper.Action = (*Check)(nil)`; `Check{ metrics, logger }`; - `NewCheck(log logging.Logger) beekeeper.Action` → `&Check{metrics: newMetrics("check_reserve_radius"), logger: log}`. -- **Run signature:** `Run(ctx context.Context, cluster orchestration.Cluster, opts any) error`, - first line `o, ok := opts.(Options); if !ok { return errors.New("invalid options type") }`. -- **Seeded RNG + random node selection** (the part the user called out — `smoke.go:116-131`). - Take `[0]` of the shuffled list; never hard-index an unshuffled list / always-node-0 (the - exact thing flagged in PR #591 review): - - ```go - rnd := random.PseudoGenerator(o.RndSeed) // pkg/random - fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) // (ctx, *rand.Rand) (ClientList, error) - if err != nil { return fmt.Errorf("get shuffled full node clients: %w", err) } - if len(fullNodeClients) < 1 { return fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(fullNodeClients)) } - node := fullNodeClients[0] // a random node, since the list is shuffled - c.logger.Infof("random seed: %d", o.RndSeed) - ``` - -- **Postage:** use `node.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel)` - with `PostageTTL time.Duration` (default `24h`), **not** a raw amount. -- **Options idioms:** `RndSeed int64` defaulting to `time.Now().UnixNano()`; durations as - `time.Duration`; a `Duration` + `scheduler.NewDurationExecutor(...)` if it's a long/repeating run. -- **Loops:** `select { case <-ctx.Done(): return nil; default: }` guard each iteration. -- **Errors:** wrap with context (`fmt.Errorf("...: %w", err)`); failure messages name the - suspected cause + the timeout hit. -- **Tests:** external `package reserveradius_test`; race detector clean. - -## Phase 2 — The beekeeper check (`pkg/check/reserveradius`) - -On branch **`ljubisa-radius-check`**. Mirrors the existing check structure -(`Action` interface, `NewCheck(logger)`, `metrics.Reporter`, the conventions above); -registered in `pkg/config/check.go`, `ci-reserve-radius` entry in `config/local.yaml`. - -- [x] `pkg/check/reserveradius/reserveradius.go` — `Run` = `waitForWarmupDone` → - baseline → `driveIncrease` (upload blobs until `storageRadius≥TargetRadius`, reuses - `test.Upload`) → settle (pushsync drain) → `observeDecrease` (assert a decrease vs the - per-node peak; watch `pullsyncRate` recovery; timeout message names the PR #581 puller stall). -- [x] `pkg/check/reserveradius/metrics.go` — `storage_radius` / `reserve_size` / - `pullsync_rate` gauges per node + `time_to_increase_seconds` / `time_to_decrease_seconds`. -- [x] Registered in `pkg/config/check.go` (type `reserve-radius`); added `IsWarmingUp` to - `pkg/bee/api/status.go` `StatusResponse` (node returns `isWarmingUp`; struct lacked it). -- [x] `config/local.yaml`: `ci-reserve-radius` (timeout 45m). Runs against the patched image. -- [x] Gate: `make build`, `go vet`, `golangci-lint` (0 issues) all green. (Unit tests deferred.) -- [x] **Validated end-to-end (2026-06-17)** on a fresh local-dns: increase 0→1 in 3s (1 MiB), - 1-min settle, **decrease observed on bee-2 after 13m16s with `pullsyncRate>0` recovery=true**, - `check completed successfully` (total 14m36s). Confirms the ~13-min stabilization-gated - decrease and that the 20-min `DecreaseTimeout` is correctly sized. - -Open follow-ups (not blockers): make `pullsyncRate>0` recovery a **hard** gate once its rate floor is -characterised on a data-heavy cluster (Phase 3); add external `_test` package; optional `increase`-only -and `both` directions. - -## Phase 2.5 — driver/observer split + parallel checks (soak) - -To test radius behavior over long runs (24h–3 days), split the **uploader** (load) from the -**observer** (reserve-radius) and run them concurrently, so the cluster oscillates repeatedly and -we get statistical coverage of resync/recovery rather than a single transition. - -- [x] **`--parallel` flag** (`cmd/beekeeper/cmd/check.go` + `pkg/check/runner.go`) — opt-in; - default stays sequential. When set, `runner.Run` runs the listed checks in goroutines via a - `WaitGroup`; each carries its own timeout and an error does not cancel the others (independent - failures — a monitor blip won't kill a 24h load run). -- [x] **`load` re-arming `decrease-hold`** (`pkg/check/load/load.go`) — new `DecreaseHold` duration - (yaml `decrease-hold`, default 0 = today's behavior). Once `committedDepth` reaches - `max-committed-depth` and then drops, pause uploads for `decrease-hold` before re-filling, then - re-arm → repeated fill→decrease→hold→re-fill cycles. State guarded by a mutex (single-uploader soak). -- [x] **`reserve-radius` `mode: drive | observe`** (`pkg/check/reserveradius/`) — `drive` = the existing - one-shot flow; `observe` = a `Duration`-bounded monitor (no uploads) that records up/down - transitions and asserts recovery (`RecoveryWait`) after each decrease. New metrics - `radius_transitions_total{node,direction}` and `recovery_observed_total{node,result}`. -- [x] Config: `ci-reserve-radius-observe` (mode observe) + `ci-load-soak` (decrease-hold) in `config/local.yaml`. - -Run the soak: `beekeeper check --checks=ci-load-soak,ci-reserve-radius-observe --parallel ...`. -Watch the Grafana `radius-cluster-behavior` / `pullsync-behavior` dashboards for repeated cycles. - -Note: observe-mode recovery is asserted via `/status` `pullsyncRate` (weak on light clusters — can -false-`timeout`); the stronger node-`/metrics` signal (`bee_puller_worker`, `chunks_delivered`) is the -Phase-3 follow-up. - -## Phase 3 — Real ephemeral cluster, no patch - -- [ ] Run the same check against an unpatched ephemeral k8s cluster with realistic - reserve sizes and timing; expect long durations. -- [ ] Wire it as a **periodic / public-testnet RC** check (per #591 review), not - standard PR CI. A/B candidate bee versions by comparing the emitted metrics. +Then `/cluster-down local-dns` → `/cluster-up local-dns`. To wire into CI, add the two `patch` +lines to bee's `.github/workflows/beekeeper.yml` "Apply patches and build" step. + +### Why the check is shaped this way (spike, 2026-06-17) + +The decrease is slow and non-obvious; each fact below maps to a check decision: + +- **Lags ~10 min, stabilization-gated** (the reserve worker's decrease loop waits on the + startup-stabilizer) → long timeouts (decrease ≥15 min), gate on `isWarmingUp==false`. +- **Overshoots, then settles in a staircase** (in-flight pushsync keeps filling after uploads stop; + each decrease triggers pull-sync that blocks the next) → wait for pushsync drain and track the + high-water peak before treating a value as settled. +- **No fixed floor** (radius held at 1 for 30+ min) → assert *"a decrease occurred AND pull-sync + recovered"*, not *"radius returned to N"*. +- **Puller drives it** — `node/puller "radius decrease"` + a `syncWorker context cancelled` storm = + the PR #581 `manage()`/`disconnectPeer()` path (the liveness risk). + +## Phase 2 — The check (done) + +`pkg/check/reserveradius/` on branch `ljubisa-radius-check`. `Run` = `waitForWarmupDone` → +baseline → `driveIncrease` → settle → `observeDecrease` (assert a decrease vs per-node peak + +recovery). Registered as type `reserve-radius` (`pkg/config/check.go`); `ci-reserve-radius` in +`config/local.yaml`; added `IsWarmingUp` to `StatusResponse`. build/vet/lint green. + +- [x] **Validated end-to-end (2026-06-17)**: increase 0→1 in 3s, 1-min settle, decrease observed + at 13m16s with recovery, check passed (total 14m36s). Confirms the ~13-min decrease lag and + the 20-min `DecreaseTimeout`. + +## Phase 2.5 — driver/observer split + parallel (done) + +For long soaks (24h+), split the **uploader** (load) from the **observer** (reserve-radius) and +run them concurrently so the cluster oscillates repeatedly. + +- [x] **`--parallel` flag** (`check.go` + `runner.go`) — opt-in; checks run in goroutines and + fail independently (a monitor blip won't kill a 24h load). +- [x] **`load` re-arming `decrease-hold`** — after `committedDepth` hits the cap and drops, pause + uploads then re-arm → repeated fill→decrease→hold cycles. +- [x] **`reserve-radius` `mode: drive | observe`** — `observe` = `Duration`-bounded monitor (no + uploads) recording transitions + asserting recovery. Config: `ci-load-soak`, + `ci-reserve-radius-observe`. + +Run: `beekeeper check --checks=ci-load-soak,ci-reserve-radius-observe --parallel`. + +## Phase 3 — Real ephemeral cluster + +- [ ] Run the check against an unpatched ~20-node ephemeral cluster (realistic timing). +- [ ] Wire as a periodic / public-testnet RC check (per #591), not standard PR CI. ## A/B testing procedure (current vs fixed bee) -The single-build check only says "this build halted / didn't halt." To **prove the pull-sync -fix works**, run the *identical* radius-decrease scenario against two builds and diff them: - -- **A = current bee** (stock `master`) — the baseline; expected to **reproduce** the halt. -- **B = fixed bee** (@sig's SWIP-25 pull-sync PR) — expected to **survive** the same scenario. - -**Why both:** if B alone doesn't halt, that may just mean the scenario was too mild. Showing **A -halts first calibrates the test** — it proves the scenario reproduces the failure, so B surviving -is real evidence. The **A→B delta is the proof**. (It's the empirical complement to the design -doc's analytical + TLA+ proof — the implementation behaving as the design predicts on a real net.) - -**Procedure:** - -1. **Scenario fixed and deterministic** — same `RndSeed`, same `load`/decrease config, same cluster - size, keys, and data volume for both runs. Any difference confounds the comparison. -2. **Build two images** — `…/bee:current` (stock) and `…/bee:fixed` (the PR branch); push both - (the existing `docker-build`/patch mechanism). For A, no pull-sync fix; for B, the PR. -3. **Run the same scenario against each** (two clusters, or redeploy the same one per image) with - `--metrics-enabled`. **Label each run** so both land in one Prometheus — e.g. distinct pushgateway - job names or an external `build=current|fixed` label per deployment. -4. **Diff the halt indicators** (Grafana side-by-side, or a small report): - - | signal | A (current) expected | B (fixed) expected | - |---|---|---| - | freeze episodes / `is_playing_errors` rate | fire | none | - | un-recovered decreases (`fully_synced` timeout) | yes | no | - | `time_to_fully_synced_seconds` | huge / never | bounded | - | redistribution `round` | stuck / skipped | keeps advancing | - | `reserve_within_radius` (completeness) | stalls low | catches up | - | `offered/delivered` redundancy ratio | ~k | ~1 (SWIP-25 target) | - -**Gating / dependencies:** - -- **Scale** — only meaningful on the **~20-node cluster**: the halt is a neighbourhood *merge* - (k→8), so on 3 local nodes **neither** A nor B halts and there is nothing to diff. A/B is a - Phase-3 activity. -- **B must exist** — depends on @sig's PR being buildable; we don't control its timing. -- **Reproducing the halt in A** is itself part of the research — the scenario must be severe enough - (operator-offline → radius decrease) to trigger it reliably. -- Likely a **documented runbook or a thin wrapper command** (build A/B → run scenario twice → label - → compare), not new check logic — the check + metrics + dashboards already exist. +Run the *identical* radius-decrease scenario against two builds and diff them: + +- **A = current bee** (stock `master`) — expected to **reproduce** the halt. +- **B = fixed bee** (@sig's SWIP-25 PR) — expected to **survive** it. ## Open questions -- Increase, decrease, or both? Decrease exercises the known puller bug (#581) and - is the higher-value regression target; increase is the common steady-state path. -- Patch vs config: how much of the small-reserve setup can come from `bee-config` - knobs alone, avoiding a source patch entirely? -- Does `/redistributionstate` need full-mode + incentives enabled on the local - cluster, or do we assert purely on `/status` + `/reservestate`? -- Metrics sink for local runs: pushgateway vs scrape vs the CSV from `radius-poll.sh`. +- Patch vs config: can the small-reserve setup come from `bee-config` knobs alone? +- Does `/redistributionstate` need full-mode + incentives locally, or assert on `/status` + + `/reservestate` only? +- Metrics sink for local runs: pushgateway vs scrape vs `radius-poll.sh` CSV. ## References - Skill: `.claude/skills/radius-testing/SKILL.md` (+ `scripts/radius-poll.sh`) -- Prior art: PR #581, PR #591, `pkg/check/gc/reserve.go`, `pkg/check/pingpong/`, - `pkg/check/smoke/metrics.go` -- Bee internals: `bee/pkg/storer/reserve.go`, `bee/pkg/postage/batchstore/store.go`, - `bee/pkg/api/{status,postage,redistribution}.go` From 53c87f42cf4a1db24ce0565ed35adb2cec57c822 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 25 Jun 2026 12:15:28 +0200 Subject: [PATCH 09/37] chore: merge from master --- config/local.yaml | 1 + pkg/orchestration/node.go | 1 + 2 files changed, 2 insertions(+) diff --git a/config/local.yaml b/config/local.yaml index 85b8fd0a..d7a762a8 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -153,6 +153,7 @@ bee-configs: blockchain-rpc-endpoint: "ws://geth-swap:8546" bootnode-mode: false bootnode: [] + bzz-token-address: "0x6AAB14FE9cccd64A502d23842d916eB5321c26E7" cache-capacity: 20000 chequebook-enable: true cors-allowed-origins: [] diff --git a/pkg/orchestration/node.go b/pkg/orchestration/node.go index 12a16569..1dce71e0 100644 --- a/pkg/orchestration/node.go +++ b/pkg/orchestration/node.go @@ -101,6 +101,7 @@ type Config struct { BlockTime *uint64 `yaml:"block-time,omitempty"` // chain block time BootnodeMode *bool `yaml:"bootnode-mode,omitempty"` // cause the node to always accept incoming connections Bootnodes *[]string `yaml:"bootnode,omitempty"` // initial nodes to connect to + BzzTokenAddress *string `yaml:"bzz-token-address,omitempty"` // BZZ ERC20 token contract address (required for chains not built into bee) CacheCapacity *uint64 `yaml:"cache-capacity,omitempty"` // cache capacity in chunks, multiply by 4096 to get approximate capacity in bytes CacheRetrieval *bool `yaml:"cache-retrieval,omitempty"` // enable forwarded content caching ChequebookEnable *bool `yaml:"chequebook-enable,omitempty"` // enable chequebook From 388350132a5f8aa2704fc1aca956f3f397264ab7 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:17:18 +0200 Subject: [PATCH 10/37] feat(check): add stake-amount/stake-groups options to reserve-radius --- docs/radius-halt-check-plan.md | 261 +++++++++++++++++++++++ pkg/check/reserveradius/README.md | 2 + pkg/check/reserveradius/reserveradius.go | 4 + pkg/config/check.go | 2 + 4 files changed, 269 insertions(+) create mode 100644 docs/radius-halt-check-plan.md diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md new file mode 100644 index 00000000..49c24b5a --- /dev/null +++ b/docs/radius-halt-check-plan.md @@ -0,0 +1,261 @@ +# Reserve-radius check — halt-reproduction mode (plan) + +Status: **Plan only — not started.** Branch `ljubisa-radius-soak`. + +Goal: fold the now-working **manual** pull-sync-halt reproduction (see `docs/radius-halt.md`) +into the `reserveradius` check so the whole flow runs **unattended in one longer run** — +stake → drive radius to N → disrupt the neighbourhood (**node churn and/or batch expiry**) → observe +convergence/halt + the staked redistribution round-loss — and so the same check can serve the +**A/B harness** (stock master vs +`pullsync-optimal-design`). Optionally run alongside the `load` check in parallel. + +This plan only covers the **check code** (`pkg/check/reserveradius`). Environment setup — +building the patched bee image, deploying + funding the cluster — stays manual / in the +`cluster-up` flow and is out of scope here. + +## What the manual flow does today (the target to automate) + +From `docs/radius-halt.md` "Staked run", the reproduced recipe is: + +1. Stake every full node (`POST /stake/1e17`, verify `stakedAmount`). +2. Drive the radius to **3** with `ci-load-soak` (`max-committed-depth: 3`); stop the load once + all nodes report `storageRadius=3`. +3. **Remove 2 of 6 full nodes** (`kubectl delete statefulset/pod`) to disrupt the populated + neighbourhood. +4. Observe the 4 survivors ≥25 min: delayed onset (~8 min) → `within_radius` 0→~2200, + `fullySynced→false`, `pullsyncRate` decays toward 0, split-brain end-state, never re-converges. +5. Confirm the staked round-loss: `is_playing_errors` cluster-wide; un-synced nodes can't produce a + `ReserveSample`, so they win nothing while rounds keep advancing. + +The check already does step 2's driving (`mode: drive`) and a generic observe loop (`mode: observe`), +but has **no** staking, **no** node removal, and asserts only a *radius decrease + recovery* — not the +disruption-driven non-convergence we actually reproduce. + +## Capabilities confirmed (no blockers) + +- **Node removal — supported.** `orchestration.Node.Delete(ctx, namespace)` (deletes statefulset + + services/ingress/secret/configmap) and `Node.Stop(ctx, namespace)` (scales statefulset to 0); also + `NodeGroup.DeleteNode(ctx, name)`. The check reaches them via `cluster.Nodes()` / + `cluster.NodeGroup(name)`, and the namespace via `cluster.Namespace()`. +- **Staking — supported.** `bee.Client.DepositStake(ctx, *big.Int)` → `POST /stake/{amount}`, + `GetStake(ctx)` → `GET /stake`. The `stake` check already uses these via `cluster.NodesClients`. +- **Batch expiry — supported (the original October lever).** `CreatePostageBatch(ctx, amount, depth, + label)` (explicit amount → controllable expiry; batches are mutable) and `GetOrCreateMutableBatch` + (TTL→amount via chainstate price). Read expiry via `PostageStamp(batchID).BatchTTL`/`Exists` and + `GetChainState().TotalAmount`. Bee side (confirmed): expiry → `EvictBatch` → `evictExpiredBatches` + → commitment drop → gated radius decrease in `pkg/storer/reserve.go`. No in-repo `stampexpiry` + check yet (PR #591 is external); `pkg/check/gc/reserve.go` is the tiny-amount-batch precedent. +- **Redistribution round-loss — assertable WITHOUT `/metrics` scraping.** Checks can't scrape + Prometheus, but `/redistributionstate` already returns `Round`, `LastPlayedRound`, `LastWonRound`, + `LastSelectedRound`, `LastSampleDurationSeconds`, `IsFullySynced`, `IsFrozen`. Round-loss = + `Round` advances while `LastPlayedRound`/`LastWonRound` stalls (and `IsFullySynced=false`). The + check reads `/redistributionstate` today but ignores these fields. +- **Not available:** the `bee_pullsync_chunks_delivered` "delivered-plateau" smoking gun (needs a new + `/metrics` scrape on the bee client). The structured equivalents — `pullsyncRate` decay + + `reserveSizeWithinRadius` plateau + `fullySynced` stuck false — are enough; treat `/metrics` + scraping as a deferred nice-to-have (Phase 7). + +## Design + +A new self-driving **`mode: halt`** that composes the existing pieces plus two new stages, so one +run reproduces the whole recipe: + +``` +stake(optional) → waitWarmup → driveIncrease(to DisruptAtRadius) → settle + → disrupt(node-churn | batch-expiry) → observeOutcome(on survivors) +``` + +The disruption + staking + outcome-classification are written as **composable stages gated by +options**, so the parallel-with-load shape reuses them: `mode: observe` + a disruption mechanism means +"don't upload — let the `load` check drive the radius — but still stake, disrupt at the target, and +classify the outcome." + +**Both shapes are first-class (decided).** Write the stages composable so the radius can be driven +either by the check itself (`mode: halt`) **or** by a parallel `load` check (`mode: observe` + +disrupt). To avoid a fragile cross-check coordination channel, the observer **tolerates ongoing +uploads**: with `node-churn` the halt is a neighbourhood-redundancy drop, not a committed-depth effect, +so the assertions (`fullySynced`, round-loss) hold whether or not `load` keeps running. The manual run +*stopped* load only for a cleaner CSV; it is not required for the node-churn halt. **Caveat for +`batch-expiry`:** that lever works by *dropping* commitment, so ongoing `load` (which re-adds +commitment) counteracts it — in parallel-with-load, prefer `node-churn`, or have `load` wind down +around the expiry. Single-mode remains the most faithful reproduction; parallel-mode is the +soak/realism shape. + +**Disruption mechanisms (pluggable: node-churn and/or batch-expiry).** `disrupt-mechanism` chooses how +the neighbourhood is disrupted, both feeding the same observe/classify/verdict stage: +- `node-churn` (default) — stop/delete randomly-chosen full nodes. The reliable redundancy-drop lever + (what the manual repro used): removes neighbourhood replicas so survivors must re-replicate. +- `batch-expiry` — the **original October mechanism**. Create a short-lived postage batch, fill the + reserve under it, then let it expire: bee evicts its chunks cluster-wide (`evictExpiredBatches`), + commitment drops, and on a synced node below the decrease threshold the **storage radius decreases** + → neighbourhood merge → pull-sync re-sync. More faithful to October than node-churn, but the + decrease is **gated** (`SyncRate==0` + `countWithinRadius < threshold`) so it fires intermittently — + the `radius_threshold.patch` (threshold = capacity) makes it reachable on a small cluster. +- `both` — expire a batch *and* churn nodes for a harsher merge; `none` — monitor-only. + +**Tunable disruption + a non-failing verdict (decided).** Each mechanism's intensity is tunable +(`disrupt-node-count` for node-churn, **randomly selected** seeded by `rnd-seed`; `expiring-batch-ttl` +for batch-expiry) and the whole disruption can be turned off (`disrupt-mechanism: none`, or +`disrupt-node-count: 0`) so the check just monitors radius / sync / round state for `duration` (watch a +healthy cluster, or build a baseline). Either way the check **reports the outcome rather than failing +on it** — so the *same* check can reproduce the halt, deliberately *not* reproduce it (low intensity), +or just monitor, all without a red result. + +**Outcomes** the observe stage classifies and emits (a metric + an end-of-run summary line): +- `MONITORED` — `disrupt-node-count: 0`; no disruption, state recorded over `duration`. +- `HALT` — after disruption, sustained non-convergence (survivors stuck `fullySynced=false` past a + bound) and/or staked round-loss. +- `RECOVERED` — after disruption, all survivors returned to `fullySynced` (and resumed playing rounds) + within `recovery-wait`. + +**Verdict policy** (`verdict`): +- `report` (**default**) — always return success on any observed outcome; emit the outcome + metrics + + a clear summary. Fail **only** on operational errors (warmup timeout, stake/removal failure, all + nodes unreachable). This is the explore / soak / reproduce-or-not mode. +- `assert` — gate on the expectation: fail iff the observed outcome contradicts `expect-recovery` + (`false` ⇒ expect `HALT`, `true` ⇒ expect `RECOVERED`). For the A/B regression gate. `MONITORED` + (no disruption requested) is always a pass. + +## Phases + +### Phase 0 — Shape decisions (RESOLVED) +- [x] **Both shapes first-class**: `mode: halt` (self-driving) **and** `mode: observe` + disrupt + (parallel with `load`). Stages written composable to serve both. +- [x] **Parallel contract**: observer **tolerates ongoing uploads** (no cross-check IPC); assertions + are upload-robust. Stopping `load` is optional, not required. +- [x] **Node-removal method default = `stop`** (scale statefulset to 0; restorable; emptyDir means a + restart is a fresh node so restore-mid-run needs re-funding — a Phase-7 nicety). `delete` stays + available as an option. +- [x] **A-side semantics = PASS when halt reproduced** (`expect-recovery: false` is the default → + reproduction gate, not a regression detector). +- [x] **Tunable disruption + non-failing default**: `disrupt-node-count` randomly selects nodes + (`0` = monitor-only), and `verdict: report` (default) records `HALT`/`RECOVERED`/`MONITORED` + and only fails on operational errors. `verdict: assert` opts into A/B gating via `expect-recovery`. + +### Phase 1 — Staking pre-step +- [x] Add options: `StakeAmount` (config `stake-amount` string→parsed `*big.Int` in the stage, + e.g. `"100000000000000000"`; empty/`"0"` = skip), `StakeGroups` (`stake-groups`, default = + observed groups). Held as a `string` in `Options` (parsed at use-time) so config mapping and the + empty=skip sentinel stay trivial. +- [ ] New stage `ensureStaked`: for each selected node, `GetStake`; if `< StakeAmount`, + `DepositStake`; re-read to verify. Idempotent, logged per node. Tolerate "already staked". +- [ ] Budget the ~10-block usable wait if a fresh batch/stake needs confirmations. + +### Phase 2 — Drive (or wait) to the disruption radius +- [ ] Add `DisruptAtRadius` (default 3) distinct from `TargetRadius`. +- [ ] `mode: halt`: reuse `driveIncrease` to push **all** observed nodes (not just max) to + `DisruptAtRadius`; then run the existing `settle` window. +- [ ] `mode: observe`+disrupt: poll until every observed node reports + `storageRadius >= DisruptAtRadius` (load drives), with a timeout. +- [ ] Record a pre-disruption baseline snapshot (radius, within, fullySynced, round, stake). + +### Phase 3 — Disruption mechanisms (`disrupt-mechanism`) +Shared: a `disruption_total` metric + timestamp marks the onset reference. `disrupt-mechanism: none` +(or node-churn with `disrupt-node-count: 0`) skips straight to observe (monitor-only). + +**3a — node-churn** +- [ ] Add `DisruptNodeCount` (default 2; `0` = skip) and `DisruptMethod` (`stop` default | `delete`). +- [ ] **Randomly** select `DisruptNodeCount` nodes, seeded by `rnd-seed` (reproducible); in `halt` mode + exclude the uploader so driving still works. Build the **survivor set** the observe loop polls — + removed nodes drop out of polling. +- [ ] Remove via `node.Stop(ctx, cluster.Namespace())` / `node.Delete(...)`. Log + timestamp the + removal and the chosen node names. Emit `disruption_total`. +- [ ] Guard: require `len(survivors) >= MinSurvivors` (default 3); if the count would breach it, error + out **before** touching the cluster. + +**3b — batch-expiry** +- [ ] Create a soon-to-expire mutable batch — `CreatePostageBatch(ctx, amount, depth, label)` with a + small explicit `amount`, or `GetOrCreateMutableBatch` with a short `expiring-batch-ttl`. Drive the + radius (Phase 2) by uploading the fill **under this batch** so its chunks dominate the reserve. +- [ ] Detect expiry: poll `PostageStamp(batchID)` for `BatchTTL`→0 / `Exists=false`, cross-checking + `GetChainState().TotalAmount` against the batch `Amount`; timestamp it as onset. +- [ ] Observe eviction: `reserveSize` / `reserveSizeWithinRadius` drop as `evictExpiredBatches` runs; + then watch for the **gated** radius decrease (synced + `countthreshold OR + `fullySynced` true→false), the **stall** (`fullySynced` stuck false, `pullsyncRate` decaying, + `within_radius` plateau), and **round participation** from `/redistributionstate` + (`Round` advancing while `LastPlayedRound`/`LastWonRound` stalls = round-loss). +- [ ] Classify the run outcome → `MONITORED` | `HALT` | `RECOVERED`; emit it as a labelled metric and + a clear end-of-run summary (stuck vs recovered survivors, onset time, rounds lost). +- [ ] New metrics: `outcome` (labelled), `disruption_total`, `onset_seconds` (removal→onset), + `round_loss_total`, plus reuse `fully_synced`, `frozen`, `redistribution_round`, + `reserve_within_radius`, `pullsync_rate`, `time_to_fully_synced_seconds`. +- [ ] Apply the **verdict policy**: + - `report` (default) → always return success on the outcome; only operational errors fail. + - `assert` → fail iff the outcome contradicts `expect-recovery`; `MONITORED` always passes. +- [ ] Bound the whole observe by `Duration` (default ≥30 min; onset is delayed/variable ~3–9 min). + +### Phase 5 — Wire options, config, docs +- [ ] Add the new fields to `pkg/config/check.go` `"reserve-radius"` `NewOptions` struct + defaults in + `NewDefaultOptions`. +- [ ] Add config entries to `config/local.yaml`: `ci-radius-halt` (single self-driving) and, if Phase 0 + keeps it, a parallel pair reusing `ci-load-soak` + `ci-reserve-radius-observe`+disrupt. +- [ ] Update `pkg/check/reserveradius/README.md`, the `radius-testing` skill, and cross-link + `docs/radius-halt.md` ↔ this plan. + +### Phase 6 — A/B validation +- [ ] Run `ci-radius-halt` against **A** (patched stock master) → expect halt reproduced + (`expect-recovery: false` passes; matches the manual run's numbers). +- [ ] Exercise **both mechanisms** on A: `node-churn` (matches the manual repro) and `batch-expiry` + (the commitment-drop → radius-decrease → merge path); confirm each reaches `HALT` or, when the + decrease gate doesn't trip, `MONITORED` (not a failure). +- [ ] Build **B** (`pullsync-optimal-design` + the same `radius_*.patch`) and run with + `expect-recovery: true` → expect survivors re-converge. +- [ ] `make build vet lint test-race` green. No unit tests required for the check (per prior decision); + validate live on `local-dns`. + +### Phase 7 — Deferred: real signals & scale +- [ ] Optional `/metrics` scrape on the bee client to capture the `chunks_delivered` plateau and + `bee_storageincentives_is_playing_errors`/`winner` directly (richer round-loss proof). +- [ ] Run unpatched on a ~20-node ephemeral cluster (the merge needs scale); periodic/RC cadence. + +## New options (summary) + +| Option | Default | Purpose | +| --- | --- | --- | +| `mode` | `drive` | add `halt` (self-driving stake→drive→disrupt→observe) | +| `stake-amount` | `""` (skip) | per-node stake to ensure before driving (e.g. `100000000000000000`) | +| `disrupt-mechanism` | `node-churn` | `node-churn`, `batch-expiry`, `both`, or `none` (monitor-only) | +| `disrupt-at-radius` | `3` | radius all observed nodes must reach before disruption | +| `disrupt-node-count` | `2` | node-churn: full nodes to remove, **randomly** (seeded); `0` = none | +| `disrupt-method` | `stop` | node-churn: `stop` (scale-0, default) or `delete` (statefulset+resources) | +| `expiring-batch-ttl` | `0` | batch-expiry: TTL for the short-lived fill batch (0 = use a small explicit amount) | +| `min-survivors` | `3` | refuse to disrupt below this | +| `verdict` | `report` | `report` (never fail on outcome; only operational errors) or `assert` (gate on `expect-recovery`) | +| `expect-recovery` | `false` | **`assert` mode only**: expected outcome (false=`HALT`, true=`RECOVERED`) | +| `recovery-wait` | `10m` | per-survivor convergence bound after onset | +| `duration` | `30m` | total observe window (onset is delayed ~3–9 min) | + +## Decisions made (Phase 0) + +- Both shapes first-class (single `halt` + parallel `observe`+disrupt); observer tolerates ongoing load. +- Disruption is a **pluggable mechanism** — `node-churn` (default), `batch-expiry` (the October + commitment-drop → radius-decrease lever), `both`, or `none`. `disrupt-node-count` is tunable and + **randomly** selects nodes; any mechanism can be turned off for monitor-only. +- Default `verdict: report` → the check **records the outcome (`HALT`/`RECOVERED`/`MONITORED`) and + does not fail** whether the halt reproduces, doesn't, or no disruption was requested. Only + operational errors fail. `verdict: assert` + `expect-recovery` opts into the A/B regression gate. +- Node removal default = `stop` (scale-0); `delete` available as an option. + +## Remaining open question + +- **`/metrics` scraping** (Phase 7): add a `/metrics` scrape on the bee client now for the + `chunks_delivered` plateau + `bee_storageincentives_is_playing_errors`/`winner` (richer round-loss + proof), or ship v1 on structured `/redistributionstate` signals only? Plan currently **defers** it. + +## References + +- `docs/radius-halt.md` — the reproduced recipe + measured numbers (mechanism, steps, round-loss, A/B). +- `docs/radius-check-plan.md` — the original check design (Phases 0–2.5 done). +- `pkg/check/reserveradius/` — current check (drive/observe modes). +- `pkg/check/stake/stake.go` — staking-API usage precedent. +- `pkg/orchestration/{node,nodegroup,cluster}.go` + `k8s/orchestrator.go` — node Delete/Stop. +- `pkg/bee/api/postage.go` + `client.go` (`CreatePostageBatch`, `GetOrCreateMutableBatch`, + `PostageStamp`, `GetChainState`) — batch creation + expiry/chainstate reads. +- bee `pkg/postage/batchstore/store.go` (`cleanup`→`evictFn`) + `pkg/storer/reserve.go` + (`evictExpiredBatches` → gated radius decrease) — the expiry→eviction→decrease path. +- `.claude/skills/radius-testing/` — operating know-how + `scripts/radius-poll.sh`. diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index b2a925e4..6d7d3383 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -65,6 +65,8 @@ Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in | `Duration` | `duration` | `12h` | observe | total monitor run length | | `RecoveryWait` | `recovery-wait` | `5m` | observe | max wait for pull-sync recovery after each decrease | | `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | both | seed for `random.PseudoGenerator` → shuffled node pick | +| `StakeAmount` | `stake-amount` | `""` (skip) | both | per-node stake (wei) to ensure before driving, e.g. `"100000000000000000"`; empty/`"0"` skips | +| `StakeGroups` | `stake-groups` | `nil` (observed) | both | node groups to stake (empty = the observed/selected groups) | | `UploadGroups` | `upload-groups` | `[bee]` | both | node groups to observe (and, in drive, upload to) | | `PollInterval` | `poll-interval` | `15s` | both | poll cadence | | `PostageTTL` | `postage-ttl` | `24h` | drive | batch TTL (use TTL, not a raw amount) | diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 74ac0901..72b71ced 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -37,6 +37,8 @@ type Options struct { Mode string // "drive" (upload to force a change) or "observe" (monitor a change driven externally) Duration time.Duration // observe mode: total monitor run length RndSeed int64 + StakeAmount string // per-node stake to ensure before driving (wei, e.g. "100000000000000000"); empty/"0" = skip + StakeGroups []string // node groups to stake (empty = the observed/selected groups) PostageTTL time.Duration PostageDepth uint64 PostageLabel string @@ -58,6 +60,8 @@ func NewDefaultOptions() Options { Mode: ModeDrive, Duration: 12 * time.Hour, RndSeed: time.Now().UnixNano(), + StakeAmount: "", // skip staking unless set + StakeGroups: nil, PostageTTL: 24 * time.Hour, PostageDepth: 22, PostageLabel: "reserve-radius", diff --git a/pkg/config/check.go b/pkg/config/check.go index be7d7445..46456101 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -504,6 +504,8 @@ var Checks = map[string]CheckType{ Mode *string `yaml:"mode"` Duration *time.Duration `yaml:"duration"` RndSeed *int64 `yaml:"rnd-seed"` + StakeAmount *string `yaml:"stake-amount"` + StakeGroups *[]string `yaml:"stake-groups"` PostageTTL *time.Duration `yaml:"postage-ttl"` PostageDepth *uint64 `yaml:"postage-depth"` PostageLabel *string `yaml:"postage-label"` From 4bfbdccf7643e1d5a301b32768d87505d2aa36a1 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:20:53 +0200 Subject: [PATCH 11/37] feat(check): add ensureStaked pre-step with on-chain confirm to reserve-radius --- docs/radius-halt-check-plan.md | 10 +- pkg/check/reserveradius/README.md | 9 +- pkg/check/reserveradius/reserveradius.go | 164 ++++++++++++++++++----- pkg/config/check.go | 37 ++--- 4 files changed, 162 insertions(+), 58 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 49c24b5a..59969db4 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -137,9 +137,13 @@ or just monitor, all without a red result. e.g. `"100000000000000000"`; empty/`"0"` = skip), `StakeGroups` (`stake-groups`, default = observed groups). Held as a `string` in `Options` (parsed at use-time) so config mapping and the empty=skip sentinel stay trivial. -- [ ] New stage `ensureStaked`: for each selected node, `GetStake`; if `< StakeAmount`, - `DepositStake`; re-read to verify. Idempotent, logged per node. Tolerate "already staked". -- [ ] Budget the ~10-block usable wait if a fresh batch/stake needs confirmations. +- [x] New stage `ensureStaked` (+`ensureNodeStaked`): for each node in the staking set (`StakeGroups`, + or the observed nodes), `GetStake`; if `< StakeAmount`, `DepositStake`; re-read to verify. + Idempotent (already-at-target nodes skipped), logged per node. Wired before warmup in both + `runDrive` and `runObserve`, gated on `stake-amount` set. +- [x] Budget the ~10-block usable wait via `waitStakeAtLeast` + `stake-confirm-wait` (default `2m`): + `POST /stake` returns a tx hash, so poll `GetStake` until the deposit confirms on-chain (or the + budget expires). Already-mined deposits pass immediately. ### Phase 2 — Drive (or wait) to the disruption radius - [ ] Add `DisruptAtRadius` (default 3) distinct from `TargetRadius`. diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 6d7d3383..3d6c3ac8 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -20,7 +20,13 @@ It has two modes (`Options.Mode`): ## What it does (`Run` flow) -`Run` dispatches on `Mode`. +`Run` dispatches on `Mode`. Both modes first run an optional **staking pre-step**. + +**staking pre-step** (`ensureStaked`, both modes): when `stake-amount` is set, ensure every node in +the staking set (`stake-groups`, or the observed nodes) has at least that stake — `GetStake`, and if +below, `DepositStake` then poll until the deposit confirms on-chain (within `stake-confirm-wait`). +Idempotent: nodes already at/above target are skipped. Runs before warmup so deposits confirm while +the cluster stabilizes. **`drive`** (`runDrive`): @@ -67,6 +73,7 @@ Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in | `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | both | seed for `random.PseudoGenerator` → shuffled node pick | | `StakeAmount` | `stake-amount` | `""` (skip) | both | per-node stake (wei) to ensure before driving, e.g. `"100000000000000000"`; empty/`"0"` skips | | `StakeGroups` | `stake-groups` | `nil` (observed) | both | node groups to stake (empty = the observed/selected groups) | +| `StakeConfirmWait` | `stake-confirm-wait` | `2m` | both | max wait for a deposit to confirm on-chain (~10 blocks) | | `UploadGroups` | `upload-groups` | `[bee]` | both | node groups to observe (and, in drive, upload to) | | `PollInterval` | `poll-interval` | `15s` | both | poll cadence | | `PostageTTL` | `postage-ttl` | `24h` | drive | batch TTL (use TTL, not a raw amount) | diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 72b71ced..706e0ef3 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -21,6 +21,8 @@ import ( crand "crypto/rand" "errors" "fmt" + "math/big" + "strings" "time" "github.com/ethersphere/beekeeper/pkg/bee" @@ -34,47 +36,49 @@ import ( // Options represents reserve-radius check options. type Options struct { - Mode string // "drive" (upload to force a change) or "observe" (monitor a change driven externally) - Duration time.Duration // observe mode: total monitor run length - RndSeed int64 - StakeAmount string // per-node stake to ensure before driving (wei, e.g. "100000000000000000"); empty/"0" = skip - StakeGroups []string // node groups to stake (empty = the observed/selected groups) - PostageTTL time.Duration - PostageDepth uint64 - PostageLabel string - UploadGroups []string // node groups to upload to / observe (empty = all full nodes) - BlobSize int64 // bytes per upload - MaxUploads int // cap on uploads during the increase phase - TargetRadius uint8 // storageRadius to reach before stopping uploads - WarmupWait time.Duration // max wait for nodes to leave warmup before staging - IncreaseTimeout time.Duration // max time to reach TargetRadius - SettleWait time.Duration // wait after uploads for pushsync overshoot to drain - DecreaseTimeout time.Duration // max time to observe a decrease after uploads stop - RecoveryWait time.Duration // observe mode: max wait for pull-sync recovery after each decrease - PollInterval time.Duration + Mode string // "drive" (upload to force a change) or "observe" (monitor a change driven externally) + Duration time.Duration // observe mode: total monitor run length + RndSeed int64 + StakeAmount string // per-node stake to ensure before driving (wei, e.g. "100000000000000000"); empty/"0" = skip + StakeGroups []string // node groups to stake (empty = the observed/selected groups) + StakeConfirmWait time.Duration // max wait for a deposit to reflect on-chain (~10 blocks) + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string + UploadGroups []string // node groups to upload to / observe (empty = all full nodes) + BlobSize int64 // bytes per upload + MaxUploads int // cap on uploads during the increase phase + TargetRadius uint8 // storageRadius to reach before stopping uploads + WarmupWait time.Duration // max wait for nodes to leave warmup before staging + IncreaseTimeout time.Duration // max time to reach TargetRadius + SettleWait time.Duration // wait after uploads for pushsync overshoot to drain + DecreaseTimeout time.Duration // max time to observe a decrease after uploads stop + RecoveryWait time.Duration // observe mode: max wait for pull-sync recovery after each decrease + PollInterval time.Duration } // NewDefaultOptions returns new default options. func NewDefaultOptions() Options { return Options{ - Mode: ModeDrive, - Duration: 12 * time.Hour, - RndSeed: time.Now().UnixNano(), - StakeAmount: "", // skip staking unless set - StakeGroups: nil, - PostageTTL: 24 * time.Hour, - PostageDepth: 22, - PostageLabel: "reserve-radius", - UploadGroups: []string{"bee"}, - BlobSize: 1 << 20, // 1 MiB - MaxUploads: 60, - TargetRadius: 1, - WarmupWait: 15 * time.Minute, - IncreaseTimeout: 5 * time.Minute, - SettleWait: time.Minute, - DecreaseTimeout: 20 * time.Minute, - RecoveryWait: 5 * time.Minute, - PollInterval: 15 * time.Second, + Mode: ModeDrive, + Duration: 12 * time.Hour, + RndSeed: time.Now().UnixNano(), + StakeAmount: "", // skip staking unless set + StakeGroups: nil, + StakeConfirmWait: 2 * time.Minute, + PostageTTL: 24 * time.Hour, + PostageDepth: 22, + PostageLabel: "reserve-radius", + UploadGroups: []string{"bee"}, + BlobSize: 1 << 20, // 1 MiB + MaxUploads: 60, + TargetRadius: 1, + WarmupWait: 15 * time.Minute, + IncreaseTimeout: 5 * time.Minute, + SettleWait: time.Minute, + DecreaseTimeout: 20 * time.Minute, + RecoveryWait: 5 * time.Minute, + PollInterval: 15 * time.Second, } } @@ -136,6 +140,85 @@ func (c *Check) selectNodes(ctx context.Context, cluster orchestration.Cluster, return nodes, nil } +// ensureStaked makes sure every node in the staking set has at least StakeAmount +// staked, depositing the shortfall and confirming on-chain. It is idempotent: a +// node already at/above the target is skipped, and "already staked" is tolerated. +// Skips entirely when StakeAmount is empty or "0". The staking set is StakeGroups +// (full nodes filtered) or, when unset, the observed nodes. +func (c *Check) ensureStaked(ctx context.Context, cluster orchestration.Cluster, observed orchestration.ClientList, o Options) error { + amount := strings.TrimSpace(o.StakeAmount) + if amount == "" || amount == "0" { + return nil // staking disabled + } + want, ok := new(big.Int).SetString(amount, 10) + if !ok || want.Sign() <= 0 { + return fmt.Errorf("invalid stake-amount %q (want a positive base-10 wei integer)", o.StakeAmount) + } + + nodes := observed + if len(o.StakeGroups) > 0 { + rnd := random.PseudoGenerator(o.RndSeed) + full, err := cluster.ShuffledFullNodeClients(ctx, rnd) + if err != nil { + return fmt.Errorf("get full node clients for staking: %w", err) + } + nodes = full.FilterByNodeGroups(o.StakeGroups) + } + if len(nodes) == 0 { + return errors.New("ensureStaked: no nodes selected for staking") + } + + c.logger.Infof("ensureStaked: ensuring >= %s wei staked on %d node(s)", want, len(nodes)) + for _, n := range nodes { + if err := c.ensureNodeStaked(ctx, n, want, o); err != nil { + return fmt.Errorf("ensureStaked %s: %w", n.Name(), err) + } + } + return nil +} + +// ensureNodeStaked tops a single node up to want (if below) and confirms the deposit. +func (c *Check) ensureNodeStaked(ctx context.Context, n *bee.Client, want *big.Int, o Options) error { + cur, err := n.GetStake(ctx) + if err != nil { + return fmt.Errorf("get stake: %w", err) + } + if cur.Cmp(want) >= 0 { + c.logger.Infof("ensureStaked: %s already staked %s wei (>= %s) — skip", n.Name(), cur, want) + return nil + } + c.logger.Infof("ensureStaked: %s staked %s wei < target %s — depositing", n.Name(), cur, want) + if _, err := n.DepositStake(ctx, want); err != nil { + return fmt.Errorf("deposit stake: %w", err) + } + // POST /stake returns a tx hash; the staked amount may not reflect until the tx + // mines (~a few blocks), so poll until it confirms (or already mined → immediate). + return c.waitStakeAtLeast(ctx, n, want, o) +} + +// waitStakeAtLeast polls a node's stake until it reaches want or the budget expires. +func (c *Check) waitStakeAtLeast(ctx context.Context, n *bee.Client, want *big.Int, o Options) error { + deadline := time.Now().Add(o.StakeConfirmWait) + var last error + for { + cur, err := n.GetStake(ctx) + last = err + if err == nil && cur.Cmp(want) >= 0 { + c.logger.Infof("ensureStaked: %s confirmed staked %s wei", n.Name(), cur) + return nil + } + if time.Now().After(deadline) { + if last != nil { + return fmt.Errorf("stake not confirmed within %s: %w", o.StakeConfirmWait, last) + } + return fmt.Errorf("stake not confirmed within %s (still below %s wei)", o.StakeConfirmWait, want) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + // runDrive uploads to force a radius increase, then observes the decrease + recovery. func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o Options) error { if o.TargetRadius == 0 { @@ -148,6 +231,11 @@ func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o O uploader := nodes[0] // a random node, since the list is shuffled c.logger.Infof("mode=drive uploader: %s, observing %d node(s)", uploader.Name(), len(nodes)) + // 0. Stake (optional) — runs before warmup so deposits confirm while we wait. + if err := c.ensureStaked(ctx, cluster, nodes, o); err != nil { + return err + } + // 1. Wait for warmup/stabilization — the decrease loop is gated on it. if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { return err @@ -206,6 +294,10 @@ func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o } c.logger.Infof("mode=observe monitoring %d node(s) for %s (no uploads; drive the radius externally, e.g. the load check)", len(nodes), o.Duration) + if err := c.ensureStaked(ctx, cluster, nodes, o); err != nil { + return err + } + if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { return err } diff --git a/pkg/config/check.go b/pkg/config/check.go index 46456101..f2bf0d11 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -501,24 +501,25 @@ var Checks = map[string]CheckType{ NewAction: reserveradius.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - Mode *string `yaml:"mode"` - Duration *time.Duration `yaml:"duration"` - RndSeed *int64 `yaml:"rnd-seed"` - StakeAmount *string `yaml:"stake-amount"` - StakeGroups *[]string `yaml:"stake-groups"` - PostageTTL *time.Duration `yaml:"postage-ttl"` - PostageDepth *uint64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` - UploadGroups *[]string `yaml:"upload-groups"` - BlobSize *int64 `yaml:"blob-size"` - MaxUploads *int `yaml:"max-uploads"` - TargetRadius *uint8 `yaml:"target-radius"` - WarmupWait *time.Duration `yaml:"warmup-wait"` - IncreaseTimeout *time.Duration `yaml:"increase-timeout"` - SettleWait *time.Duration `yaml:"settle-wait"` - DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` - RecoveryWait *time.Duration `yaml:"recovery-wait"` - PollInterval *time.Duration `yaml:"poll-interval"` + Mode *string `yaml:"mode"` + Duration *time.Duration `yaml:"duration"` + RndSeed *int64 `yaml:"rnd-seed"` + StakeAmount *string `yaml:"stake-amount"` + StakeGroups *[]string `yaml:"stake-groups"` + StakeConfirmWait *time.Duration `yaml:"stake-confirm-wait"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + UploadGroups *[]string `yaml:"upload-groups"` + BlobSize *int64 `yaml:"blob-size"` + MaxUploads *int `yaml:"max-uploads"` + TargetRadius *uint8 `yaml:"target-radius"` + WarmupWait *time.Duration `yaml:"warmup-wait"` + IncreaseTimeout *time.Duration `yaml:"increase-timeout"` + SettleWait *time.Duration `yaml:"settle-wait"` + DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + RecoveryWait *time.Duration `yaml:"recovery-wait"` + PollInterval *time.Duration `yaml:"poll-interval"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From d75645f803a5a66711bfb7a6c1219fa46e03926f Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:26:25 +0200 Subject: [PATCH 12/37] feat(check): add halt mode driving all nodes to disrupt-at-radius --- docs/radius-halt-check-plan.md | 7 +- pkg/check/reserveradius/README.md | 22 +++- pkg/check/reserveradius/reserveradius.go | 124 +++++++++++++++++++++-- pkg/config/check.go | 1 + 4 files changed, 137 insertions(+), 17 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 59969db4..1858d05a 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -146,9 +146,10 @@ or just monitor, all without a red result. budget expires). Already-mined deposits pass immediately. ### Phase 2 — Drive (or wait) to the disruption radius -- [ ] Add `DisruptAtRadius` (default 3) distinct from `TargetRadius`. -- [ ] `mode: halt`: reuse `driveIncrease` to push **all** observed nodes (not just max) to - `DisruptAtRadius`; then run the existing `settle` window. +- [x] Add `DisruptAtRadius` (default 3, `disrupt-at-radius`) distinct from `TargetRadius`. +- [x] `mode: halt` (`runHalt`): stake → warmup → `driveAllToRadius` (gates on the **min** node radius, + not the max, so ALL observed nodes reach `DisruptAtRadius`) → settle window. `updatePeak` now + returns `(min, max)`. Disruption (Phase 3) + outcome observation (Phase 4) are stubbed with TODOs. - [ ] `mode: observe`+disrupt: poll until every observed node reports `storageRadius >= DisruptAtRadius` (load drives), with a timeout. - [ ] Record a pre-disruption baseline snapshot (radius, within, fullySynced, round, stake). diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 3d6c3ac8..eaf42696 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -5,7 +5,7 @@ recovers afterwards — catching regressions in the puller's reaction to a radiu change (the `manage()` / `disconnectPeer()` path that caused the liveness bug in beekeeper PR #581). -It has two modes (`Options.Mode`): +It has three modes (`Options.Mode`): - **`drive`** (default) — a one-shot run: upload to force the radius up, then watch it come back down and assert pull-sync recovers. @@ -13,6 +13,10 @@ It has two modes (`Options.Mode`): transitions for `Duration` while something else drives the cluster (typically the `load` check running in parallel via `--parallel`), recording every up/down transition and asserting recovery after each decrease. This is the soak mode. +- **`halt`** — self-driving pull-sync-halt reproduction: stake → drive **all** observed + nodes to `DisruptAtRadius` → settle → disrupt the neighbourhood → observe the outcome. + Disruption mechanisms and outcome classification are being built out (Phases 3–4); + currently it runs the stake/drive/settle prefix. > Keep this file in sync with the code. If you change `Options`, the `Run` flow, the > emitted metrics, the registration, or the bee-patch requirement, update the matching @@ -60,6 +64,17 @@ October failure: Un-recovered decreases or freezes fail the check at the end. Never uploads (the radius is driven externally, e.g. by a parallel `load` check). +**`halt`** (`runHalt`): the self-driving reproduction. + +1. **stake** (optional `ensureStaked` pre-step, as above). +2. **`waitForWarmupDone`** + **baseline** snapshot. +3. **`driveAllToRadius`** — buy a mutable batch and upload until **every** observed node's + `storageRadius` reaches `DisruptAtRadius` (gates on the **min** across nodes, so the whole + neighbourhood is populated before disruption), or `MaxUploads` / `IncreaseTimeout`. +4. **settle** — poll for `SettleWait`. +5. **disrupt + observe-outcome** — Phase 3/4 (node-churn / batch-expiry, then classify + `HALT`/`RECOVERED`/`MONITORED` and apply the verdict). Currently stubbed with TODOs. + ## Options Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in @@ -67,7 +82,7 @@ Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in | field | yaml | default | mode | purpose | | --- | --- | --- | --- | --- | -| `Mode` | `mode` | `drive` | both | `drive` (upload to force a change) or `observe` (monitor only) | +| `Mode` | `mode` | `drive` | all | `drive` (force a change), `observe` (monitor only), or `halt` (self-driving reproduction) | | `Duration` | `duration` | `12h` | observe | total monitor run length | | `RecoveryWait` | `recovery-wait` | `5m` | observe | max wait for pull-sync recovery after each decrease | | `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | both | seed for `random.PseudoGenerator` → shuffled node pick | @@ -81,7 +96,8 @@ Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in | `PostageLabel` | `postage-label` | `reserve-radius` | drive | batch label | | `BlobSize` | `blob-size` | `1048576` (1 MiB) | drive | bytes per upload | | `MaxUploads` | `max-uploads` | `60` | drive | cap on uploads in the increase phase | -| `TargetRadius` | `target-radius` | `1` | drive | storageRadius to reach before stopping uploads | +| `TargetRadius` | `target-radius` | `1` | drive | storageRadius any node must reach before stopping uploads | +| `DisruptAtRadius` | `disrupt-at-radius` | `3` | halt | storageRadius **all** observed nodes must reach before disruption | | `WarmupWait` | `warmup-wait` | `15m` | both | max wait for nodes to finish warmup | | `IncreaseTimeout` | `increase-timeout` | `5m` | drive | max time to reach `TargetRadius` | | `SettleWait` | `settle-wait` | `1m` | drive | post-upload window (pushsync drain + peak tracking) | diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 706e0ef3..4c9fdfaa 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -48,7 +48,8 @@ type Options struct { UploadGroups []string // node groups to upload to / observe (empty = all full nodes) BlobSize int64 // bytes per upload MaxUploads int // cap on uploads during the increase phase - TargetRadius uint8 // storageRadius to reach before stopping uploads + TargetRadius uint8 // drive mode: storageRadius any node must reach before stopping uploads + DisruptAtRadius uint8 // halt mode: storageRadius ALL observed nodes must reach before disruption WarmupWait time.Duration // max wait for nodes to leave warmup before staging IncreaseTimeout time.Duration // max time to reach TargetRadius SettleWait time.Duration // wait after uploads for pushsync overshoot to drain @@ -73,6 +74,7 @@ func NewDefaultOptions() Options { BlobSize: 1 << 20, // 1 MiB MaxUploads: 60, TargetRadius: 1, + DisruptAtRadius: 3, WarmupWait: 15 * time.Minute, IncreaseTimeout: 5 * time.Minute, SettleWait: time.Minute, @@ -103,10 +105,12 @@ func NewCheck(log logging.Logger) beekeeper.Action { const ( ModeDrive = "drive" // upload to force a radius change, then observe the decrease ModeObserve = "observe" // monitor radius changes driven externally (e.g. by the load check) + ModeHalt = "halt" // self-driving: stake → drive all nodes → disrupt → observe outcome ) -// Run dispatches on Mode: drive (force a change and observe it) or observe -// (monitor changes driven externally, e.g. by a parallel load check). +// Run dispatches on Mode: drive (force a change and observe it), observe +// (monitor changes driven externally, e.g. by a parallel load check), or halt +// (self-driving stake → drive → disrupt → observe-outcome reproduction). func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { o, ok := opts.(Options) if !ok { @@ -117,8 +121,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return c.runDrive(ctx, cluster, o) case ModeObserve: return c.runObserve(ctx, cluster, o) + case ModeHalt: + return c.runHalt(ctx, cluster, o) default: - return fmt.Errorf("invalid mode %q (want %q or %q)", o.Mode, ModeDrive, ModeObserve) + return fmt.Errorf("invalid mode %q (want %q, %q or %q)", o.Mode, ModeDrive, ModeObserve, ModeHalt) } } @@ -271,6 +277,60 @@ func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o O return c.observeDecrease(ctx, nodes, peak, o) } +// runHalt self-drives the whole reproduction: stake (optional) → warmup → drive +// ALL observed nodes to DisruptAtRadius → settle → disrupt → observe the outcome. +// The disruption (Phase 3) and outcome classification/verdict (Phase 4) stages +// land in later cycles; for now it completes the stake/drive/settle prefix. +func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Options) error { + if o.DisruptAtRadius == 0 { + return errors.New("disrupt-at-radius must be > 0") + } + nodes, err := c.selectNodes(ctx, cluster, o) + if err != nil { + return err + } + uploader := nodes[0] // a random node, since the list is shuffled + c.logger.Infof("mode=halt uploader: %s, %d observed node(s), disrupt-at-radius=%d", uploader.Name(), len(nodes), o.DisruptAtRadius) + + // 0. Stake (optional) — runs before warmup so deposits confirm while we wait. + if err := c.ensureStaked(ctx, cluster, nodes, o); err != nil { + return err + } + + // 1. Wait for warmup/stabilization. + if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { + return err + } + c.snapshot(ctx, nodes, "baseline") + + // 2. Drive every observed node up to DisruptAtRadius (a populated neighbourhood + // is the precondition for the disruption to bite). + batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + return fmt.Errorf("create batch on %s: %w", uploader.Name(), err) + } + c.logger.WithField("batch_id", batchID).Infof("node %s: using batch", uploader.Name()) + peak := make(map[string]uint8, len(nodes)) + if err := c.driveAllToRadius(ctx, uploader, nodes, batchID, o, peak); err != nil { + return err + } + + // 3. Settle window — let pushsync overshoot drain before disrupting. + c.logger.Infof("halt: all nodes at radius %d; settling for %s before disruption", o.DisruptAtRadius, o.SettleWait) + settleDeadline := time.Now().Add(o.SettleWait) + for time.Now().Before(settleDeadline) { + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + c.updatePeak(ctx, nodes, peak) + } + + // TODO(Phase 3): disrupt the neighbourhood (node-churn / batch-expiry). + // TODO(Phase 4): observe the outcome, classify HALT|RECOVERED|MONITORED, apply the verdict. + c.logger.Info("halt: stake+drive+settle complete; disruption and outcome observation land in Phase 3/4") + return nil +} + // obsNode is the per-node state the observe monitor tracks across polls. type obsNode struct { lastRadius uint8 @@ -449,7 +509,7 @@ func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes o c.logger.Errorf("upload #%d failed: %v", i, err) continue } - mx := c.updatePeak(ctx, nodes, peak) + _, mx := c.updatePeak(ctx, nodes, peak) c.logger.Infof("increase: upload #%d, max storageRadius=%d (target %d)", i, mx, o.TargetRadius) if mx >= o.TargetRadius { c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) @@ -461,6 +521,43 @@ func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes o return fmt.Errorf("storageRadius did not reach %d after %d uploads", o.TargetRadius, o.MaxUploads) } +// driveAllToRadius uploads blobs to the uploader until EVERY observed node reaches +// DisruptAtRadius (gates on the min, not the max — the whole neighbourhood must be +// populated before disruption). Otherwise it mirrors driveIncrease. +func (c *Check) driveAllToRadius(ctx context.Context, uploader *bee.Client, nodes orchestration.ClientList, batchID string, o Options, peak map[string]uint8) error { + c.logger.Infof("halt drive: %d-byte blobs to %s until ALL %d node(s) storageRadius>=%d (max %d uploads, timeout %s)", + o.BlobSize, uploader.Name(), len(nodes), o.DisruptAtRadius, o.MaxUploads, o.IncreaseTimeout) + start := time.Now() + deadline := start.Add(o.IncreaseTimeout) + t := test.NewTest(c.logger) + data := make([]byte, o.BlobSize) + + for i := 1; i <= o.MaxUploads; i++ { + if err := ctx.Err(); err != nil { + return err + } + if time.Now().After(deadline) { + return fmt.Errorf("not all nodes reached storageRadius %d within %s (%d uploads) — is the bee reserve patch active and is there enough data?", o.DisruptAtRadius, o.IncreaseTimeout, i-1) + } + if _, err := crand.Read(data); err != nil { + return fmt.Errorf("generate random data: %w", err) + } + if _, _, err := t.Upload(ctx, uploader, data, batchID, nil); err != nil { + c.logger.Errorf("upload #%d failed: %v", i, err) + continue + } + mn, mx := c.updatePeak(ctx, nodes, peak) + c.logger.Infof("halt drive: upload #%d, min storageRadius=%d max=%d (target %d)", i, mn, mx, o.DisruptAtRadius) + if mn >= o.DisruptAtRadius { + c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) + c.logger.Infof("all nodes reached storageRadius %d after %d uploads (~%.1f MiB) in %s", + o.DisruptAtRadius, i, float64(int64(i)*o.BlobSize)/(1<<20), time.Since(start).Round(time.Second)) + return nil + } + } + return fmt.Errorf("not all nodes reached storageRadius %d after %d uploads", o.DisruptAtRadius, o.MaxUploads) +} + // observeDecrease watches for any node's radius to fall below its peak, and for // pull-sync to recover (pullsyncRate>0). No decrease within the timeout is the // failure signal (cf. PR #581 puller manage()/disconnectPeer() stall). @@ -528,9 +625,10 @@ func (c *Check) snapshot(ctx context.Context, nodes orchestration.ClientList, ph } // updatePeak polls each node, emits metrics, raises the per-node high-water peak, -// logs a line, and returns the current max storageRadius across nodes. -func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8) uint8 { - var mx uint8 +// logs a line, and returns the min and max current storageRadius across the nodes +// that responded (min is 0 if none responded). +func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8) (minR, maxR uint8) { + seen := false for _, n := range nodes { s, err := n.Status(ctx) if err != nil { @@ -541,13 +639,17 @@ func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, if s.StorageRadius > peak[n.Name()] { peak[n.Name()] = s.StorageRadius } - if s.StorageRadius > mx { - mx = s.StorageRadius + if s.StorageRadius > maxR { + maxR = s.StorageRadius + } + if !seen || s.StorageRadius < minR { + minR = s.StorageRadius + seen = true } c.logger.Infof("%s: storageRadius=%d (peak %d) reserveSize=%d withinR=%d pullsyncRate=%.4f", n.Name(), s.StorageRadius, peak[n.Name()], s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate) } - return mx + return minR, maxR } func (c *Check) emit(node string, s *api.StatusResponse) { diff --git a/pkg/config/check.go b/pkg/config/check.go index f2bf0d11..c5490a07 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -514,6 +514,7 @@ var Checks = map[string]CheckType{ BlobSize *int64 `yaml:"blob-size"` MaxUploads *int `yaml:"max-uploads"` TargetRadius *uint8 `yaml:"target-radius"` + DisruptAtRadius *uint8 `yaml:"disrupt-at-radius"` WarmupWait *time.Duration `yaml:"warmup-wait"` IncreaseTimeout *time.Duration `yaml:"increase-timeout"` SettleWait *time.Duration `yaml:"settle-wait"` From c892ddadd2d573e513bb0a7adcb0d0204d4d5a66 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:30:18 +0200 Subject: [PATCH 13/37] feat(check): add pre-disruption baselineSnapshot to halt mode --- docs/radius-halt-check-plan.md | 13 ++++++- pkg/check/reserveradius/README.md | 3 +- pkg/check/reserveradius/reserveradius.go | 44 +++++++++++++++++++++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 1858d05a..2079c809 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -152,12 +152,23 @@ or just monitor, all without a red result. returns `(min, max)`. Disruption (Phase 3) + outcome observation (Phase 4) are stubbed with TODOs. - [ ] `mode: observe`+disrupt: poll until every observed node reports `storageRadius >= DisruptAtRadius` (load drives), with a timeout. -- [ ] Record a pre-disruption baseline snapshot (radius, within, fullySynced, round, stake). + **(Resequenced → Phase 3: the wait-to-radius helper is dead code until the observe+disrupt + dispatch exists, which needs the Phase-3 `disrupt-mechanism`/`disrupt-node-count` options.)** +- [x] Record a pre-disruption baseline snapshot (`baselineSnapshot`): per node, storageRadius + + reserveSizeWithinRadius (/status), isFullySynced + round/lastPlayed/lastWon + (/redistributionstate), and stake — emitted + logged. Wired into `runHalt` (replaces the plain + `snapshot("baseline")`). Phase 4 will extend it to return the values for onset/round-loss compare. ### Phase 3 — Disruption mechanisms (`disrupt-mechanism`) Shared: a `disruption_total` metric + timestamp marks the onset reference. `disrupt-mechanism: none` (or node-churn with `disrupt-node-count: 0`) skips straight to observe (monitor-only). +**3.0 — observe+disrupt dispatch + staging** (resequenced from Phase 2) +- [ ] Add `disrupt-mechanism` dispatch so `mode: observe` with a mechanism set runs the staged + reproduction (wait-to-radius → baseline → disrupt → observe) instead of the plain soak monitor. +- [ ] `mode: observe`+disrupt staging: poll until every observed node reports + `storageRadius >= DisruptAtRadius` (load drives), with a timeout; then `baselineSnapshot`. + **3a — node-churn** - [ ] Add `DisruptNodeCount` (default 2; `0` = skip) and `DisruptMethod` (`stop` default | `delete`). - [ ] **Randomly** select `DisruptNodeCount` nodes, seeded by `rnd-seed` (reproducible); in `halt` mode diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index eaf42696..e504b3fc 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -67,7 +67,8 @@ driven externally, e.g. by a parallel `load` check). **`halt`** (`runHalt`): the self-driving reproduction. 1. **stake** (optional `ensureStaked` pre-step, as above). -2. **`waitForWarmupDone`** + **baseline** snapshot. +2. **`waitForWarmupDone`** + **`baselineSnapshot`** — the pre-disruption reference: per node, storage + radius + `reserveSizeWithinRadius`, `isFullySynced` + round/lastPlayed/lastWon, and stake. 3. **`driveAllToRadius`** — buy a mutable batch and upload until **every** observed node's `storageRadius` reaches `DisruptAtRadius` (gates on the **min** across nodes, so the whole neighbourhood is populated before disruption), or `MaxUploads` / `IncreaseTimeout`. diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 4c9fdfaa..cbb664d0 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "strconv" "strings" "time" @@ -301,7 +302,7 @@ func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Op if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { return err } - c.snapshot(ctx, nodes, "baseline") + c.baselineSnapshot(ctx, nodes) // 2. Drive every observed node up to DisruptAtRadius (a populated neighbourhood // is the precondition for the disruption to bite). @@ -624,6 +625,47 @@ func (c *Check) snapshot(ctx context.Context, nodes orchestration.ClientList, ph return mx } +// baselineSnapshot logs and emits the pre-disruption reference state for every node: +// storage radius + reserveSizeWithinRadius (/status), isFullySynced + round +// participation (/redistributionstate), and stake. Best-effort per node — a failed +// read shows as "?" in the log. Phase 4 will extend this to return the captured +// values for onset and staked-round-loss comparison. +func (c *Check) baselineSnapshot(ctx context.Context, nodes orchestration.ClientList) { + for _, n := range nodes { + name := n.Name() + + radius, within := "?", "?" + if s, err := n.Status(ctx); err == nil { + c.emit(name, s) + radius = strconv.Itoa(int(s.StorageRadius)) + within = strconv.FormatUint(s.ReserveSizeWithinRadius, 10) + } else { + c.logger.Debugf("baseline %s: status error: %v", name, err) + } + + synced, round, played, won := "?", "?", "?", "?" + if r, err := n.RedistributionState(ctx); err == nil { + c.emitRedist(name, r) + synced = strconv.FormatBool(r.IsFullySynced) + round = strconv.FormatUint(r.Round, 10) + played = strconv.FormatUint(r.LastPlayedRound, 10) + won = strconv.FormatUint(r.LastWonRound, 10) + } else { + c.logger.Debugf("baseline %s: redistributionstate error: %v", name, err) + } + + stake := "?" + if st, err := n.GetStake(ctx); err == nil { + stake = st.String() + } else { + c.logger.Debugf("baseline %s: stake error: %v", name, err) + } + + c.logger.Infof("[baseline] %s: storageRadius=%s withinR=%s fullySynced=%s round=%s lastPlayed=%s lastWon=%s stake=%s", + name, radius, within, synced, round, played, won, stake) + } +} + // updatePeak polls each node, emits metrics, raises the per-node high-water peak, // logs a line, and returns the min and max current storageRadius across the nodes // that responded (min is 0 if none responded). From ad8990c58efc9a7f6587770d1d3fbd701af14a86 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:34:07 +0200 Subject: [PATCH 14/37] feat(check): add node-churn disruption option surface to reserve-radius --- docs/radius-halt-check-plan.md | 9 ++++++++- pkg/check/reserveradius/README.md | 4 ++++ pkg/check/reserveradius/reserveradius.go | 22 ++++++++++++++++++++++ pkg/config/check.go | 4 ++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 2079c809..caea9b1b 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -164,13 +164,20 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr (or node-churn with `disrupt-node-count: 0`) skips straight to observe (monitor-only). **3.0 — observe+disrupt dispatch + staging** (resequenced from Phase 2) +> **Deferred until 3a (disrupt stage) + Phase 4 (observe-outcome) land.** The dispatch wires together +> stages that don't exist yet, and `runHalt` is the ready consumer for those stages — so build the +> mechanism + observe-outcome first, then add this dispatch to reuse them. Working 3a first. - [ ] Add `disrupt-mechanism` dispatch so `mode: observe` with a mechanism set runs the staged reproduction (wait-to-radius → baseline → disrupt → observe) instead of the plain soak monitor. - [ ] `mode: observe`+disrupt staging: poll until every observed node reports `storageRadius >= DisruptAtRadius` (load drives), with a timeout; then `baselineSnapshot`. **3a — node-churn** -- [ ] Add `DisruptNodeCount` (default 2; `0` = skip) and `DisruptMethod` (`stop` default | `delete`). +- [x] Add the disruption option surface: `DisruptMechanism` (`disrupt-mechanism`, default `node-churn`), + `DisruptNodeCount` (default 2; `0` = skip), `DisruptMethod` (`stop` default | `delete`), and + `MinSurvivors` (default 3) — with mechanism (`DisruptNodeChurn`/`DisruptBatchExpiry`/`DisruptBoth`/ + `DisruptNone`) and method (`RemoveStop`/`RemoveDelete`) constants, config mapping, defaults, README. + *(MinSurvivors + DisruptMechanism folded in here since the node-churn stage consumes all of them.)* - [ ] **Randomly** select `DisruptNodeCount` nodes, seeded by `rnd-seed` (reproducible); in `halt` mode exclude the uploader so driving still works. Build the **survivor set** the observe loop polls — removed nodes drop out of polling. diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index e504b3fc..0052bda1 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -99,6 +99,10 @@ Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in | `MaxUploads` | `max-uploads` | `60` | drive | cap on uploads in the increase phase | | `TargetRadius` | `target-radius` | `1` | drive | storageRadius any node must reach before stopping uploads | | `DisruptAtRadius` | `disrupt-at-radius` | `3` | halt | storageRadius **all** observed nodes must reach before disruption | +| `DisruptMechanism` | `disrupt-mechanism` | `node-churn` | halt/observe+disrupt | `node-churn`, `batch-expiry`, `both`, or `none` (monitor-only) | +| `DisruptNodeCount` | `disrupt-node-count` | `2` | halt/observe+disrupt | node-churn: full nodes to remove (randomly, seeded by `rnd-seed`); `0` = skip | +| `DisruptMethod` | `disrupt-method` | `stop` | halt/observe+disrupt | node-churn: `stop` (scale statefulset to 0) or `delete` (statefulset + resources) | +| `MinSurvivors` | `min-survivors` | `3` | halt/observe+disrupt | refuse to disrupt below this many surviving nodes | | `WarmupWait` | `warmup-wait` | `15m` | both | max wait for nodes to finish warmup | | `IncreaseTimeout` | `increase-timeout` | `5m` | drive | max time to reach `TargetRadius` | | `SettleWait` | `settle-wait` | `1m` | drive | post-upload window (pushsync drain + peak tracking) | diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index cbb664d0..26750e97 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -51,6 +51,10 @@ type Options struct { MaxUploads int // cap on uploads during the increase phase TargetRadius uint8 // drive mode: storageRadius any node must reach before stopping uploads DisruptAtRadius uint8 // halt mode: storageRadius ALL observed nodes must reach before disruption + DisruptMechanism string // "node-churn" (default) | "batch-expiry" | "both" | "none" + DisruptNodeCount int // node-churn: full nodes to remove (randomly, seeded by RndSeed); 0 = skip + DisruptMethod string // node-churn: "stop" (scale-0, default) | "delete" + MinSurvivors int // refuse to disrupt below this many surviving nodes WarmupWait time.Duration // max wait for nodes to leave warmup before staging IncreaseTimeout time.Duration // max time to reach TargetRadius SettleWait time.Duration // wait after uploads for pushsync overshoot to drain @@ -76,6 +80,10 @@ func NewDefaultOptions() Options { MaxUploads: 60, TargetRadius: 1, DisruptAtRadius: 3, + DisruptMechanism: DisruptNodeChurn, + DisruptNodeCount: 2, + DisruptMethod: RemoveStop, + MinSurvivors: 3, WarmupWait: 15 * time.Minute, IncreaseTimeout: 5 * time.Minute, SettleWait: time.Minute, @@ -109,6 +117,20 @@ const ( ModeHalt = "halt" // self-driving: stake → drive all nodes → disrupt → observe outcome ) +// Disruption mechanisms for Options.DisruptMechanism. +const ( + DisruptNodeChurn = "node-churn" // stop/delete random full nodes (default) + DisruptBatchExpiry = "batch-expiry" // expire a fill batch → commitment drop → radius decrease + DisruptBoth = "both" // batch-expiry + node-churn for a harsher merge + DisruptNone = "none" // monitor-only (no disruption) +) + +// Node-removal methods for Options.DisruptMethod. +const ( + RemoveStop = "stop" // scale statefulset to 0 (restorable) + RemoveDelete = "delete" // delete statefulset + services/ingress/secret/configmap +) + // Run dispatches on Mode: drive (force a change and observe it), observe // (monitor changes driven externally, e.g. by a parallel load check), or halt // (self-driving stake → drive → disrupt → observe-outcome reproduction). diff --git a/pkg/config/check.go b/pkg/config/check.go index c5490a07..ca02de58 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -515,6 +515,10 @@ var Checks = map[string]CheckType{ MaxUploads *int `yaml:"max-uploads"` TargetRadius *uint8 `yaml:"target-radius"` DisruptAtRadius *uint8 `yaml:"disrupt-at-radius"` + DisruptMechanism *string `yaml:"disrupt-mechanism"` + DisruptNodeCount *int `yaml:"disrupt-node-count"` + DisruptMethod *string `yaml:"disrupt-method"` + MinSurvivors *int `yaml:"min-survivors"` WarmupWait *time.Duration `yaml:"warmup-wait"` IncreaseTimeout *time.Duration `yaml:"increase-timeout"` SettleWait *time.Duration `yaml:"settle-wait"` From 5fc29a7acf025a48c5b00942459c1e347b11d21c Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:40:15 +0200 Subject: [PATCH 15/37] feat(check): add node-churn disruption stage to halt mode --- docs/radius-halt-check-plan.md | 16 ++-- pkg/check/reserveradius/README.md | 9 ++- pkg/check/reserveradius/metrics.go | 10 +++ pkg/check/reserveradius/reserveradius.go | 99 +++++++++++++++++++++++- 4 files changed, 122 insertions(+), 12 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index caea9b1b..73b99b31 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -178,13 +178,15 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr `MinSurvivors` (default 3) — with mechanism (`DisruptNodeChurn`/`DisruptBatchExpiry`/`DisruptBoth`/ `DisruptNone`) and method (`RemoveStop`/`RemoveDelete`) constants, config mapping, defaults, README. *(MinSurvivors + DisruptMechanism folded in here since the node-churn stage consumes all of them.)* -- [ ] **Randomly** select `DisruptNodeCount` nodes, seeded by `rnd-seed` (reproducible); in `halt` mode - exclude the uploader so driving still works. Build the **survivor set** the observe loop polls — - removed nodes drop out of polling. -- [ ] Remove via `node.Stop(ctx, cluster.Namespace())` / `node.Delete(...)`. Log + timestamp the - removal and the chosen node names. Emit `disruption_total`. -- [ ] Guard: require `len(survivors) >= MinSurvivors` (default 3); if the count would breach it, error - out **before** touching the cluster. +- [x] **Randomly** select `DisruptNodeCount` nodes, seeded by `rnd-seed` (`rnd.Perm`, reproducible); + excludes `excludeName` (the uploader in `halt` mode). Builds the **survivor set** (observed minus + removed) returned for the observe loop. `DisruptNodeCount <= 0` = monitor-only (returns all). +- [x] Remove via `node.Stop(ctx, namespace)` / `node.Delete(...)` (looked up from `cluster.Nodes()` by + name), selected by `DisruptMethod`. Logs + RFC3339-timestamps each removal; emits `disruption_total` + (labelled by mechanism). Implemented as `disruptNodeChurn`, behind a `disrupt` mechanism dispatcher + (node-churn/none done; batch-expiry/both error as Phase-3b). Wired into `runHalt`. +- [x] Guard: `len(observed) - DisruptNodeCount >= MinSurvivors`, plus a candidate-count check — both + error out **before** any removal touches the cluster. **3b — batch-expiry** - [ ] Create a soon-to-expire mutable batch — `CreatePostageBatch(ctx, amount, depth, label)` with a diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 0052bda1..2a13851e 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -73,8 +73,12 @@ driven externally, e.g. by a parallel `load` check). `storageRadius` reaches `DisruptAtRadius` (gates on the **min** across nodes, so the whole neighbourhood is populated before disruption), or `MaxUploads` / `IncreaseTimeout`. 4. **settle** — poll for `SettleWait`. -5. **disrupt + observe-outcome** — Phase 3/4 (node-churn / batch-expiry, then classify - `HALT`/`RECOVERED`/`MONITORED` and apply the verdict). Currently stubbed with TODOs. +5. **`disrupt`** — apply `DisruptMechanism`. **node-churn** (`disruptNodeChurn`): randomly pick + `DisruptNodeCount` nodes (seeded by `rnd-seed`, excluding the uploader), guard `MinSurvivors` + **before** touching the cluster, then `Stop` (scale-0) or `Delete` each; returns the **survivor set**. + `none` / `disrupt-node-count: 0` is monitor-only; `batch-expiry`/`both` are Phase 3b (not yet). +6. **observe-outcome** — Phase 4: poll survivors, classify `HALT`/`RECOVERED`/`MONITORED`, apply the + verdict. Currently stubbed with a TODO (a post-disruption snapshot is taken). ## Options @@ -121,6 +125,7 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - `…_reserve_within_radius{node}` — gauge, reserve chunks within radius (completeness signal) - `…_radius_transitions_total{node,direction}` — counter, observed up/down transitions (observe mode) - `…_recovery_observed_total{node,result}` — counter, recovery outcome `recovered`/`timeout` (observe mode) +- `…_disruption_total{mechanism}` — counter, neighbourhood disruptions applied (halt mode, e.g. `node-churn`) - `…_time_to_fully_synced_seconds` — gauge, decrease → isFullySynced again (observe mode) - from `/redistributionstate` (observe mode): `…_fully_synced{node}`, `…_frozen{node}`, `…_redistribution_round{node}`, `…_last_sample_duration_seconds{node}` — the halt indicators diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go index f5c817eb..c699434f 100644 --- a/pkg/check/reserveradius/metrics.go +++ b/pkg/check/reserveradius/metrics.go @@ -15,6 +15,7 @@ type metrics struct { TimeToFullySynced prometheus.Gauge RadiusTransitions *prometheus.CounterVec RecoveryObserved *prometheus.CounterVec + Disruptions *prometheus.CounterVec // redistribution-game liveness (observe mode), from /redistributionstate FullySynced *prometheus.GaugeVec Frozen *prometheus.GaugeVec @@ -85,6 +86,15 @@ func newMetrics(subsystem string) metrics { }, []string{"node", "result"}, ), + Disruptions: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "disruption_total", + Help: "Neighbourhood disruptions applied, by mechanism (node-churn/batch-expiry).", + }, + []string{"mechanism"}, + ), ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node (completeness signal)."), TimeToFullySynced: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: m.Namespace, Subsystem: subsystem, diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 26750e97..879d51b9 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -348,12 +348,105 @@ func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Op c.updatePeak(ctx, nodes, peak) } - // TODO(Phase 3): disrupt the neighbourhood (node-churn / batch-expiry). - // TODO(Phase 4): observe the outcome, classify HALT|RECOVERED|MONITORED, apply the verdict. - c.logger.Info("halt: stake+drive+settle complete; disruption and outcome observation land in Phase 3/4") + // 4. Disrupt the neighbourhood; the survivor set is what the observe loop polls. + survivors, err := c.disrupt(ctx, cluster, nodes, uploader.Name(), o) + if err != nil { + return err + } + c.snapshot(ctx, survivors, "post-disrupt") + + // TODO(Phase 4): observe survivors, classify HALT|RECOVERED|MONITORED, apply the verdict. + c.logger.Infof("halt: disruption applied (%s); %d survivor(s); outcome observation lands in Phase 4", o.DisruptMechanism, len(survivors)) return nil } +// disrupt applies the configured disruption mechanism and returns the survivor set +// the observe loop should poll. excludeName is kept out of node-churn selection (the +// uploader in halt mode); pass "" when there is nothing to protect. +func (c *Check) disrupt(ctx context.Context, cluster orchestration.Cluster, observed orchestration.ClientList, excludeName string, o Options) (orchestration.ClientList, error) { + switch o.DisruptMechanism { + case DisruptNone: + c.logger.Info("disrupt: mechanism=none (monitor-only); no nodes removed") + return observed, nil + case "", DisruptNodeChurn: + return c.disruptNodeChurn(ctx, cluster, observed, excludeName, o) + case DisruptBatchExpiry, DisruptBoth: + return nil, fmt.Errorf("disrupt-mechanism %q not yet implemented (Phase 3b)", o.DisruptMechanism) + default: + return nil, fmt.Errorf("invalid disrupt-mechanism %q (want %q, %q, %q or %q)", o.DisruptMechanism, DisruptNodeChurn, DisruptBatchExpiry, DisruptBoth, DisruptNone) + } +} + +// disruptNodeChurn randomly selects DisruptNodeCount nodes (seeded by RndSeed, +// reproducible), excluding excludeName, and removes them via Stop (scale-0) or +// Delete. The MinSurvivors guard is checked BEFORE any removal, so a too-aggressive +// count fails without touching the cluster. DisruptNodeCount <= 0 is monitor-only. +// Returns the survivor ClientList (observed minus removed). +func (c *Check) disruptNodeChurn(ctx context.Context, cluster orchestration.Cluster, observed orchestration.ClientList, excludeName string, o Options) (orchestration.ClientList, error) { + if o.DisruptNodeCount <= 0 { + c.logger.Info("disrupt: node-churn disrupt-node-count=0 (monitor-only); no nodes removed") + return observed, nil + } + switch o.DisruptMethod { + case "", RemoveStop, RemoveDelete: + default: + return nil, fmt.Errorf("invalid disrupt-method %q (want %q or %q)", o.DisruptMethod, RemoveStop, RemoveDelete) + } + + // Candidate pool excludes the protected node (the uploader in halt mode). + candidates := make(orchestration.ClientList, 0, len(observed)) + for _, n := range observed { + if n.Name() != excludeName { + candidates = append(candidates, n) + } + } + if o.DisruptNodeCount > len(candidates) { + return nil, fmt.Errorf("disrupt: cannot remove %d node(s), only %d candidate(s) available (excluding %q)", o.DisruptNodeCount, len(candidates), excludeName) + } + if survivors := len(observed) - o.DisruptNodeCount; survivors < o.MinSurvivors { + return nil, fmt.Errorf("disrupt: removing %d of %d node(s) would leave %d survivor(s), below min-survivors=%d", o.DisruptNodeCount, len(observed), survivors, o.MinSurvivors) + } + + // Reproducible random pick of DisruptNodeCount candidates. + rnd := random.PseudoGenerator(o.RndSeed) + pick := rnd.Perm(len(candidates))[:o.DisruptNodeCount] + removed := make(map[string]bool, len(pick)) + nodesByName := cluster.Nodes() + ns := cluster.Namespace() + for _, ci := range pick { + name := candidates[ci].Name() + node, ok := nodesByName[name] + if !ok { + return nil, fmt.Errorf("disrupt: node %q not found in cluster", name) + } + method := o.DisruptMethod + if method == "" { + method = RemoveStop + } + var rerr error + if method == RemoveDelete { + rerr = node.Delete(ctx, ns) + } else { + rerr = node.Stop(ctx, ns) + } + if rerr != nil { + return nil, fmt.Errorf("disrupt: %s node %q: %w", method, name, rerr) + } + removed[name] = true + c.metrics.Disruptions.WithLabelValues(DisruptNodeChurn).Inc() + c.logger.Warningf("disrupt: %s node %q (%d/%d) at %s", method, name, len(removed), o.DisruptNodeCount, time.Now().Format(time.RFC3339)) + } + + survivors := make(orchestration.ClientList, 0, len(observed)-len(removed)) + for _, n := range observed { + if !removed[n.Name()] { + survivors = append(survivors, n) + } + } + c.logger.Infof("disrupt: node-churn removed %d node(s) via %s; %d survivor(s) remain", len(removed), o.DisruptMethod, len(survivors)) + return survivors, nil +} + // obsNode is the per-node state the observe monitor tracks across polls. type obsNode struct { lastRadius uint8 From ef80af3d5f6341b2f6164d3f03a242b3d484e6d6 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:47:03 +0200 Subject: [PATCH 16/37] feat(check): add observe-outcome classification to halt mode --- docs/radius-halt-check-plan.md | 26 ++-- pkg/check/reserveradius/README.md | 9 +- pkg/check/reserveradius/metrics.go | 10 ++ pkg/check/reserveradius/reserveradius.go | 145 ++++++++++++++++++++++- 4 files changed, 175 insertions(+), 15 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 73b99b31..00bdaf6e 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -188,7 +188,11 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr - [x] Guard: `len(observed) - DisruptNodeCount >= MinSurvivors`, plus a candidate-count check — both error out **before** any removal touches the cluster. -**3b — batch-expiry** +**3b — batch-expiry** *(deferred until after Phase 4)* +> **Deferred:** its defining items (detect expiry → observe eviction → gated radius decrease, else +> `MONITORED`) are observe-phase logic that must fold into the Phase-4 observe loop, which doesn't +> exist yet. Node-churn (the primary mechanism the manual repro used) is fully wired, so build Phase 4 +> against it first, then add batch-expiry with its detection folded into the existing loop. - [ ] Create a soon-to-expire mutable batch — `CreatePostageBatch(ctx, amount, depth, label)` with a small explicit `amount`, or `GetOrCreateMutableBatch` with a short `expiring-batch-ttl`. Drive the radius (Phase 2) by uploading the fill **under this batch** so its chunks dominate the reserve. @@ -200,19 +204,19 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr - [ ] No node removal — all nodes stay in the observe set. ### Phase 4 — Observe, classify, report (optionally assert) + round-loss -- [ ] Extend the per-node observe state: track **onset** (`within_radius` 0→>threshold OR - `fullySynced` true→false), the **stall** (`fullySynced` stuck false, `pullsyncRate` decaying, - `within_radius` plateau), and **round participation** from `/redistributionstate` - (`Round` advancing while `LastPlayedRound`/`LastWonRound` stalls = round-loss). -- [ ] Classify the run outcome → `MONITORED` | `HALT` | `RECOVERED`; emit it as a labelled metric and - a clear end-of-run summary (stuck vs recovered survivors, onset time, rounds lost). -- [ ] New metrics: `outcome` (labelled), `disruption_total`, `onset_seconds` (removal→onset), - `round_loss_total`, plus reuse `fully_synced`, `frozen`, `redistribution_round`, - `reserve_within_radius`, `pullsync_rate`, `time_to_fully_synced_seconds`. +- [x] Extend the per-node observe state (`outcomeNode` in `observeOutcome`): track **onset** + (`fullySynced` true→false relative to the first post-disruption reference), **recovery** (back to + `fullySynced` within `RecoveryWait`), and **round participation** (`Round` advancing while + `LastPlayedRound`/`LastWonRound` stall = staked round-loss). +- [x] Classify the run outcome → `MONITORED` | `HALT` | `RECOVERED` (`classifyOutcome`); emit the + `outcome` gauge (one-hot) + a clear end-of-run summary (stuck vs recovered survivors, round-loss). +- [ ] New metrics: `outcome` (labelled) ✅, `disruption_total` ✅, plus **`onset_seconds`** (disrupt→onset) + and **`round_loss_total`** still to add, reusing `fully_synced`/`frozen`/`redistribution_round`/ + `reserve_within_radius`/`pullsync_rate`/`time_to_fully_synced_seconds`. - [ ] Apply the **verdict policy**: - `report` (default) → always return success on the outcome; only operational errors fail. - `assert` → fail iff the outcome contradicts `expect-recovery`; `MONITORED` always passes. -- [ ] Bound the whole observe by `Duration` (default ≥30 min; onset is delayed/variable ~3–9 min). +- [x] Bound the whole observe by `Duration` (`observeOutcome` deadline; onset is delayed/variable ~3–9 min). ### Phase 5 — Wire options, config, docs - [ ] Add the new fields to `pkg/config/check.go` `"reserve-radius"` `NewOptions` struct + defaults in diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 2a13851e..af1a0472 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -77,8 +77,12 @@ driven externally, e.g. by a parallel `load` check). `DisruptNodeCount` nodes (seeded by `rnd-seed`, excluding the uploader), guard `MinSurvivors` **before** touching the cluster, then `Stop` (scale-0) or `Delete` each; returns the **survivor set**. `none` / `disrupt-node-count: 0` is monitor-only; `batch-expiry`/`both` are Phase 3b (not yet). -6. **observe-outcome** — Phase 4: poll survivors, classify `HALT`/`RECOVERED`/`MONITORED`, apply the - verdict. Currently stubbed with a TODO (a post-disruption snapshot is taken). +6. **`observeOutcome`** — poll the survivors for `Duration`, tracking per node the **onset** of de-sync + (`isFullySynced` true→false), **recovery** (back within `RecoveryWait`), and staked **round-loss** + (`round` advancing while `lastPlayed`/`lastWon` stall). `classifyOutcome` reduces this to + `MONITORED` (no disruption) / `HALT` (a survivor stuck de-synced past `RecoveryWait` and/or + round-loss) / `RECOVERED` (all de-synced survivors re-converged), emitting the `outcome` gauge. + The **verdict** (report-vs-assert) is the next step (currently always reports). ## Options @@ -126,6 +130,7 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - `…_radius_transitions_total{node,direction}` — counter, observed up/down transitions (observe mode) - `…_recovery_observed_total{node,result}` — counter, recovery outcome `recovered`/`timeout` (observe mode) - `…_disruption_total{mechanism}` — counter, neighbourhood disruptions applied (halt mode, e.g. `node-churn`) +- `…_outcome{outcome}` — gauge, one-hot halt-run classification (`MONITORED`/`HALT`/`RECOVERED`); the classified one is 1 - `…_time_to_fully_synced_seconds` — gauge, decrease → isFullySynced again (observe mode) - from `/redistributionstate` (observe mode): `…_fully_synced{node}`, `…_frozen{node}`, `…_redistribution_round{node}`, `…_last_sample_duration_seconds{node}` — the halt indicators diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go index c699434f..1b102ad9 100644 --- a/pkg/check/reserveradius/metrics.go +++ b/pkg/check/reserveradius/metrics.go @@ -16,6 +16,7 @@ type metrics struct { RadiusTransitions *prometheus.CounterVec RecoveryObserved *prometheus.CounterVec Disruptions *prometheus.CounterVec + Outcome *prometheus.GaugeVec // redistribution-game liveness (observe mode), from /redistributionstate FullySynced *prometheus.GaugeVec Frozen *prometheus.GaugeVec @@ -95,6 +96,15 @@ func newMetrics(subsystem string) metrics { }, []string{"mechanism"}, ), + Outcome: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "outcome", + Help: "Halt-run outcome, one-hot per label (MONITORED/HALT/RECOVERED): the classified one is 1.", + }, + []string{"outcome"}, + ), ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node (completeness signal)."), TimeToFullySynced: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: m.Namespace, Subsystem: subsystem, diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 879d51b9..1e5de725 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -131,6 +131,13 @@ const ( RemoveDelete = "delete" // delete statefulset + services/ingress/secret/configmap ) +// Run outcomes classified by the halt observe-outcome stage. +const ( + OutcomeMonitored = "MONITORED" // no disruption requested; state recorded only + OutcomeHalt = "HALT" // post-disruption sustained non-convergence and/or staked round-loss + OutcomeRecovered = "RECOVERED" // post-disruption, all de-synced survivors re-converged +) + // Run dispatches on Mode: drive (force a change and observe it), observe // (monitor changes driven externally, e.g. by a parallel load check), or halt // (self-driving stake → drive → disrupt → observe-outcome reproduction). @@ -353,10 +360,16 @@ func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Op if err != nil { return err } + disrupted := len(survivors) < len(nodes) // node-churn removed nodes; none/count-0 = monitor-only c.snapshot(ctx, survivors, "post-disrupt") - // TODO(Phase 4): observe survivors, classify HALT|RECOVERED|MONITORED, apply the verdict. - c.logger.Infof("halt: disruption applied (%s); %d survivor(s); outcome observation lands in Phase 4", o.DisruptMechanism, len(survivors)) + // 5. Observe the outcome on the survivors and classify it. + outcome, err := c.observeOutcome(ctx, survivors, disrupted, o) + if err != nil { + return err + } + c.logger.Infof("halt: run outcome = %s", outcome) + // TODO(Phase 4 verdict): apply the verdict policy (report default / assert on expect-recovery). return nil } @@ -447,6 +460,134 @@ func (c *Check) disruptNodeChurn(ctx context.Context, cluster orchestration.Clus return survivors, nil } +// outcomeNode tracks a survivor's post-disruption trajectory for classification. +type outcomeNode struct { + haveRef bool + refSynced bool // was the node fullySynced at the first post-disruption reading + refRound uint64 // round participation reference (for staked round-loss) + refPlayed uint64 + refWon uint64 + + onset bool // lost sync (fullySynced true->false) after disruption + onsetAt time.Time // when the onset was first seen + recovered bool // returned to fullySynced after an onset + recoverBy time.Time // recovery deadline after onset (onsetAt + RecoveryWait) + + haveLast bool + lastRound uint64 + lastPlayed uint64 + lastWon uint64 +} + +// observeOutcome polls the survivor set for Duration after disruption, tracking per +// node the onset of de-sync (fullySynced true->false), recovery (back to fullySynced +// within RecoveryWait), and staked round-loss (Round advancing while +// LastPlayedRound/LastWonRound stall). It classifies the run as MONITORED (no +// disruption), HALT (a survivor stuck de-synced past RecoveryWait and/or round-loss), +// or RECOVERED (all de-synced survivors re-converged), emits the outcome metric + a +// summary, and returns the outcome. The verdict policy is applied by the caller. +func (c *Check) observeOutcome(ctx context.Context, survivors orchestration.ClientList, disrupted bool, o Options) (string, error) { + st := make(map[string]*outcomeNode, len(survivors)) + for _, n := range survivors { + st[n.Name()] = &outcomeNode{} + } + c.logger.Infof("observe-outcome: watching %d survivor(s) for %s (disrupted=%t, recovery-wait=%s)", len(survivors), o.Duration, disrupted, o.RecoveryWait) + + deadline := time.Now().Add(o.Duration) + for time.Now().Before(deadline) { + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return "", err // parent context cancelled (e.g. check timeout) + } + for _, n := range survivors { + name := n.Name() + ns := st[name] + + s, err := n.Status(ctx) + if err != nil { + continue + } + c.emit(name, s) + + r, rerr := n.RedistributionState(ctx) + if rerr != nil { + continue // the redistribution signal is the classifier; skip this poll for the node + } + c.emitRedist(name, r) + + if !ns.haveRef { + ns.haveRef = true + ns.refSynced = r.IsFullySynced + ns.refRound, ns.refPlayed, ns.refWon = r.Round, r.LastPlayedRound, r.LastWonRound + } + ns.haveLast = true + ns.lastRound, ns.lastPlayed, ns.lastWon = r.Round, r.LastPlayedRound, r.LastWonRound + + // onset: a node that was synced loses sync after disruption + if disrupted && ns.refSynced && !ns.onset && !r.IsFullySynced { + ns.onset = true + ns.onsetAt = time.Now() + ns.recoverBy = ns.onsetAt.Add(o.RecoveryWait) + c.logger.Warningf("observe-outcome: %s de-synced (fullySynced=false) at round %d — onset", name, r.Round) + } + // recovery: back to fullySynced (and not frozen) after an onset + if ns.onset && !ns.recovered && r.IsFullySynced && !r.IsFrozen { + ns.recovered = true + c.logger.Infof("observe-outcome: %s re-converged %s after onset", name, time.Since(ns.onsetAt).Round(time.Second)) + } + } + } + + return c.classifyOutcome(st, disrupted, o), nil +} + +// classifyOutcome reduces the per-node trajectories to a single run outcome and emits it. +func (c *Check) classifyOutcome(st map[string]*outcomeNode, disrupted bool, o Options) string { + if !disrupted { + c.setOutcome(OutcomeMonitored) + c.logger.Infof("observe-outcome: MONITORED — no disruption requested; %d node(s) observed for %s", len(st), o.Duration) + return OutcomeMonitored + } + + desynced, recovered, stuck, roundLoss := 0, 0, 0, 0 + for name, ns := range st { + if ns.onset { + desynced++ + } + if ns.onset && ns.recovered { + recovered++ + } + // stuck: de-synced, never recovered, past its recovery deadline + if ns.onset && !ns.recovered && !ns.recoverBy.IsZero() && time.Now().After(ns.recoverBy) { + stuck++ + } + // staked round-loss: rounds advanced after onset but the node never played/won + if ns.onset && ns.haveLast && ns.lastRound > ns.refRound && ns.lastPlayed == ns.refPlayed && ns.lastWon == ns.refWon { + roundLoss++ + c.logger.Warningf("observe-outcome: %s round-loss — round %d->%d while lastPlayed/Won stalled (%d/%d) since de-sync", name, ns.refRound, ns.lastRound, ns.refPlayed, ns.refWon) + } + } + + if stuck > 0 || roundLoss > 0 { + c.setOutcome(OutcomeHalt) + c.logger.Warningf("observe-outcome: HALT — %d/%d survivor(s) stuck de-synced past %s, %d with staked round-loss (%d de-synced, %d recovered)", stuck, len(st), o.RecoveryWait, roundLoss, desynced, recovered) + return OutcomeHalt + } + c.setOutcome(OutcomeRecovered) + c.logger.Infof("observe-outcome: RECOVERED — all %d survivor(s) converged (%d de-synced then recovered)", len(st), recovered) + return OutcomeRecovered +} + +// setOutcome one-hot-encodes the classified outcome into the outcome gauge. +func (c *Check) setOutcome(outcome string) { + for _, name := range []string{OutcomeMonitored, OutcomeHalt, OutcomeRecovered} { + v := 0.0 + if name == outcome { + v = 1 + } + c.metrics.Outcome.WithLabelValues(name).Set(v) + } +} + // obsNode is the per-node state the observe monitor tracks across polls. type obsNode struct { lastRadius uint8 From 8a25262b84309ca322d2c78e3d276665b627ec39 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:53:09 +0200 Subject: [PATCH 17/37] feat(check): add onset_seconds/round_loss_total metrics --- docs/radius-halt-check-plan.md | 7 ++++--- pkg/check/reserveradius/README.md | 2 ++ pkg/check/reserveradius/metrics.go | 20 ++++++++++++++++++++ pkg/check/reserveradius/reserveradius.go | 9 ++++++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 00bdaf6e..53d10307 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -210,9 +210,10 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr `LastPlayedRound`/`LastWonRound` stall = staked round-loss). - [x] Classify the run outcome → `MONITORED` | `HALT` | `RECOVERED` (`classifyOutcome`); emit the `outcome` gauge (one-hot) + a clear end-of-run summary (stuck vs recovered survivors, round-loss). -- [ ] New metrics: `outcome` (labelled) ✅, `disruption_total` ✅, plus **`onset_seconds`** (disrupt→onset) - and **`round_loss_total`** still to add, reusing `fully_synced`/`frozen`/`redistribution_round`/ - `reserve_within_radius`/`pullsync_rate`/`time_to_fully_synced_seconds`. +- [x] New metrics: `outcome` (labelled), `disruption_total`, `onset_seconds{node}` (disrupt→onset, set + on onset) and `round_loss_total{node}` (incremented per round-loss survivor), reusing + `fully_synced`/`frozen`/`redistribution_round`/`reserve_within_radius`/`pullsync_rate`/ + `time_to_fully_synced_seconds`. - [ ] Apply the **verdict policy**: - `report` (default) → always return success on the outcome; only operational errors fail. - `assert` → fail iff the outcome contradicts `expect-recovery`; `MONITORED` always passes. diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index af1a0472..9f95fc88 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -131,6 +131,8 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - `…_recovery_observed_total{node,result}` — counter, recovery outcome `recovered`/`timeout` (observe mode) - `…_disruption_total{mechanism}` — counter, neighbourhood disruptions applied (halt mode, e.g. `node-churn`) - `…_outcome{outcome}` — gauge, one-hot halt-run classification (`MONITORED`/`HALT`/`RECOVERED`); the classified one is 1 +- `…_onset_seconds{node}` — gauge, seconds from disruption to a survivor's de-sync onset (halt mode) +- `…_round_loss_total{node}` — counter, survivors with staked round-loss (rounds advanced while lastPlayed/Won stalled) - `…_time_to_fully_synced_seconds` — gauge, decrease → isFullySynced again (observe mode) - from `/redistributionstate` (observe mode): `…_fully_synced{node}`, `…_frozen{node}`, `…_redistribution_round{node}`, `…_last_sample_duration_seconds{node}` — the halt indicators diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go index 1b102ad9..5b8b9beb 100644 --- a/pkg/check/reserveradius/metrics.go +++ b/pkg/check/reserveradius/metrics.go @@ -17,6 +17,8 @@ type metrics struct { RecoveryObserved *prometheus.CounterVec Disruptions *prometheus.CounterVec Outcome *prometheus.GaugeVec + OnsetSeconds *prometheus.GaugeVec + RoundLoss *prometheus.CounterVec // redistribution-game liveness (observe mode), from /redistributionstate FullySynced *prometheus.GaugeVec Frozen *prometheus.GaugeVec @@ -105,6 +107,24 @@ func newMetrics(subsystem string) metrics { }, []string{"outcome"}, ), + OnsetSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "onset_seconds", + Help: "Seconds from disruption to a survivor's de-sync onset, per node (halt mode).", + }, + []string{"node"}, + ), + RoundLoss: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "round_loss_total", + Help: "Survivors with staked round-loss (rounds advanced while lastPlayed/lastWon stalled), per node.", + }, + []string{"node"}, + ), ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node (completeness signal)."), TimeToFullySynced: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: m.Namespace, Subsystem: subsystem, diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 1e5de725..14a54892 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -361,10 +361,11 @@ func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Op return err } disrupted := len(survivors) < len(nodes) // node-churn removed nodes; none/count-0 = monitor-only + disruptedAt := time.Now() c.snapshot(ctx, survivors, "post-disrupt") // 5. Observe the outcome on the survivors and classify it. - outcome, err := c.observeOutcome(ctx, survivors, disrupted, o) + outcome, err := c.observeOutcome(ctx, survivors, disrupted, disruptedAt, o) if err != nil { return err } @@ -486,7 +487,7 @@ type outcomeNode struct { // disruption), HALT (a survivor stuck de-synced past RecoveryWait and/or round-loss), // or RECOVERED (all de-synced survivors re-converged), emits the outcome metric + a // summary, and returns the outcome. The verdict policy is applied by the caller. -func (c *Check) observeOutcome(ctx context.Context, survivors orchestration.ClientList, disrupted bool, o Options) (string, error) { +func (c *Check) observeOutcome(ctx context.Context, survivors orchestration.ClientList, disrupted bool, disruptedAt time.Time, o Options) (string, error) { st := make(map[string]*outcomeNode, len(survivors)) for _, n := range survivors { st[n.Name()] = &outcomeNode{} @@ -527,7 +528,8 @@ func (c *Check) observeOutcome(ctx context.Context, survivors orchestration.Clie ns.onset = true ns.onsetAt = time.Now() ns.recoverBy = ns.onsetAt.Add(o.RecoveryWait) - c.logger.Warningf("observe-outcome: %s de-synced (fullySynced=false) at round %d — onset", name, r.Round) + c.metrics.OnsetSeconds.WithLabelValues(name).Set(ns.onsetAt.Sub(disruptedAt).Seconds()) + c.logger.Warningf("observe-outcome: %s de-synced (fullySynced=false) at round %d, %s after disruption — onset", name, r.Round, ns.onsetAt.Sub(disruptedAt).Round(time.Second)) } // recovery: back to fullySynced (and not frozen) after an onset if ns.onset && !ns.recovered && r.IsFullySynced && !r.IsFrozen { @@ -563,6 +565,7 @@ func (c *Check) classifyOutcome(st map[string]*outcomeNode, disrupted bool, o Op // staked round-loss: rounds advanced after onset but the node never played/won if ns.onset && ns.haveLast && ns.lastRound > ns.refRound && ns.lastPlayed == ns.refPlayed && ns.lastWon == ns.refWon { roundLoss++ + c.metrics.RoundLoss.WithLabelValues(name).Inc() c.logger.Warningf("observe-outcome: %s round-loss — round %d->%d while lastPlayed/Won stalled (%d/%d) since de-sync", name, ns.refRound, ns.lastRound, ns.refPlayed, ns.refWon) } } From fd7b646e51aa368065f678e8c027ace6ea536973 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 11:55:32 +0200 Subject: [PATCH 18/37] feat(check): add verdict policy (report/assert) to halt mode --- docs/radius-halt-check-plan.md | 2 +- pkg/check/reserveradius/README.md | 6 +++- pkg/check/reserveradius/reserveradius.go | 41 ++++++++++++++++++++++-- pkg/config/check.go | 2 ++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 53d10307..ae2f539e 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -214,7 +214,7 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr on onset) and `round_loss_total{node}` (incremented per round-loss survivor), reusing `fully_synced`/`frozen`/`redistribution_round`/`reserve_within_radius`/`pullsync_rate`/ `time_to_fully_synced_seconds`. -- [ ] Apply the **verdict policy**: +- [x] Apply the **verdict policy** (`applyVerdict`, `verdict`/`expect-recovery` options + constants): - `report` (default) → always return success on the outcome; only operational errors fail. - `assert` → fail iff the outcome contradicts `expect-recovery`; `MONITORED` always passes. - [x] Bound the whole observe by `Duration` (`observeOutcome` deadline; onset is delayed/variable ~3–9 min). diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 9f95fc88..f6a17a1a 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -82,7 +82,9 @@ driven externally, e.g. by a parallel `load` check). (`round` advancing while `lastPlayed`/`lastWon` stall). `classifyOutcome` reduces this to `MONITORED` (no disruption) / `HALT` (a survivor stuck de-synced past `RecoveryWait` and/or round-loss) / `RECOVERED` (all de-synced survivors re-converged), emitting the `outcome` gauge. - The **verdict** (report-vs-assert) is the next step (currently always reports). +7. **`applyVerdict`** — `report` (default) always succeeds on the outcome (only operational errors + fail); `assert` fails iff the outcome contradicts `expect-recovery` (`false` ⇒ expect `HALT`, + `true` ⇒ expect `RECOVERED`), with `MONITORED` always passing. This is the A/B regression gate. ## Options @@ -111,6 +113,8 @@ Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in | `DisruptNodeCount` | `disrupt-node-count` | `2` | halt/observe+disrupt | node-churn: full nodes to remove (randomly, seeded by `rnd-seed`); `0` = skip | | `DisruptMethod` | `disrupt-method` | `stop` | halt/observe+disrupt | node-churn: `stop` (scale statefulset to 0) or `delete` (statefulset + resources) | | `MinSurvivors` | `min-survivors` | `3` | halt/observe+disrupt | refuse to disrupt below this many surviving nodes | +| `Verdict` | `verdict` | `report` | halt | `report` (never fail on outcome) or `assert` (gate on `expect-recovery`) | +| `ExpectRecovery` | `expect-recovery` | `false` | halt | `assert` only: `false` ⇒ expect `HALT`, `true` ⇒ expect `RECOVERED` | | `WarmupWait` | `warmup-wait` | `15m` | both | max wait for nodes to finish warmup | | `IncreaseTimeout` | `increase-timeout` | `5m` | drive | max time to reach `TargetRadius` | | `SettleWait` | `settle-wait` | `1m` | drive | post-upload window (pushsync drain + peak tracking) | diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 14a54892..a9f5946f 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -55,6 +55,8 @@ type Options struct { DisruptNodeCount int // node-churn: full nodes to remove (randomly, seeded by RndSeed); 0 = skip DisruptMethod string // node-churn: "stop" (scale-0, default) | "delete" MinSurvivors int // refuse to disrupt below this many surviving nodes + Verdict string // halt mode: "report" (default, never fail on outcome) | "assert" (gate on ExpectRecovery) + ExpectRecovery bool // halt mode + verdict=assert: false ⇒ expect HALT, true ⇒ expect RECOVERED WarmupWait time.Duration // max wait for nodes to leave warmup before staging IncreaseTimeout time.Duration // max time to reach TargetRadius SettleWait time.Duration // wait after uploads for pushsync overshoot to drain @@ -84,6 +86,8 @@ func NewDefaultOptions() Options { DisruptNodeCount: 2, DisruptMethod: RemoveStop, MinSurvivors: 3, + Verdict: VerdictReport, + ExpectRecovery: false, WarmupWait: 15 * time.Minute, IncreaseTimeout: 5 * time.Minute, SettleWait: time.Minute, @@ -138,6 +142,12 @@ const ( OutcomeRecovered = "RECOVERED" // post-disruption, all de-synced survivors re-converged ) +// Verdict policies for Options.Verdict. +const ( + VerdictReport = "report" // never fail on the outcome; only operational errors fail (default) + VerdictAssert = "assert" // fail iff the outcome contradicts expect-recovery (A/B regression gate) +) + // Run dispatches on Mode: drive (force a change and observe it), observe // (monitor changes driven externally, e.g. by a parallel load check), or halt // (self-driving stake → drive → disrupt → observe-outcome reproduction). @@ -370,8 +380,35 @@ func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Op return err } c.logger.Infof("halt: run outcome = %s", outcome) - // TODO(Phase 4 verdict): apply the verdict policy (report default / assert on expect-recovery). - return nil + return c.applyVerdict(outcome, o) +} + +// applyVerdict turns a classified outcome into a check result per the verdict policy. +// report (default) always succeeds on any outcome (only operational errors, raised +// earlier, fail). assert fails iff the outcome contradicts expect-recovery; MONITORED +// always passes (no disruption was requested). +func (c *Check) applyVerdict(outcome string, o Options) error { + switch o.Verdict { + case "", VerdictReport: + c.logger.Infof("verdict=report: outcome %s (reported, not gated)", outcome) + return nil + case VerdictAssert: + if outcome == OutcomeMonitored { + c.logger.Info("verdict=assert: outcome MONITORED (no disruption requested) — pass") + return nil + } + want := OutcomeHalt + if o.ExpectRecovery { + want = OutcomeRecovered + } + if outcome == want { + c.logger.Infof("verdict=assert: outcome %s matches expect-recovery=%t — pass", outcome, o.ExpectRecovery) + return nil + } + return fmt.Errorf("verdict=assert: outcome %s contradicts expect-recovery=%t (expected %s)", outcome, o.ExpectRecovery, want) + default: + return fmt.Errorf("invalid verdict %q (want %q or %q)", o.Verdict, VerdictReport, VerdictAssert) + } } // disrupt applies the configured disruption mechanism and returns the survivor set diff --git a/pkg/config/check.go b/pkg/config/check.go index ca02de58..177f8f1f 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -519,6 +519,8 @@ var Checks = map[string]CheckType{ DisruptNodeCount *int `yaml:"disrupt-node-count"` DisruptMethod *string `yaml:"disrupt-method"` MinSurvivors *int `yaml:"min-survivors"` + Verdict *string `yaml:"verdict"` + ExpectRecovery *bool `yaml:"expect-recovery"` WarmupWait *time.Duration `yaml:"warmup-wait"` IncreaseTimeout *time.Duration `yaml:"increase-timeout"` SettleWait *time.Duration `yaml:"settle-wait"` From 1c77e02423dcb02078e9ddb6000a3c07399faa32 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 12:02:30 +0200 Subject: [PATCH 19/37] feat(check): add observe+disrupt staged dispatch to reserve-radius --- config/local.yaml | 10 ++-- docs/radius-halt-check-plan.md | 16 ++--- pkg/check/reserveradius/README.md | 20 +++++-- pkg/check/reserveradius/reserveradius.go | 76 ++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 16 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index d7a762a8..65923532 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -40,12 +40,12 @@ clusters: bee: bee-config: bee-local-dns config: local-dns - count: 3 + count: 6 mode: node light: bee-config: bee-local-light config: local-light - count: 2 + count: 0 mode: node local-dns-autotls: _inherit: "local" @@ -394,6 +394,7 @@ checks: ci-reserve-radius-observe: options: mode: observe + disrupt-mechanism: none # monitor-only soak; set node-churn to run the staged observe+disrupt reproduction duration: 30m recovery-wait: 10m poll-interval: 15s @@ -403,14 +404,15 @@ checks: type: reserve-radius ci-load-soak: options: - content-size: 1048576 + content-size: 8388608 postage-ttl: 24h postage-depth: 22 postage-label: load-soak duration: 30m uploader-count: 1 downloader-count: 0 - max-committed-depth: 1 + nodes-sync-wait: 10s + max-committed-depth: 3 committed-depth-check-wait: 15s decrease-hold: 2m upload-groups: diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index ae2f539e..6cc0a4ef 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -163,14 +163,14 @@ or just monitor, all without a red result. Shared: a `disruption_total` metric + timestamp marks the onset reference. `disrupt-mechanism: none` (or node-churn with `disrupt-node-count: 0`) skips straight to observe (monitor-only). -**3.0 — observe+disrupt dispatch + staging** (resequenced from Phase 2) -> **Deferred until 3a (disrupt stage) + Phase 4 (observe-outcome) land.** The dispatch wires together -> stages that don't exist yet, and `runHalt` is the ready consumer for those stages — so build the -> mechanism + observe-outcome first, then add this dispatch to reuse them. Working 3a first. -- [ ] Add `disrupt-mechanism` dispatch so `mode: observe` with a mechanism set runs the staged - reproduction (wait-to-radius → baseline → disrupt → observe) instead of the plain soak monitor. -- [ ] `mode: observe`+disrupt staging: poll until every observed node reports - `storageRadius >= DisruptAtRadius` (load drives), with a timeout; then `baselineSnapshot`. +**3.0 — observe+disrupt dispatch + staging** (resequenced from Phase 2; built once 3a + Phase 4 landed) +- [x] `disrupt-mechanism` dispatch (`disruptionActive` + `runObserveDisrupt`): `mode: observe` with an + active mechanism runs the staged reproduction (wait-to-radius → baseline → disrupt → observe → + verdict) instead of the plain soak monitor. **Behaviour change:** with the default + `node-churn`/count-2, a bare `mode: observe` now disrupts; monitor-only requires + `disrupt-mechanism: none` (or count 0). The existing `ci-reserve-radius-observe` was set to `none`. +- [x] `mode: observe`+disrupt staging (`waitAllReachRadius`): poll until every observed node reports + `storageRadius >= DisruptAtRadius` (load drives) within `IncreaseTimeout`; then `baselineSnapshot`. **3a — node-churn** - [x] Add the disruption option surface: `DisruptMechanism` (`disrupt-mechanism`, default `node-churn`), diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index f6a17a1a..29545a5d 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -49,10 +49,22 @@ the cluster stabilizes. `peak`, watching `pullsyncRate` for recovery. No decrease within `DecreaseTimeout` fails with a message pointing at the PR #581 puller stall. -**`observe`** (`runObserve`): `waitForWarmupDone`, then for `Duration` poll every -`PollInterval`, reading both `/status` and **`/redistributionstate`** per node. It records -up/down `storageRadius` transitions, and asserts the **halt indicators** that define the -October failure: +**`observe`** (`runObserve`): `waitForWarmupDone`, then **dispatches**: + +- **with disruption configured** (`disruptionActive`: an active `disrupt-mechanism` — by default + `node-churn` with `disrupt-node-count > 0`) → the **observe+disrupt** staged reproduction + (`runObserveDisrupt`): `waitAllReachRadius` (wait for the externally-driven radius to reach + `DisruptAtRadius`) → `baselineSnapshot` → `disrupt` → `observeOutcome` → `applyVerdict`. The + parallel-with-`load` shape (same stages as `halt`, minus the self-driving upload). +- **monitor-only** (`disrupt-mechanism: none`, or `node-churn` with `disrupt-node-count: 0`) → the + plain soak monitor below. + +> **Behaviour note:** with the default `node-churn`/count-2, a bare `mode: observe` now runs +> observe+disrupt (which **stops nodes**). Set `disrupt-mechanism: none` for a pure soak monitor. + +The soak monitor: for `Duration` poll every `PollInterval`, reading both `/status` and +**`/redistributionstate`** per node. It records up/down `storageRadius` transitions, and asserts the +**halt indicators** that define the October failure: - **recovery after a decrease** — after a **down** transition, the node must recover within `RecoveryWait`. Recovery prefers the redistribution signal **`isFullySynced && !isFrozen`** diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index a9f5946f..4dc4089f 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -644,6 +644,75 @@ type obsNode struct { // when /redistributionstate is available. It also flags freeze episodes directly // (a frozen node skips rounds — the October halt symptom). Halt indicators // (un-recovered decreases or freezes) fail the check at the end. +// disruptionActive reports whether the configured mechanism will actually disrupt, so +// observe mode runs the staged reproduction rather than the plain soak monitor. +// Monitor-only = disrupt-mechanism none, or node-churn with disrupt-node-count 0. +func disruptionActive(o Options) bool { + switch o.DisruptMechanism { + case DisruptNone: + return false + case "", DisruptNodeChurn: + return o.DisruptNodeCount > 0 + default: // batch-expiry / both + return true + } +} + +// runObserveDisrupt is the parallel-with-load shape: the radius is driven externally +// (e.g. by a parallel load check), so this check waits for the neighbourhood to +// populate, snapshots a baseline, disrupts, then observes + classifies the outcome — +// the same stages as halt mode, minus the self-driving upload. +func (c *Check) runObserveDisrupt(ctx context.Context, cluster orchestration.Cluster, nodes orchestration.ClientList, o Options) error { + if o.DisruptAtRadius == 0 { + return errors.New("disrupt-at-radius must be > 0") + } + c.logger.Infof("mode=observe+disrupt: staged reproduction (mechanism=%s, count=%d); radius driven externally", o.DisruptMechanism, o.DisruptNodeCount) + + if err := c.waitAllReachRadius(ctx, nodes, o); err != nil { + return err + } + c.baselineSnapshot(ctx, nodes) + + survivors, err := c.disrupt(ctx, cluster, nodes, "", o) // no uploader to protect in observe mode + if err != nil { + return err + } + disrupted := len(survivors) < len(nodes) + disruptedAt := time.Now() + c.snapshot(ctx, survivors, "post-disrupt") + + outcome, err := c.observeOutcome(ctx, survivors, disrupted, disruptedAt, o) + if err != nil { + return err + } + c.logger.Infof("observe+disrupt: run outcome = %s", outcome) + return c.applyVerdict(outcome, o) +} + +// waitAllReachRadius blocks until every node reports storageRadius >= DisruptAtRadius +// (driven externally, e.g. by a parallel load check) or IncreaseTimeout elapses. +func (c *Check) waitAllReachRadius(ctx context.Context, nodes orchestration.ClientList, o Options) error { + c.logger.Infof("observe+disrupt: waiting up to %s for all %d node(s) to reach storageRadius>=%d (driven externally)", o.IncreaseTimeout, len(nodes), o.DisruptAtRadius) + deadline := time.Now().Add(o.IncreaseTimeout) + peak := make(map[string]uint8, len(nodes)) + for { + if err := ctx.Err(); err != nil { + return err + } + mn, mx := c.updatePeak(ctx, nodes, peak) + if mn >= o.DisruptAtRadius { + c.logger.Infof("observe+disrupt: all nodes reached storageRadius %d (max %d)", o.DisruptAtRadius, mx) + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("not all nodes reached storageRadius %d within %s (min=%d) — is the radius being driven (e.g. the load check) and is the reserve patch active?", o.DisruptAtRadius, o.IncreaseTimeout, mn) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o Options) error { nodes, err := c.selectNodes(ctx, cluster, o) if err != nil { @@ -659,6 +728,13 @@ func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o return err } + // observe + disruption configured → run the staged reproduction (parallel-with-load + // shape) instead of the plain soak monitor. Monitor-only requires disrupt-mechanism: + // none (or node-churn with disrupt-node-count: 0). + if disruptionActive(o) { + return c.runObserveDisrupt(ctx, cluster, nodes, o) + } + st := make(map[string]*obsNode, len(nodes)) for _, n := range nodes { st[n.Name()] = &obsNode{} From ac8b2a2097286c2b37ea6dd4856f6924fd548a3d Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 12:07:44 +0200 Subject: [PATCH 20/37] feat(config): add ci-radius-halt and observe-disrupt check entries --- config/local.yaml | 53 +++++++++++++++++++++++++++++++ docs/radius-halt-check-plan.md | 22 +++++++------ pkg/check/reserveradius/README.md | 13 +++++++- 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index 65923532..e1875b2e 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -402,6 +402,59 @@ checks: - bee timeout: 35m type: reserve-radius + # Self-driving pull-sync-halt reproduction (single check): stake → drive all nodes to radius 3 → + # remove 2 of 6 → observe the survivors. verdict:report records HALT/RECOVERED/MONITORED without + # failing; for the A/B gate set verdict:assert + expect-recovery (false on A=stock, true on B=fix). + ci-radius-halt: + options: + mode: halt + stake-amount: "100000000000000000" # 1e17 per node (quoted: option is a string) + stake-confirm-wait: 2m + disrupt-at-radius: 3 + disrupt-mechanism: node-churn + disrupt-node-count: 2 + disrupt-method: stop + min-survivors: 3 + verdict: report + expect-recovery: false + recovery-wait: 10m + duration: 30m # observe ≥25 min — onset is delayed ~3–9 min + poll-interval: 30s + postage-ttl: 24h + postage-depth: 22 + postage-label: radius-halt + blob-size: 8388608 # 8 MiB + max-uploads: 1000 + warmup-wait: 15m + increase-timeout: 30m + settle-wait: 2m + upload-groups: + - bee + timeout: 90m + type: reserve-radius + # Parallel-with-load shape: run alongside ci-load-soak (which drives the radius via + # max-committed-depth:3). This check stakes, waits for radius 3, removes 2 of 6, then observes. + ci-reserve-radius-observe-disrupt: + options: + mode: observe + stake-amount: "100000000000000000" + stake-confirm-wait: 2m + disrupt-at-radius: 3 + disrupt-mechanism: node-churn + disrupt-node-count: 2 + disrupt-method: stop + min-survivors: 3 + verdict: report + expect-recovery: false + recovery-wait: 10m + duration: 30m + poll-interval: 30s + increase-timeout: 30m # max wait for the load check to drive all nodes to radius 3 + warmup-wait: 15m + upload-groups: + - bee + timeout: 90m + type: reserve-radius ci-load-soak: options: content-size: 8388608 diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 6cc0a4ef..d45e4fc0 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -188,11 +188,13 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr - [x] Guard: `len(observed) - DisruptNodeCount >= MinSurvivors`, plus a candidate-count check — both error out **before** any removal touches the cluster. -**3b — batch-expiry** *(deferred until after Phase 4)* -> **Deferred:** its defining items (detect expiry → observe eviction → gated radius decrease, else -> `MONITORED`) are observe-phase logic that must fold into the Phase-4 observe loop, which doesn't -> exist yet. Node-churn (the primary mechanism the manual repro used) is fully wired, so build Phase 4 -> against it first, then add batch-expiry with its detection folded into the existing loop. +**3b — batch-expiry** *(sequenced last among the code items)* +> **Sequenced after Phase 5 (not a hard blocker).** Phase 4 (the observe loop it folds into) now exists, +> so 3b is unblocked. But batch-expiry is the **secondary** mechanism and its core behaviour — the +> *gated* radius decrease (`SyncRate==0` + `count question that can't be behaviour-verified blind. Phase 5 (config + docs for the complete node-churn +> primary path) is fully verifiable now and is the prerequisite for Phase-6 validation, so it goes first; +> 3b is implemented best-effort and validated live alongside Phase 6. - [ ] Create a soon-to-expire mutable batch — `CreatePostageBatch(ctx, amount, depth, label)` with a small explicit `amount`, or `GetOrCreateMutableBatch` with a short `expiring-batch-ttl`. Drive the radius (Phase 2) by uploading the fill **under this batch** so its chunks dominate the reserve. @@ -220,10 +222,12 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr - [x] Bound the whole observe by `Duration` (`observeOutcome` deadline; onset is delayed/variable ~3–9 min). ### Phase 5 — Wire options, config, docs -- [ ] Add the new fields to `pkg/config/check.go` `"reserve-radius"` `NewOptions` struct + defaults in - `NewDefaultOptions`. -- [ ] Add config entries to `config/local.yaml`: `ci-radius-halt` (single self-driving) and, if Phase 0 - keeps it, a parallel pair reusing `ci-load-soak` + `ci-reserve-radius-observe`+disrupt. +- [x] Add the new fields to `pkg/config/check.go` `"reserve-radius"` `NewOptions` struct + defaults in + `NewDefaultOptions`. (Done incrementally as each option landed — all 25 options mapped + defaulted.) +- [x] Add config entries to `config/local.yaml`: `ci-radius-halt` (single self-driving halt: stake 1e17 + → drive all to radius 3 → remove 2 of 6 → observe) and `ci-reserve-radius-observe-disrupt` (the + parallel-with-`ci-load-soak` observe+disrupt shape). Both default `verdict: report`; flip to + `assert` + `expect-recovery` for the A/B gate. YAML validated (`stake-amount` quoted = string). - [ ] Update `pkg/check/reserveradius/README.md`, the `radius-testing` skill, and cross-link `docs/radius-halt.md` ↔ this plan. diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 29545a5d..d6357662 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -175,10 +175,21 @@ Against a patched local cluster (`local-dns`): ./dist/beekeeper check --cluster-name=local-dns \ --checks=ci-load-soak,ci-reserve-radius-observe --parallel \ --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info + +# halt (self-driving): stake → drive all nodes to radius 3 → remove 2 of 6 → observe + classify +./dist/beekeeper check --cluster-name=local-dns --checks=ci-radius-halt \ + --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info + +# halt (parallel-with-load): load drives the radius, reserve-radius stakes/disrupts/observes +./dist/beekeeper check --cluster-name=local-dns \ + --checks=ci-load-soak,ci-reserve-radius-observe-disrupt --parallel \ + --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info ``` `--parallel` runs the listed checks in goroutines instead of sequentially; -checks fail independently (a monitor failure does not stop the load run). +checks fail independently (a monitor failure does not stop the load run). For the A/B +regression gate set `verdict: assert` + `expect-recovery` (`false` on A=stock master, +`true` on B=pullsync-optimal-design). ## Observed behavior (local, patched cluster) From e35cc5a0994bd1025dcd8de05ce37a324cd4cf33 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 12:10:22 +0200 Subject: [PATCH 21/37] docs(radius): cross-link radius-halt recipe with the check plan --- docs/radius-halt-check-plan.md | 6 +- docs/radius-halt.md | 162 +++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 docs/radius-halt.md diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index d45e4fc0..992c4d61 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -228,8 +228,10 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr → drive all to radius 3 → remove 2 of 6 → observe) and `ci-reserve-radius-observe-disrupt` (the parallel-with-`ci-load-soak` observe+disrupt shape). Both default `verdict: report`; flip to `assert` + `expect-recovery` for the A/B gate. YAML validated (`stake-amount` quoted = string). -- [ ] Update `pkg/check/reserveradius/README.md`, the `radius-testing` skill, and cross-link - `docs/radius-halt.md` ↔ this plan. +- [x] Update `pkg/check/reserveradius/README.md` (kept current each cycle), the `radius-testing` skill + (new "reserve-radius check modes (current)" section: drive/observe/observe+disrupt/halt, mechanisms, + verdict, footgun), and cross-link `docs/radius-halt.md` ↔ this plan (back-link added; plan already + links the recipe). ### Phase 6 — A/B validation - [ ] Run `ci-radius-halt` against **A** (patched stock master) → expect halt reproduced diff --git a/docs/radius-halt.md b/docs/radius-halt.md new file mode 100644 index 00000000..dec984cb --- /dev/null +++ b/docs/radius-halt.md @@ -0,0 +1,162 @@ +# Bee pull-sync halt — reproduction & findings + +**The halt:** after a populated neighbourhood is disrupted, stock-master pull-sync cannot re-converge — +affected nodes stay permanently un-synced and, being staked, can no longer win redistribution rounds. +Reproduced locally on the `local-dns` cluster against **A = stock master + radius patches**. + +**Status:** mechanism + pull-sync non-convergence reproduced and characterized across 4 runs, incl. a +fully-staked run (2026-06-26) that pins down the **staked round-loss**: every survivor hits +`is_playing_errors`, and a node that cannot finish the re-sync cannot produce a `ReserveSample`, so it +loses every round. Remaining: the **A/B** confirmation against **B = `pullsync-optimal-design`**. + +**Automation:** this manual recipe is now folded into the `reserveradius` check's `mode: halt` (and the +`observe`+disrupt shape) — stake → drive radius → disrupt → observe/classify/verdict, unattended. See +the check plan [`docs/radius-halt-check-plan.md`](radius-halt-check-plan.md) and the check +[`pkg/check/reserveradius/README.md`](../pkg/check/reserveradius/README.md); the `ci-radius-halt` and +`ci-reserve-radius-observe-disrupt` entries live in `config/local.yaml`. + +## Mechanism + +The storage radius must be **≥3** to reproduce the halt. At radius 1–2 on a small cluster the reserve is +capacity-driven and `reserveSizeWithinRadius` is degenerate (**0**) — there is no real neighbourhood +data, so nothing can stall. At radius 3 each node holds real in-radius data (~2200 chunks). Removing +nodes from that neighbourhood drops redundancy below what the survivors can re-replicate; the stock +puller **stops delivering before convergence**, so the nodes never re-sync. It is not a crash or a hard +freeze — it is pull-sync silently failing to converge. + +Two ways to disrupt the neighbourhood: + +- **Node removal (reliable).** Remove nodes from a populated radius-3 neighbourhood. The halt does + **not** require a radius decrease — disruption at a populated radius is enough, and this is the + deterministic lever used below. +- **Radius decrease (the faithful October trigger, intermittent).** A decrease *expands* the + neighbourhood (radius N covers half the keyspace of N+1), so a node becomes responsible for far more + chunks and must pull-sync them at once. But the decrease is **gated** (`pkg/storer/reserve.go`): it + fires only when `countWithinRadius < threshold(capacity)` **AND** `syncer.SyncRate()==0` **AND** + `radius > minimumRadius`. Stock `threshold = capacity*50%`; `radius_threshold.patch` raises it to + `capacity`. `SyncRate()==0` rarely holds on a busy cluster, so decreases are slow/intermittent — + node scale-down triggered one only 1 of 4 attempts; stop-load and `dilute` never did (`dilute` is the + *wrong* lever — it raises committed capacity). The faithful decrease lever is a commitment drop + (batch expiry), which is slow/hard to control locally. + +## Prerequisites + +- beelocal/k3d substrate + `geth-swap` + metrics stack (pushgateway 9091 / prometheus 9090 / grafana 3000). +- Patched image `k3d-registry.localhost:5000/ethersphere/bee:latest` = `radius_reserve.patch` + (DefaultReserveCapacity 4000, ReserveWakeUpDuration 10s) + `radius_threshold.patch` (threshold = capacity). + Build it (apply both patches to the bee source, build, push, revert source): + + ``` + patch pkg/storer/storer.go .github/patches/radius_reserve.patch + patch pkg/storer/reserve.go .github/patches/radius_threshold.patch + make docker-build PLATFORM=linux/arm64 BEE_IMAGE=k3d-registry.localhost:5000/ethersphere/bee:latest \ + REACHABILITY_OVERRIDE_PUBLIC=true BATCHFACTOR_OVERRIDE_PUBLIC=2 + docker push k3d-registry.localhost:5000/ethersphere/bee:latest + git checkout pkg/storer/storer.go pkg/storer/reserve.go + ``` + +- `config/local.yaml`: `local-dns` bee `count: 6`, light `count: 0`; `ci-load-soak` `max-committed-depth: 3`. +- Geth **chain price must be non-zero** (e.g. 24000); price 0 logs `invalid chain price` and breaks postage. +- Env prefix for every beekeeper command (local chain; `~/.beekeeper.yaml` is Sepolia): + `BEEKEEPER_GETH_URL=http://geth-swap.localhost BEEKEEPER_BZZ_TOKEN_ADDRESS=0x6aab14fe9cccd64a502d23842d916eb5321c26e7 BEEKEEPER_ETH_ACCOUNT=0x62cab2b3b55f341f10348720ca18063cdb779ad5 BEEKEEPER_WALLET_KEY=4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d` + +## Reproducible steps + +1. **Recreate a clean cluster** (NEVER `kubectl rollout restart` — storage is emptyDir; a restart wipes + reserve+statestore+chequebook and needs re-funding). Geth survives (deployed separately): + + ``` + beekeeper delete bee-cluster --cluster-name=local-dns --geth-url http://geth-swap.localhost + beekeeper create bee-cluster --cluster-name=local-dns --geth-url http://geth-swap.localhost --wallet-key + ``` + +2. **Stake every node** via the API (minimum `1e17`; height 0 because `reserve-capacity-doubling=0`). + Staking is not needed for the *sync* signals, but it is needed to measure the redistribution + round-loss. NOTE: `ci-stake` does NOT stake the cluster — it is a single-node contract test that + deposits then withdraws to 0. + + ``` + for n in bee-0 bee-1 bee-2 bee-3 bee-4 bee-5; do + curl -s -XPOST http://$n.localhost/stake/100000000000000000 + done + # verify: curl -s http://bee-0.localhost/stake | jq .stakedAmount → 100000000000000000 on all six + ``` + +3. **Drive the radius to 3** (stop the load once every node reports `storageRadius=3`): + + ``` + beekeeper check --cluster-name=local-dns --checks=ci-load-soak --metrics-enabled=true --metrics-pusher-address=localhost:9091 + pkill -f 'beekeeper check' # reserveSizeWithinRadius is now REAL (~2000+), not 0 + ``` + +4. **Trigger the halt — remove 2 of the 6 nodes** (disrupt the populated neighbourhood): + + ``` + kubectl delete statefulset bee-4 bee-5 -n local + kubectl delete pod bee-4-0 bee-5-0 -n local --grace-period=3 + ``` + +5. **Observe the 4 survivors for ≥25 min** (onset is delayed ~3–9 min — short windows miss it): + + ``` + for n in bee-0 bee-1 bee-2 bee-3; do + curl -s http://$n.localhost/status | jq '{r:.storageRadius,w:.reserveSizeWithinRadius,ps:.pullsyncRate}' + curl -s http://$n.localhost/redistributionstate | jq '{fs:.isFullySynced,fr:.isFrozen,rnd:.round}' + curl -s http://$n.localhost/metrics | grep '^bee_pullsync_chunks_delivered ' + done + ``` + +6. **Cleanup:** `beekeeper delete bee-cluster --cluster-name=local-dns --geth-url http://geth-swap.localhost`. + +## What happens (measured; consistent across runs, numbers from the 31-min staked run) + +| Phase | t after removal | Observation | +| --- | --- | --- | +| Quiet | 0–~8 min | radius 3, `within_radius=0`, `fullySynced=true`, rounds advance. Delayed, variable onset (~3–9 min). | +| Onset | ~8 min | `within_radius` jumps 0→~2200 on all survivors; some radius 3→2; `pullsyncRate` spikes (0.7–0.9); `fullySynced→false`. | +| Stall | ~9–29 min | all survivors stuck `fullySynced=false`; `pullsyncRate` decays monotonically toward 0; `bee_pullsync_chunks_delivered` **plateaus** after one onset burst while `offered` ran 24k–34k; rounds advance but stuck nodes can't participate. | +| End | ~30 min | **split-brain, never re-converged** — nodes at *different* radii (3/2/2/3) and four `within_radius` plateaus. Some flip *back* to `fullySynced=true` once `pullsyncRate→0`, but `delivered` stayed flat — they "synced" by the **puller giving up** (SyncRate=0) at inconsistent radii, not by completing. No hard `isFrozen`. No recovery in 30 min. | + +**Smoking gun:** `bee_pullsync_chunks_delivered` goes flat while `fullySynced=false` — the puller gives +up before completing the re-sync. `offered/delivered` ran ~2.4×–17.7× — SWIP-25's k×-redundancy problem. + +## Redistribution round-loss (staked run, 2026-06-26 — the economic impact) + +With all six nodes staked, `bee_storageincentives_*` at the stall showed: + +- `is_playing_errors = 3` on **all four** survivors — every node errors when it tries to play. +- A node that never finishes the re-sync has `reserve_sample_duration = 0` and `winner = 0` (e.g. + bee-2) — it **cannot produce a valid `ReserveSample`, so it cannot win any round**. +- Rounds keep advancing (no hard freeze at this scale), so the chain does not pause — staked survivors + simply **lose every round**. At network scale, with frozen rounds and slashing, this is the October halt. + +## Findings + +- The radius **increase** is deterministic (committed-depth lever); the **decrease** is not reliably + stageable on a small cluster (see Mechanism). The halt does not require a decrease. +- On **A** the cluster **never re-converges** after disruption (persistent ≥30 min, no recovery). +- The small-cluster signature is a **soft stall / non-convergence**, not a hard `isFrozen` freeze — but + staked un-synced survivors still lose every redistribution round, which is the halt at scale. + +## Gotchas + +- **Radius 3 is required** — radius 1–2 is degenerate (`within_radius=0`, nothing to stall on). +- **Ephemeral storage (emptyDir)** — recreate, don't restart; re-stake after every recreate. If a node + was restarted and lost its chequebook (`/status` 503, "no chequebook found"), recover with + `beekeeper node-funder --namespace local --min-native 1 --min-swarm 100`. +- **Staking** — use `POST /stake/` per node; `ci-stake` does not stake the cluster. +- **Pull-sync must be healthy** — if node logs show `pushsync … context deadline exceeded` cluster-wide, + chunks won't propagate, the reserve won't fill, and the radius stays 0. Restart the beelocal substrate + (a cluster recreate alone does not fix it). +- **Chain price must be non-zero** (price 0 breaks postage/uploads). +- **Observe ≥25 min** — the delayed onset means short windows wrongly conclude "no effect". + +## A/B proof (next) + +Run this identical recipe against **B = `pullsync-optimal-design`** (built with the same `radius_*` +patches) and confirm the survivors re-converge (`fullySynced→true`) where A stays stuck. The staked run +gives the decisive B-side metric: survivors should keep `is_playing_errors`/`winner` healthy because +they finish the re-sync and can still sample. That is the proof the redesign fixes the halt. + +Artifacts from the staked run: `radius-drive.csv` (radius 0→3) and `halt-timeline.csv` (survivors, 30 s, +full signal set incl. stake + round + delivered). From 3695640d70bc2ffba7a96e30be3881a8229b7e7e Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 12:12:07 +0200 Subject: [PATCH 22/37] docs: mark resequenced observe-disrupt wait item done --- docs/radius-halt-check-plan.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 992c4d61..83580f4e 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -150,10 +150,9 @@ or just monitor, all without a red result. - [x] `mode: halt` (`runHalt`): stake → warmup → `driveAllToRadius` (gates on the **min** node radius, not the max, so ALL observed nodes reach `DisruptAtRadius`) → settle window. `updatePeak` now returns `(min, max)`. Disruption (Phase 3) + outcome observation (Phase 4) are stubbed with TODOs. -- [ ] `mode: observe`+disrupt: poll until every observed node reports +- [x] `mode: observe`+disrupt: poll until every observed node reports `storageRadius >= DisruptAtRadius` (load drives), with a timeout. - **(Resequenced → Phase 3: the wait-to-radius helper is dead code until the observe+disrupt - dispatch exists, which needs the Phase-3 `disrupt-mechanism`/`disrupt-node-count` options.)** + **(Resequenced → done in Phase 3.0 as `waitAllReachRadius`.)** - [x] Record a pre-disruption baseline snapshot (`baselineSnapshot`): per node, storageRadius + reserveSizeWithinRadius (/status), isFullySynced + round/lastPlayed/lastWon (/redistributionstate), and stake — emitted + logged. Wired into `runHalt` (replaces the plain From 560d0324df711531397b9ef2dfa17ed7089e947b Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 12:14:46 +0200 Subject: [PATCH 23/37] docs: defer batch-expiry to live co-development in phase 6 --- docs/radius-halt-check-plan.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 83580f4e..267eb53f 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -187,13 +187,14 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr - [x] Guard: `len(observed) - DisruptNodeCount >= MinSurvivors`, plus a candidate-count check — both error out **before** any removal touches the cluster. -**3b — batch-expiry** *(sequenced last among the code items)* -> **Sequenced after Phase 5 (not a hard blocker).** Phase 4 (the observe loop it folds into) now exists, -> so 3b is unblocked. But batch-expiry is the **secondary** mechanism and its core behaviour — the -> *gated* radius decrease (`SyncRate==0` + `count question that can't be behaviour-verified blind. Phase 5 (config + docs for the complete node-churn -> primary path) is fully verifiable now and is the prerequisite for Phase-6 validation, so it goes first; -> 3b is implemented best-effort and validated live alongside Phase 6. +**3b — batch-expiry** *(DEFERRED to live co-development in Phase 6 — decided 2026-06-29)* +> **Decision (user, 2026-06-29):** defer batch-expiry to live co-development. The whole primary +> deliverable (node-churn halt, end-to-end, both shapes, config, docs) is complete and verified. +> Batch-expiry is the **secondary** mechanism and its defining behaviour — the *gated* radius decrease +> (`SyncRate==0` + `count patched bee image + a live `local-dns` cluster. Rather than ship speculative, rework-prone code, it is +> implemented **during Phase 6**, when the cluster is up and the eviction → gated-decrease path can be +> observed and tuned against real signals. The autonomous code loop ended here. - [ ] Create a soon-to-expire mutable batch — `CreatePostageBatch(ctx, amount, depth, label)` with a small explicit `amount`, or `GetOrCreateMutableBatch` with a short `expiring-batch-ttl`. Drive the radius (Phase 2) by uploading the fill **under this batch** so its chunks dominate the reserve. From a2d67f2c845c24cf7ad635a04a00f39a6336d51f Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 13:11:43 +0200 Subject: [PATCH 24/37] docs(check): require --timeout=95m for ci-radius-halt; record live validation --- docs/radius-halt-check-plan.md | 16 +++++++++++----- pkg/check/reserveradius/README.md | 7 +++++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 267eb53f..2a556832 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -234,11 +234,17 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr links the recipe). ### Phase 6 — A/B validation -- [ ] Run `ci-radius-halt` against **A** (patched stock master) → expect halt reproduced - (`expect-recovery: false` passes; matches the manual run's numbers). -- [ ] Exercise **both mechanisms** on A: `node-churn` (matches the manual repro) and `batch-expiry` - (the commitment-drop → radius-decrease → merge path); confirm each reaches `HALT` or, when the - decrease gate doesn't trip, `MONITORED` (not a failure). +- [x] Run `ci-radius-halt` against **A** (patched stock master) → **halt reproduced live on `local-dns` + (6 nodes, 2026-06-29)**, matching the manual numbers exactly: staking confirmed on-chain on all 6; + `driveAllToRadius` took all nodes 0→3 in 8 uploads (~64 MiB, 3m42s); node-churn stopped 2 random + nodes (uploader excluded), 4 survivors; onset at ~8m (`within_radius` 0→~2250, `pullsyncRate` spike, + `fullySynced→false`); end-state stuck `fullySynced=false` with **staked round-loss** (round 17, + `lastPlayed/Won=0`) + a split-brain node at radius 1. **Operational fix:** the global `--timeout` + flag DEFAULTS TO 30m and caps the per-check `timeout: 90m` (child context) — must pass + `--timeout=95m` or the run is killed mid-observe. README updated; a clean verdict run with the + correct timeout confirms the `HALT` classification. +- [ ] Exercise **both mechanisms** on A: `node-churn` ✅ (matches the manual repro) and `batch-expiry` + (deferred — see Phase 3b; the commitment-drop → radius-decrease → merge path, co-developed here). - [ ] Build **B** (`pullsync-optimal-design` + the same `radius_*.patch`) and run with `expect-recovery: true` → expect survivors re-converge. - [ ] `make build vet lint test-race` green. No unit tests required for the check (per prior decision); diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index d6357662..3072e8de 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -177,11 +177,14 @@ Against a patched local cluster (`local-dns`): --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info # halt (self-driving): stake → drive all nodes to radius 3 → remove 2 of 6 → observe + classify -./dist/beekeeper check --cluster-name=local-dns --checks=ci-radius-halt \ +# NOTE: pass --timeout >= the per-check timeout (90m). The global --timeout flag DEFAULTS TO 30m and +# caps the per-check context, so without it the ~36-min pipeline (stake+drive+settle + 30m observe) +# is killed mid-observe with "context deadline exceeded". +./dist/beekeeper check --cluster-name=local-dns --checks=ci-radius-halt --timeout=95m \ --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info # halt (parallel-with-load): load drives the radius, reserve-radius stakes/disrupts/observes -./dist/beekeeper check --cluster-name=local-dns \ +./dist/beekeeper check --cluster-name=local-dns --timeout=95m \ --checks=ci-load-soak,ci-reserve-radius-observe-disrupt --parallel \ --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info ``` From 1335327baea7adc06a9d9bb0a2b548e24d9147be Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 14:22:44 +0200 Subject: [PATCH 25/37] fix(check): gate halt recovery on recovery-wait so late re-sync counts as stuck --- pkg/check/reserveradius/reserveradius.go | 25 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 4dc4089f..394ba370 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -506,10 +506,11 @@ type outcomeNode struct { refPlayed uint64 refWon uint64 - onset bool // lost sync (fullySynced true->false) after disruption - onsetAt time.Time // when the onset was first seen - recovered bool // returned to fullySynced after an onset - recoverBy time.Time // recovery deadline after onset (onsetAt + RecoveryWait) + onset bool // lost sync (fullySynced true->false) after disruption + onsetAt time.Time // when the onset was first seen + recovered bool // returned to fullySynced WITHIN RecoveryWait after an onset + lateResync bool // re-synced, but only after RecoveryWait (too slow — a halt symptom) + recoverBy time.Time // recovery deadline after onset (onsetAt + RecoveryWait) haveLast bool lastRound uint64 @@ -568,10 +569,18 @@ func (c *Check) observeOutcome(ctx context.Context, survivors orchestration.Clie c.metrics.OnsetSeconds.WithLabelValues(name).Set(ns.onsetAt.Sub(disruptedAt).Seconds()) c.logger.Warningf("observe-outcome: %s de-synced (fullySynced=false) at round %d, %s after disruption — onset", name, r.Round, ns.onsetAt.Sub(disruptedAt).Round(time.Second)) } - // recovery: back to fullySynced (and not frozen) after an onset - if ns.onset && !ns.recovered && r.IsFullySynced && !r.IsFrozen { - ns.recovered = true - c.logger.Infof("observe-outcome: %s re-converged %s after onset", name, time.Since(ns.onsetAt).Round(time.Second)) + // recovery: back to fullySynced (and not frozen) WITHIN RecoveryWait. A + // re-sync after the deadline is too slow to count — it stays not-recovered + // so it classifies as stuck (the "puller gave up at SyncRate=0" false-sync + // is exactly this: it re-flags fullySynced late without truly converging). + if ns.onset && !ns.recovered && !ns.lateResync && r.IsFullySynced && !r.IsFrozen { + if time.Now().Before(ns.recoverBy) { + ns.recovered = true + c.logger.Infof("observe-outcome: %s re-converged %s after onset (within recovery-wait)", name, time.Since(ns.onsetAt).Round(time.Second)) + } else { + ns.lateResync = true + c.logger.Warningf("observe-outcome: %s re-synced %s after onset — PAST recovery-wait %s, too slow (counts as halt)", name, time.Since(ns.onsetAt).Round(time.Second), o.RecoveryWait) + } } } } From 8d9fdba2f31eca2afbbf8a48fdf46e707c2585a2 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 29 Jun 2026 14:23:05 +0200 Subject: [PATCH 26/37] docs: record clean HALT verdict and recovery-gate fix from live run --- docs/radius-halt-check-plan.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 2a556832..4386b6a9 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -241,8 +241,13 @@ Shared: a `disruption_total` metric + timestamp marks the onset reference. `disr `fullySynced→false`); end-state stuck `fullySynced=false` with **staked round-loss** (round 17, `lastPlayed/Won=0`) + a split-brain node at radius 1. **Operational fix:** the global `--timeout` flag DEFAULTS TO 30m and caps the per-check `timeout: 90m` (child context) — must pass - `--timeout=95m` or the run is killed mid-observe. README updated; a clean verdict run with the - correct timeout confirms the `HALT` classification. + `--timeout=95m` or the run is killed mid-observe. README updated. A clean `--timeout=95m` run then + **completed successfully with `outcome=HALT`** (verdict=report). **Two findings from the clean run:** + (a) classification bug — recovery wasn't gated on `recovery-wait`, so a too-slow re-sync was + mis-counted as recovered; **fixed** (late re-sync now counts as stuck; the "puller gave up at + SyncRate=0" false-sync is exactly this). (b) **scale caveat** — on 4 survivors the halt shows as + de-sync + round-loss + *late* re-sync (~26m), not permanent stuck; the forever-stuck halt likely + needs ~20 nodes (Phase 7). A/B discrimination must confirm B recovers *within* recovery-wait. - [ ] Exercise **both mechanisms** on A: `node-churn` ✅ (matches the manual repro) and `batch-expiry` (deferred — see Phase 3b; the commitment-drop → radius-decrease → merge path, co-developed here). - [ ] Build **B** (`pullsync-optimal-design` + the same `radius_*.patch`) and run with From ee03d17861385eba9e8639ea061f275411fb232f Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Sat, 11 Jul 2026 21:35:34 +0200 Subject: [PATCH 27/37] feat: implement radius check --- config/local.yaml | 106 +- docs/radius-check-plan.md | 76 +- docs/radius-halt-check-plan.md | 11 +- docs/radius-halt.md | 12 +- pkg/check/reserveradius/README.md | 275 ++---- pkg/check/reserveradius/metrics.go | 129 +-- pkg/check/reserveradius/reserveradius.go | 1128 +++++----------------- pkg/config/check.go | 43 +- 8 files changed, 424 insertions(+), 1356 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index a702d84f..7e661679 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -374,104 +374,34 @@ checks: - light timeout: 30m type: load + # Self-driving radius check: upload random data to random full nodes (own postage label) to + # drive the storage radius up to target-radius (3); then wait for pull-sync to go idle + # (pullsyncRate==0 — the reserve worker only decreases while SyncRate()==0), dilute one batch + # down to the chain's minimum validity so it expires soon (commitment drop), and verify the + # radius ticks back down below its peak (3 -> 2). Needs the patched bee image (small reserve). + # Locally the reserve-worker idle tick drops the radius in ~10 min (before any expiry); on + # testnet the drop is expiry-driven (~minimumValidityBlocks after diluting, a few hours) — + # there, raise decrease-timeout to ~6h and timeout to ~8h (a chain postage price must be set). ci-reserve-radius: options: - postage-ttl: 24h + postage-ttl: 48h # must exceed the 24h min-validity floor (24h lands on the boundary and is rejected) postage-depth: 22 postage-label: reserve-radius - blob-size: 1048576 - max-uploads: 60 - target-radius: 1 + blob-size: 8388608 # 8 MiB — bigger blobs reach the higher target radius faster + max-uploads: 200 + target-radius: 3 + force-decrease: true # dilute one batch to minimum validity to hasten its expiry (commitment drop) + dilute-step: 1 # +1 per dilution ~halves TTL; larger jumps revert at the min-validity floor + max-dilutions: 20 warmup-wait: 15m - increase-timeout: 5m - settle-wait: 1m - decrease-timeout: 20m + increase-timeout: 30m + sync-settle-wait: 10m # wait for pullsyncRate==0 before diluting/observing (the decrease gate) + decrease-timeout: 30m # local: the idle tick drops it in ~10min. testnet (expiry-driven): ~6h poll-interval: 15s upload-groups: - bee timeout: 45m type: reserve-radius - ci-reserve-radius-observe: - options: - mode: observe - disrupt-mechanism: none # monitor-only soak; set node-churn to run the staged observe+disrupt reproduction - duration: 30m - recovery-wait: 10m - poll-interval: 15s - upload-groups: - - bee - timeout: 35m - type: reserve-radius - # Self-driving pull-sync-halt reproduction (single check): stake → drive all nodes to radius 3 → - # remove 2 of 6 → observe the survivors. verdict:report records HALT/RECOVERED/MONITORED without - # failing; for the A/B gate set verdict:assert + expect-recovery (false on A=stock, true on B=fix). - ci-radius-halt: - options: - mode: halt - stake-amount: "100000000000000000" # 1e17 per node (quoted: option is a string) - stake-confirm-wait: 2m - disrupt-at-radius: 3 - disrupt-mechanism: node-churn - disrupt-node-count: 2 - disrupt-method: stop - min-survivors: 3 - verdict: report - expect-recovery: false - recovery-wait: 10m - duration: 30m # observe ≥25 min — onset is delayed ~3–9 min - poll-interval: 30s - postage-ttl: 24h - postage-depth: 22 - postage-label: radius-halt - blob-size: 8388608 # 8 MiB - max-uploads: 1000 - warmup-wait: 15m - increase-timeout: 30m - settle-wait: 2m - upload-groups: - - bee - timeout: 90m - type: reserve-radius - # Parallel-with-load shape: run alongside ci-load-soak (which drives the radius via - # max-committed-depth:3). This check stakes, waits for radius 3, removes 2 of 6, then observes. - ci-reserve-radius-observe-disrupt: - options: - mode: observe - stake-amount: "100000000000000000" - stake-confirm-wait: 2m - disrupt-at-radius: 3 - disrupt-mechanism: node-churn - disrupt-node-count: 2 - disrupt-method: stop - min-survivors: 3 - verdict: report - expect-recovery: false - recovery-wait: 10m - duration: 30m - poll-interval: 30s - increase-timeout: 30m # max wait for the load check to drive all nodes to radius 3 - warmup-wait: 15m - upload-groups: - - bee - timeout: 90m - type: reserve-radius - ci-load-soak: - options: - content-size: 8388608 - postage-ttl: 24h - postage-depth: 22 - postage-label: load-soak - duration: 30m - uploader-count: 1 - downloader-count: 0 - nodes-sync-wait: 10s - max-committed-depth: 3 - committed-depth-check-wait: 15s - decrease-hold: 2m - upload-groups: - - bee - timeout: 35m - type: load ci-soc: options: postage-ttl: 24h diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md index 296e6443..4731eca9 100644 --- a/docs/radius-check-plan.md +++ b/docs/radius-check-plan.md @@ -1,6 +1,6 @@ # Reserve-radius change check — plan -Status: **Phases 0–2.5 done; Phase 3 + A/B pending.** Branch `ljubisa-radius-soak`. +Status: **Local check done (single idle-driven flow); Phase 3 + A/B pending.** Branch `ljubisa-radius-soak-v2`. A repeatable, automated way to stage a **storage-radius change** across a Bee cluster and verify pull-sync recovers afterwards. End product: a beekeeper check @@ -28,11 +28,18 @@ delay) and stalls livelock. Result: incomplete reserves → wrong `ReserveSample - **Bounded convergence** — resync finishes in bounded time; no stuck `STALLED` bins. - **Efficiency (SWIP-25)** — redundant deliveries drop ~k×→1× (`offered/delivered` ratio). -**Implemented so far:** observe mode reads `/status` + `/redistributionstate` and asserts the -halt indicators — un-recovered decreases (prefers `isFullySynced`, falls back to `pullsyncRate`) -and **freeze episodes**. Metrics: `fully_synced`, `frozen`, `redistribution_round`, -`reserve_within_radius`, `last_sample_duration_seconds`, `time_to_fully_synced_seconds`. The -`radius-cluster-behavior` dashboard has a **Halt indicators** row. +**Implemented so far:** a single self-driving check (`pkg/check/reserveradius`) drives the radius +up by uploading random data to random nodes to a target (≥3), stops, waits for pull-sync to go idle +(`pullsyncRate<=0.05` — the reserve worker only decreases while `SyncRate()==0`), dilutes one batch +to the chain's min-validity floor to hasten its expiry (force-decrease), then asserts the radius +ticks back down below its peak within a timeout. Metrics per node: `storage_radius`, `reserve_size`, +`reserve_within_radius`, `pullsync_rate`, plus `time_to_increase_seconds` / `time_to_decrease_seconds` +/ `dilution_total`. Validated live 2026-07-11 on the patched 6-node local-dns: 0→3 in ~64 MiB / +~6 min, batch diluted 48h→6h (floored), radius 3→2 at ~8 min. **Locally the decrease is pull-sync- +idle-gated** (threshold patched to 100% ⇒ commitment/expiry irrelevant); **on testnet it is +expiry-driven** — dilution hastens the batch expiry that drops commitment. The deeper liveness +signals (`/redistributionstate` freeze / round-loss) +are documented in the `radius-testing` skill for follow-up. **Still needs @marios + scale:** exact pass/fail thresholds; the per-`(peer,bin)` `STALLED` / `MaxStallsPerBin` metrics (only @sig's fixed PR exposes them); the **~20-node cluster** (the @@ -115,29 +122,40 @@ The decrease is slow and non-obvious; each fact below maps to a check decision: ## Phase 2 — The check (done) -`pkg/check/reserveradius/` on branch `ljubisa-radius-check`. `Run` = `waitForWarmupDone` → -baseline → `driveIncrease` → settle → `observeDecrease` (assert a decrease vs per-node peak + -recovery). Registered as type `reserve-radius` (`pkg/config/check.go`); `ci-reserve-radius` in -`config/local.yaml`; added `IsWarmingUp` to `StatusResponse`. build/vet/lint green. - -- [x] **Validated end-to-end (2026-06-17)**: increase 0→1 in 3s, 1-min settle, decrease observed - at 13m16s with recovery, check passed (total 14m36s). Confirms the ~13-min decrease lag and - the 20-min `DecreaseTimeout`. - -## Phase 2.5 — driver/observer split + parallel (done) - -For long soaks (24h+), split the **uploader** (load) from the **observer** (reserve-radius) and -run them concurrently so the cluster oscillates repeatedly. - -- [x] **`--parallel` flag** (`check.go` + `runner.go`) — opt-in; checks run in goroutines and - fail independently (a monitor blip won't kill a 24h load). -- [x] **`load` re-arming `decrease-hold`** — after `committedDepth` hits the cap and drops, pause - uploads then re-arm → repeated fill→decrease→hold cycles. -- [x] **`reserve-radius` `mode: drive | observe`** — `observe` = `Duration`-bounded monitor (no - uploads) recording transitions + asserting recovery. Config: `ci-load-soak`, - `ci-reserve-radius-observe`. - -Run: `beekeeper check --checks=ci-load-soak,ci-reserve-radius-observe --parallel`. +`pkg/check/reserveradius/`. `Run` = `selectNodes` → `waitForWarmupDone` → baseline → +`driveIncrease` (upload random data to random nodes under this check's own postage label, to +`target-radius`) → `waitPullSyncIdle` (wait `pullsyncRate<=0.05` on all nodes) → +`diluteToMinValidity` (force-decrease: dilute one batch to the min-validity floor to hasten expiry) +→ `observeDecrease` (assert the radius ticks below its peak within `decrease-timeout`). Registered +as type `reserve-radius` (`pkg/config/check.go`); `ci-reserve-radius` in `config/local.yaml`; added +`IsWarmingUp` to `StatusResponse`. build/vet/lint green. + +- [x] **Validated end-to-end (2026-06-17)**: increase 0→1 in 3s, decrease observed at 13m16s. +- [x] **Re-validated (2026-07-11, target 3, dilute-to-min-validity + drop)**: 0→3 in ~64 MiB / + ~6 min, pull-sync idled, diluted a batch 48h→24h→12h→6h (floored at the min-validity), radius + 3→2 at ~8 min, check passed. Needs the patched image (the default registry image is unpatched + — the radius won't climb). + +## Phase 2.5 — the decrease lever: pull-sync idle (local) / batch expiry (testnet) (done) + +Two paths reach the decrease, both gated on `SyncRate()==0`: + +- **Local (patched cluster)** — the reserve worker's gate (`reserve.go`: + `countWithinRadius < threshold && SyncRate()==0 && radius > minimumRadius`) reduces to just + **`SyncRate()==0`** (`threshold` patched to 100%, `countWithinRadius` well below it), so once + pull-sync idles the radius ticks down on its own — measured ~8–13 min after idle. `waitPullSyncIdle` + (`pullsyncRate<=0.05`; the rate settles to a small residual, never exactly 0) opens that gate. +- **Testnet (real reserve)** — the decrease is **commitment-driven**: it needs a batch to expire so + its chunks are evicted. `force-decrease` **dilutes one batch to the chain's `minimumValidityBlocks` + floor** (each `+1` ~halves TTL; the contract reverts any dilution below the floor, so expiry itself + is unreachable — dilution only *hastens* it) so the batch expires in ~a few hours instead of its + full TTL. Requires a non-zero chain price (see the `change-storage-price` skill); with price 0 + batches have infinite TTL. Full mechanics: [`docs/radius-halt.md`] and the radius-testing notes. + +The general `load` primitives added along the way remain available but are not wired into the radius +check: the `--parallel` check flag (`check.go` + `runner.go`) and `load`'s re-arming `decrease-hold`. + +Run: `beekeeper check --checks=ci-reserve-radius --timeout=45m` (local; raise `decrease-timeout`/`timeout` on testnet). ## Phase 3 — Real ephemeral cluster diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md index 4386b6a9..d3c0c28b 100644 --- a/docs/radius-halt-check-plan.md +++ b/docs/radius-halt-check-plan.md @@ -1,6 +1,15 @@ # Reserve-radius check — halt-reproduction mode (plan) -Status: **Plan only — not started.** Branch `ljubisa-radius-soak`. +> **Superseded (2026-07).** The `reserveradius` check was simplified to a **single +> self-driving flow with one disruption mechanism: batch-expiry via dilution** (see +> `docs/radius-check-plan.md` and `pkg/check/reserveradius/README.md`). The `mode: halt` +> design below — modes, **node churn**, staking, verdict/assert, and HALT/RECOVERED +> outcome classification — was implemented on this branch and then removed. This doc is +> kept for the background it captures (the bee-side batch-expiry → commitment-drop → +> gated-radius-decrease path, and the empirical halt findings that motivated the check); +> it no longer describes the current code. + +Status: **Superseded — see the banner above.** Branch `ljubisa-radius-soak`. Goal: fold the now-working **manual** pull-sync-halt reproduction (see `docs/radius-halt.md`) into the `reserveradius` check so the whole flow runs **unattended in one longer run** — diff --git a/docs/radius-halt.md b/docs/radius-halt.md index dec984cb..6f5507ff 100644 --- a/docs/radius-halt.md +++ b/docs/radius-halt.md @@ -9,11 +9,13 @@ fully-staked run (2026-06-26) that pins down the **staked round-loss**: every su `is_playing_errors`, and a node that cannot finish the re-sync cannot produce a `ReserveSample`, so it loses every round. Remaining: the **A/B** confirmation against **B = `pullsync-optimal-design`**. -**Automation:** this manual recipe is now folded into the `reserveradius` check's `mode: halt` (and the -`observe`+disrupt shape) — stake → drive radius → disrupt → observe/classify/verdict, unattended. See -the check plan [`docs/radius-halt-check-plan.md`](radius-halt-check-plan.md) and the check -[`pkg/check/reserveradius/README.md`](../pkg/check/reserveradius/README.md); the `ci-radius-halt` and -`ci-reserve-radius-observe-disrupt` entries live in `config/local.yaml`. +**Automation (historical):** this manual recipe used **node removal** to disrupt the neighbourhood. +The `reserveradius` check was since simplified to a single self-driving flow whose only disruption +lever is **batch-expiry via dilution** — node churn was dropped (see +[`pkg/check/reserveradius/README.md`](../pkg/check/reserveradius/README.md) and +[`docs/radius-check-plan.md`](radius-check-plan.md)). The findings below still stand as the empirical +characterization of the halt; the node-churn automation design is retained in +[`docs/radius-halt-check-plan.md`](radius-halt-check-plan.md) (superseded). ## Mechanism diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 3072e8de..3ceb8b30 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -1,22 +1,14 @@ # reserve-radius check -Exercises a **storage-radius change** across a Bee cluster and verifies the node -recovers afterwards — catching regressions in the puller's reaction to a radius -change (the `manage()` / `disconnectPeer()` path that caused the liveness bug in -beekeeper PR #581). - -It has three modes (`Options.Mode`): - -- **`drive`** (default) — a one-shot run: upload to force the radius up, then watch - it come back down and assert pull-sync recovers. -- **`observe`** — a long-running monitor: does **not** upload; watches radius - transitions for `Duration` while something else drives the cluster (typically the - `load` check running in parallel via `--parallel`), recording every up/down - transition and asserting recovery after each decrease. This is the soak mode. -- **`halt`** — self-driving pull-sync-halt reproduction: stake → drive **all** observed - nodes to `DisruptAtRadius` → settle → disrupt the neighbourhood → observe the outcome. - Disruption mechanisms and outcome classification are being built out (Phases 3–4); - currently it runs the stake/drive/settle prefix. +Drives a **storage-radius change** on a Bee cluster and verifies the radius comes +back **down** once pull-sync settles — exercising the reserve worker's decrease path +(the puller reconfiguration that caused the liveness bug in beekeeper PR #581). + +One self-driving run pushes the radius **up** by uploading, then stops and waits for +pull-sync to go idle — the reserve worker only decreases the radius while +`SyncRate()==0` (== `/status pullsyncRate`) — optionally dilutes a batch down to the chain's +minimum validity so it expires soon (hastening the commitment drop), and confirms the radius +ticks back down. > Keep this file in sync with the code. If you change `Options`, the `Run` flow, the > emitted metrics, the registration, or the bee-patch requirement, update the matching @@ -24,113 +16,62 @@ It has three modes (`Options.Mode`): ## What it does (`Run` flow) -`Run` dispatches on `Mode`. Both modes first run an optional **staking pre-step**. - -**staking pre-step** (`ensureStaked`, both modes): when `stake-amount` is set, ensure every node in -the staking set (`stake-groups`, or the observed nodes) has at least that stake — `GetStake`, and if -below, `DepositStake` then poll until the deposit confirms on-chain (within `stake-confirm-wait`). -Idempotent: nodes already at/above target are skipped. Runs before warmup so deposits confirm while -the cluster stabilizes. - -**`drive`** (`runDrive`): - -1. **`waitForWarmupDone`** — block until every observed node reports - `isWarmingUp == false`. The reserve worker's *decrease* loop is gated on - stabilization, so this is a precondition for the radius ever dropping. -2. **baseline** — snapshot `/status` for all observed nodes. -3. **`driveIncrease`** — buy a mutable batch on a random full node, then upload - `BlobSize` random blobs to it until any observed node's `storageRadius` reaches - `TargetRadius` (or `MaxUploads` / `IncreaseTimeout`). Tracks a **per-node - high-water `peak`**. -4. **settle** — keep polling for `SettleWait`, still raising `peak` (on an - already-stabilized node the decrease can begin *during* settle, so `peak` must be - the max seen, not a single post-settle read). -5. **`observeDecrease`** — poll until any node's `storageRadius` falls below its - `peak`, watching `pullsyncRate` for recovery. No decrease within `DecreaseTimeout` - fails with a message pointing at the PR #581 puller stall. - -**`observe`** (`runObserve`): `waitForWarmupDone`, then **dispatches**: - -- **with disruption configured** (`disruptionActive`: an active `disrupt-mechanism` — by default - `node-churn` with `disrupt-node-count > 0`) → the **observe+disrupt** staged reproduction - (`runObserveDisrupt`): `waitAllReachRadius` (wait for the externally-driven radius to reach - `DisruptAtRadius`) → `baselineSnapshot` → `disrupt` → `observeOutcome` → `applyVerdict`. The - parallel-with-`load` shape (same stages as `halt`, minus the self-driving upload). -- **monitor-only** (`disrupt-mechanism: none`, or `node-churn` with `disrupt-node-count: 0`) → the - plain soak monitor below. - -> **Behaviour note:** with the default `node-churn`/count-2, a bare `mode: observe` now runs -> observe+disrupt (which **stops nodes**). Set `disrupt-mechanism: none` for a pure soak monitor. - -The soak monitor: for `Duration` poll every `PollInterval`, reading both `/status` and -**`/redistributionstate`** per node. It records up/down `storageRadius` transitions, and asserts the -**halt indicators** that define the October failure: - -- **recovery after a decrease** — after a **down** transition, the node must recover within - `RecoveryWait`. Recovery prefers the redistribution signal **`isFullySynced && !isFrozen`** - (the real "can play the game" signal); if `/redistributionstate` is unavailable (not - full-mode), it falls back to `pullsyncRate > 0`. -- **freeze episodes** — a node reporting `isFrozen` is skipping redistribution rounds (a halt - symptom); each freeze episode is counted. - -Un-recovered decreases or freezes fail the check at the end. Never uploads (the radius is -driven externally, e.g. by a parallel `load` check). - -**`halt`** (`runHalt`): the self-driving reproduction. - -1. **stake** (optional `ensureStaked` pre-step, as above). -2. **`waitForWarmupDone`** + **`baselineSnapshot`** — the pre-disruption reference: per node, storage - radius + `reserveSizeWithinRadius`, `isFullySynced` + round/lastPlayed/lastWon, and stake. -3. **`driveAllToRadius`** — buy a mutable batch and upload until **every** observed node's - `storageRadius` reaches `DisruptAtRadius` (gates on the **min** across nodes, so the whole - neighbourhood is populated before disruption), or `MaxUploads` / `IncreaseTimeout`. -4. **settle** — poll for `SettleWait`. -5. **`disrupt`** — apply `DisruptMechanism`. **node-churn** (`disruptNodeChurn`): randomly pick - `DisruptNodeCount` nodes (seeded by `rnd-seed`, excluding the uploader), guard `MinSurvivors` - **before** touching the cluster, then `Stop` (scale-0) or `Delete` each; returns the **survivor set**. - `none` / `disrupt-node-count: 0` is monitor-only; `batch-expiry`/`both` are Phase 3b (not yet). -6. **`observeOutcome`** — poll the survivors for `Duration`, tracking per node the **onset** of de-sync - (`isFullySynced` true→false), **recovery** (back within `RecoveryWait`), and staked **round-loss** - (`round` advancing while `lastPlayed`/`lastWon` stall). `classifyOutcome` reduces this to - `MONITORED` (no disruption) / `HALT` (a survivor stuck de-synced past `RecoveryWait` and/or - round-loss) / `RECOVERED` (all de-synced survivors re-converged), emitting the `outcome` gauge. -7. **`applyVerdict`** — `report` (default) always succeeds on the outcome (only operational errors - fail); `assert` fails iff the outcome contradicts `expect-recovery` (`false` ⇒ expect `HALT`, - `true` ⇒ expect `RECOVERED`), with `MONITORED` always passing. This is the A/B regression gate. +A single flow — there are no modes. + +1. **`selectNodes`** — shuffle the full-node clients (seeded by `rnd-seed`), optionally + filtered to `upload-groups`. +2. **`waitForWarmupDone`** — block until every observed node reports `isWarmingUp == false`. + The reserve worker's decrease loop is gated on stabilization, so this is a precondition + for the radius ever dropping. +3. **baseline** — snapshot `/status` for all observed nodes. +4. **`driveIncrease`** — upload `BlobSize` random blobs to **randomly-chosen** nodes (each + under its own mutable batch tagged with `postage-label`) until any node's `storageRadius` + reaches `TargetRadius`, tracking a per-node high-water `peak`. A transient per-node batch + error is logged and skipped (another random node is picked next). +5. **`waitPullSyncIdle`** — stop uploading and wait until every node reports + `pullsyncRate <= 0.05` (the rate settles to a small residual, never exactly 0), or + `SyncSettleWait` elapses (then it proceeds anyway). This opens the decrease gate. +6. **`diluteToMinValidity`** (when `force-decrease`) — dilute one batch, `+DiluteStep` at a + time (each step ~halves its remaining TTL), until a dilution is rejected because it would + drop validity below the chain's `minimumValidityBlocks` floor. Dilution **cannot reach + expiry** (the floor stops it, ~a few hours of validity), so this only *hastens* the natural + expiry — the batch now expires within the check's budget instead of its full TTL. On expiry + its chunks are evicted and reserve commitment drops. Diluting a single batch is a partial + expiry (drops commitment without evicting the whole reserve, which would halt). +7. **`observeDecrease`** — watch for any node's `storageRadius` to fall below its `peak`. + The first decrease passes the check; no decrease within `DecreaseTimeout` fails it (a + stuck puller keeps `SyncRate>0`, so the radius never decreases — cf. PR #581). + +**Two decrease paths.** On the patched local cluster the reserve worker's idle tick drops the +radius ~10 min after pull-sync idles (before any expiry) — validated live: 0→3 in ~64 MiB, then +3→2 at ~8 min. On testnet (real reserve, a chain price set) the decrease is **expiry-driven**: +the diluted batch expires ~`minimumValidityBlocks` after the dilution (a few hours), evicting +its chunks; raise `decrease-timeout` (~6h) and `timeout` (~8h) accordingly. ## Options Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in `pkg/config/check.go` under the `reserve-radius` entry. -| field | yaml | default | mode | purpose | -| --- | --- | --- | --- | --- | -| `Mode` | `mode` | `drive` | all | `drive` (force a change), `observe` (monitor only), or `halt` (self-driving reproduction) | -| `Duration` | `duration` | `12h` | observe | total monitor run length | -| `RecoveryWait` | `recovery-wait` | `5m` | observe | max wait for pull-sync recovery after each decrease | -| `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | both | seed for `random.PseudoGenerator` → shuffled node pick | -| `StakeAmount` | `stake-amount` | `""` (skip) | both | per-node stake (wei) to ensure before driving, e.g. `"100000000000000000"`; empty/`"0"` skips | -| `StakeGroups` | `stake-groups` | `nil` (observed) | both | node groups to stake (empty = the observed/selected groups) | -| `StakeConfirmWait` | `stake-confirm-wait` | `2m` | both | max wait for a deposit to confirm on-chain (~10 blocks) | -| `UploadGroups` | `upload-groups` | `[bee]` | both | node groups to observe (and, in drive, upload to) | -| `PollInterval` | `poll-interval` | `15s` | both | poll cadence | -| `PostageTTL` | `postage-ttl` | `24h` | drive | batch TTL (use TTL, not a raw amount) | -| `PostageDepth` | `postage-depth` | `22` | drive | batch depth | -| `PostageLabel` | `postage-label` | `reserve-radius` | drive | batch label | -| `BlobSize` | `blob-size` | `1048576` (1 MiB) | drive | bytes per upload | -| `MaxUploads` | `max-uploads` | `60` | drive | cap on uploads in the increase phase | -| `TargetRadius` | `target-radius` | `1` | drive | storageRadius any node must reach before stopping uploads | -| `DisruptAtRadius` | `disrupt-at-radius` | `3` | halt | storageRadius **all** observed nodes must reach before disruption | -| `DisruptMechanism` | `disrupt-mechanism` | `node-churn` | halt/observe+disrupt | `node-churn`, `batch-expiry`, `both`, or `none` (monitor-only) | -| `DisruptNodeCount` | `disrupt-node-count` | `2` | halt/observe+disrupt | node-churn: full nodes to remove (randomly, seeded by `rnd-seed`); `0` = skip | -| `DisruptMethod` | `disrupt-method` | `stop` | halt/observe+disrupt | node-churn: `stop` (scale statefulset to 0) or `delete` (statefulset + resources) | -| `MinSurvivors` | `min-survivors` | `3` | halt/observe+disrupt | refuse to disrupt below this many surviving nodes | -| `Verdict` | `verdict` | `report` | halt | `report` (never fail on outcome) or `assert` (gate on `expect-recovery`) | -| `ExpectRecovery` | `expect-recovery` | `false` | halt | `assert` only: `false` ⇒ expect `HALT`, `true` ⇒ expect `RECOVERED` | -| `WarmupWait` | `warmup-wait` | `15m` | both | max wait for nodes to finish warmup | -| `IncreaseTimeout` | `increase-timeout` | `5m` | drive | max time to reach `TargetRadius` | -| `SettleWait` | `settle-wait` | `1m` | drive | post-upload window (pushsync drain + peak tracking) | -| `DecreaseTimeout` | `decrease-timeout` | `20m` | drive | max time to observe a decrease | +| field | yaml | default | purpose | +| --- | --- | --- | --- | +| `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | seed for `random.PseudoGenerator` → shuffled node pick + random uploader choice | +| `PostageTTL` | `postage-ttl` | `48h` | batch TTL; must exceed the chain's 24h minimum validity once a price is set (24h lands exactly on the boundary and is rejected) | +| `PostageDepth` | `postage-depth` | `22` | batch depth | +| `PostageLabel` | `postage-label` | `reserve-radius` | this check's batch label | +| `UploadGroups` | `upload-groups` | `[bee]` | node groups to observe and upload to | +| `BlobSize` | `blob-size` | `8388608` (8 MiB) | bytes per upload (bigger blobs reach a higher target radius faster) | +| `MaxUploads` | `max-uploads` | `200` | cap on uploads in the increase phase | +| `TargetRadius` | `target-radius` | `3` | storageRadius any node must reach before stopping uploads (≥3 leaves headroom above the effective floor) | +| `ForceDecrease` | `force-decrease` | `true` | dilute one batch to minimum validity to hasten its expiry; `false` = idle-tick only | +| `DiluteStep` | `dilute-step` | `1` | depth increase per dilution (~halves TTL); larger jumps revert once they cross the min-validity floor | +| `MaxDilutions` | `max-dilutions` | `20` | cap on dilutions when driving a batch to the validity floor | +| `GasPrice` | `gas-price` | `""` | gas price for the dilute tx (empty = node default) | +| `WarmupWait` | `warmup-wait` | `15m` | max wait for nodes to finish warmup | +| `IncreaseTimeout` | `increase-timeout` | `30m` | max time to reach `TargetRadius` | +| `SyncSettleWait` | `sync-settle-wait` | `10m` | max wait for `pullsyncRate<=0.05` on all nodes (the decrease gate); proceeds anyway on timeout | +| `DecreaseTimeout` | `decrease-timeout` | `6h` | max time to observe a radius decrease (local idle tick ~10min; testnet expiry ~a few hours) | +| `PollInterval` | `poll-interval` | `15s` | poll cadence | ## Metrics (`metrics.go`) @@ -140,92 +81,54 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - `…_storage_radius{node}` — gauge, storageRadius per node - `…_reserve_size{node}` — gauge, reserve size (chunks) per node +- `…_reserve_within_radius{node}` — gauge, reserve chunks within the storage radius (completeness signal) - `…_pullsync_rate{node}` — gauge, `/status` pullsyncRate per node -- `…_time_to_increase_seconds` / `…_time_to_decrease_seconds` — gauges (drive mode) -- `…_reserve_within_radius{node}` — gauge, reserve chunks within radius (completeness signal) -- `…_radius_transitions_total{node,direction}` — counter, observed up/down transitions (observe mode) -- `…_recovery_observed_total{node,result}` — counter, recovery outcome `recovered`/`timeout` (observe mode) -- `…_disruption_total{mechanism}` — counter, neighbourhood disruptions applied (halt mode, e.g. `node-churn`) -- `…_outcome{outcome}` — gauge, one-hot halt-run classification (`MONITORED`/`HALT`/`RECOVERED`); the classified one is 1 -- `…_onset_seconds{node}` — gauge, seconds from disruption to a survivor's de-sync onset (halt mode) -- `…_round_loss_total{node}` — counter, survivors with staked round-loss (rounds advanced while lastPlayed/Won stalled) -- `…_time_to_fully_synced_seconds` — gauge, decrease → isFullySynced again (observe mode) -- from `/redistributionstate` (observe mode): `…_fully_synced{node}`, `…_frozen{node}`, - `…_redistribution_round{node}`, `…_last_sample_duration_seconds{node}` — the halt indicators +- `…_time_to_increase_seconds` — gauge, first upload → `TargetRadius` +- `…_time_to_decrease_seconds` — gauge, pull-sync idle → first observed radius decrease +- `…_dilution_total` — counter, batch dilutions applied when hastening expiry (force-decrease) ## Requirements - **Patched bee image.** On stock capacity the reserve is far too large to move. The check needs a node built with `bee/.github/patches/radius_reserve.patch` - (`DefaultReserveCapacity`→200, `ReserveWakeUpDuration`→10s) and + (`DefaultReserveCapacity`→4000, `ReserveWakeUpDuration`→10s) and `radius_threshold.patch` (decrease `threshold`→100%). See - `docs/radius-check-plan.md` and the `radius-testing` skill. -- **`isWarmingUp` in `StatusResponse`** (`pkg/bee/api/status.go`) — the node returns - it; the struct field was added for this check's warmup gate. + `docs/radius-check-plan.md` and the `radius-testing` skill. The default local-cluster + image is **not** patched — build and push it first, or the radius never climbs. +- **A non-zero postage price is optional.** The decrease is driven by pull-sync idling + (`SyncRate()==0`), not by batch expiry, so it works at price 0 (batches never expire). + If a price *is* set (see the `change-storage-price` skill), the contract enforces a 24h + minimum batch validity, so `postage-ttl` must exceed 24h — hence the `48h` default. +- **`isWarmingUp` in `StatusResponse`** (`pkg/bee/api/status.go`) — the node returns it; + the struct field was added for this check's warmup gate. ## Running it Against a patched local cluster (`local-dns`): ```sh -# drive (one-shot): force a change and assert recovery -./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius --log-verbosity=info - -# soak: load oscillates the radius, reserve-radius observes it, both run concurrently -./dist/beekeeper check --cluster-name=local-dns \ - --checks=ci-load-soak,ci-reserve-radius-observe --parallel \ - --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info - -# halt (self-driving): stake → drive all nodes to radius 3 → remove 2 of 6 → observe + classify -# NOTE: pass --timeout >= the per-check timeout (90m). The global --timeout flag DEFAULTS TO 30m and -# caps the per-check context, so without it the ~36-min pipeline (stake+drive+settle + 30m observe) -# is killed mid-observe with "context deadline exceeded". -./dist/beekeeper check --cluster-name=local-dns --checks=ci-radius-halt --timeout=95m \ - --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info - -# halt (parallel-with-load): load drives the radius, reserve-radius stakes/disrupts/observes -./dist/beekeeper check --cluster-name=local-dns --timeout=95m \ - --checks=ci-load-soak,ci-reserve-radius-observe-disrupt --parallel \ - --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info +./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius \ + --timeout=45m --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info ``` -`--parallel` runs the listed checks in goroutines instead of sequentially; -checks fail independently (a monitor failure does not stop the load run). For the A/B -regression gate set `verdict: assert` + `expect-recovery` (`false` on A=stock master, -`true` on B=pullsync-optimal-design). - ## Observed behavior (local, patched cluster) -- **Increase is fast** — ~1 MiB moves `storageRadius` 0→1 in seconds. -- **Decrease lags ~13 min** and is stabilization-gated; sized by `DecreaseTimeout` (20m). -- **No fixed floor** — the equilibrium radius depends on data volume (a data-heavy - node can stay elevated and never decrease in-window). The check asserts *a decrease - occurred*, not that the radius returns to a specific value. -- **Pull-sync on a radius change is a puller reconfiguration**: `bee_puller_worker` - contracts on increase / expands on decrease, with a `bee_puller_worker_errors` - cancel-storm. Actual resync volume (`bee_pullsync_chunks_delivered`) is ~0 on a - near-empty cluster — meaningful resync needs data (real cluster). - -## Known limitations / follow-ups - -- Observe-mode recovery uses `/redistributionstate` (`isFullySynced`/`isFrozen`) where - available, falling back to `/status` `pullsyncRate` (weak on light clusters — can - false-`timeout`). The **exact thresholds** (the `isFullySynced` bound, acceptable - convergence time, the `offered/delivered` redundancy target) need to come from the - pull-sync design owner (@marios) — see `docs/radius-check-plan.md` "Pull-sync validation". -- Per-`(peer,bin)` `STALLED` / `MaxStallsPerBin` signals from the **fixed** pull-sync - (@sig's PR) are not yet scraped (they don't exist on stock bee). -- The neighbourhood **merge** (`k→8` on a decrease) needs a ~20-node cluster to reproduce; - the local 3-node cluster proves the mechanism and pipeline only. -- Direction is fixed to increase-then-decrease (drive) / observe-both (observe); no - `increase`-only assertion mode yet. -- No unit tests yet (external `_test` package TBD). +- **Increase**: ~48–64 MiB moves the max `storageRadius` 0→3 in ~4–6 min (random uploads + distribute by proximity, so all nodes climb). +- **Local decrease is pull-sync-idle-gated**: once uploads stop and `pullsyncRate` settles, the + reserve worker ticks the radius down (3→2) after ~8–13 min; it can then oscillate 2↔3. There + is an effective floor (~2), so drive to `target-radius` ≥3 to leave headroom. +- **Dilution cannot reach expiry**: the postage contract reverts any dilution that would drop a + batch below `minimumValidityBlocks` (~a few hours here), so a batch floors at ~6h validity and + can't be diluted to 0. `force-decrease` therefore *hastens* the natural expiry (dilutes to the + floor) rather than triggering it directly. Locally this is redundant with the idle tick (which + fires first); it matters on testnet, where the decrease is expiry-driven. See + `docs/radius-check-plan.md`. ## Related -- `docs/radius-check-plan.md` — design, phases, spike findings, the driver/observer split. -- `.claude/skills/radius-testing/` — operating know-how, `radius-poll.sh`, the - Pushgateway/Prometheus/Grafana metrics stack (`metrics/`). +- `docs/radius-check-plan.md` — design, phases, spike findings. +- `.claude/skills/radius-testing/` — operating know-how, `radius-poll.sh`, the metrics stack. +- `.claude/skills/change-storage-price/` — activating a non-zero postage price on the local chain. - Prior art: PR #581 (`radiusdecrease`), PR #591 (`stampexpiry`), `pkg/check/gc`, - `pkg/check/load` (the committedDepth-gated upload primitive, reused as the soak driver), - `pkg/check/smoke`. + `pkg/check/load`, `pkg/check/smoke`. diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go index 5b8b9beb..3cf56edd 100644 --- a/pkg/check/reserveradius/metrics.go +++ b/pkg/check/reserveradius/metrics.go @@ -12,128 +12,27 @@ type metrics struct { PullsyncRate *prometheus.GaugeVec TimeToIncrease prometheus.Gauge TimeToDecrease prometheus.Gauge - TimeToFullySynced prometheus.Gauge - RadiusTransitions *prometheus.CounterVec - RecoveryObserved *prometheus.CounterVec - Disruptions *prometheus.CounterVec - Outcome *prometheus.GaugeVec - OnsetSeconds *prometheus.GaugeVec - RoundLoss *prometheus.CounterVec - // redistribution-game liveness (observe mode), from /redistributionstate - FullySynced *prometheus.GaugeVec - Frozen *prometheus.GaugeVec - RedistRound *prometheus.GaugeVec - LastSampleDuration *prometheus.GaugeVec + Dilutions prometheus.Counter } func newMetrics(subsystem string) metrics { return metrics{ - StorageRadius: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "storage_radius", - Help: "Storage radius reported by /status, per node.", - }, - []string{"node"}, - ), - ReserveSize: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "reserve_size", - Help: "Reserve size (chunks) reported by /status, per node.", - }, - []string{"node"}, - ), - PullsyncRate: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "pullsync_rate", - Help: "Pull-sync rate reported by /status, per node.", - }, - []string{"node"}, - ), - TimeToIncrease: prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "time_to_increase_seconds", - Help: "Seconds to reach the target storage radius from the first upload.", - }, - ), - TimeToDecrease: prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "time_to_decrease_seconds", - Help: "Seconds from uploads-stopped to the first observed radius decrease.", - }, - ), - RadiusTransitions: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "radius_transitions_total", - Help: "Observed storage-radius transitions, by node and direction (up/down).", - }, - []string{"node", "direction"}, - ), - RecoveryObserved: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "recovery_observed_total", - Help: "Pull-sync recovery outcome after a radius decrease, by node and result (recovered/timeout).", - }, - []string{"node", "result"}, - ), - Disruptions: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "disruption_total", - Help: "Neighbourhood disruptions applied, by mechanism (node-churn/batch-expiry).", - }, - []string{"mechanism"}, - ), - Outcome: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "outcome", - Help: "Halt-run outcome, one-hot per label (MONITORED/HALT/RECOVERED): the classified one is 1.", - }, - []string{"outcome"}, - ), - OnsetSeconds: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "onset_seconds", - Help: "Seconds from disruption to a survivor's de-sync onset, per node (halt mode).", - }, - []string{"node"}, - ), - RoundLoss: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: m.Namespace, - Subsystem: subsystem, - Name: "round_loss_total", - Help: "Survivors with staked round-loss (rounds advanced while lastPlayed/lastWon stalled), per node.", - }, - []string{"node"}, - ), + StorageRadius: nodeGauge(subsystem, "storage_radius", "Storage radius reported by /status, per node."), + ReserveSize: nodeGauge(subsystem, "reserve_size", "Reserve size (chunks) reported by /status, per node."), ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node (completeness signal)."), - TimeToFullySynced: prometheus.NewGauge(prometheus.GaugeOpts{ + PullsyncRate: nodeGauge(subsystem, "pullsync_rate", "Pull-sync rate reported by /status, per node."), + TimeToIncrease: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: m.Namespace, Subsystem: subsystem, - Name: "time_to_fully_synced_seconds", Help: "Seconds from a radius decrease to the node reporting isFullySynced again.", + Name: "time_to_increase_seconds", Help: "Seconds to reach the target storage radius from the first upload.", + }), + TimeToDecrease: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "time_to_decrease_seconds", Help: "Seconds from pull-sync going idle to the first observed radius decrease.", + }), + Dilutions: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "dilution_total", Help: "Postage-batch dilutions applied when forcing the decrease via expiry.", }), - FullySynced: nodeGauge(subsystem, "fully_synced", "Redistribution isFullySynced per node (1=yes, 0=no) — can the node play the game."), - Frozen: nodeGauge(subsystem, "frozen", "Redistribution isFrozen per node (1=frozen) — a halt symptom (skips rounds)."), - RedistRound: nodeGauge(subsystem, "redistribution_round", "Current redistribution round per node (stuck round = halt symptom)."), - LastSampleDuration: nodeGauge(subsystem, "last_sample_duration_seconds", "Last reserve-sample duration per node, from /redistributionstate."), } } diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 394ba370..4b2078fe 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -1,16 +1,13 @@ -// Package reserveradius stages a storage-radius change across a cluster and -// verifies the node recovers (pull-sync resumes) afterwards. +// Package reserveradius drives a storage-radius change on a Bee cluster and verifies the +// radius comes back down once pull-sync settles. It runs a single self-driving flow: // -// It follows the empirical sequence found by the Phase-1 spike (see -// docs/radius-check-plan.md "Spike findings"): -// -// 1. wait for warmup/stabilization — the reserve worker's decrease loop is -// gated on it, so a fresh node will not decrease for several minutes; -// 2. drive the radius UP by uploading until a node reaches the target; -// 3. let pushsync overshoot drain (uploads stopping != radius settling); -// 4. stop uploading and watch for a DECREASE and for pull-sync to recover -// (pullsyncRate > 0) — a stuck puller manage()/disconnectPeer() shows as -// no decrease / no recovery within the timeout (cf. beekeeper PR #581). +// 1. wait for warmup/stabilization — the reserve worker gates its decrease loop on it; +// 2. drive the radius UP by uploading random data to random full nodes until a target is reached; +// 3. stop uploading and wait for pull-sync to go idle (pullsyncRate==0) — the reserve worker +// only decreases the radius while SyncRate()==0 (== /status pullsyncRate), so the whole +// neighbourhood must finish syncing first; +// 4. observe the radius tick back down below its peak (the reserve worker's decrease), failing +// if no decrease appears within the timeout — a stuck puller shows as no decrease. // // This check requires a bee node patched for a small reserve (see // bee/.github/patches/radius_*.patch); on stock capacity the radius never moves. @@ -21,9 +18,7 @@ import ( crand "crypto/rand" "errors" "fmt" - "math/big" - "strconv" - "strings" + "math/rand" "time" "github.com/ethersphere/beekeeper/pkg/bee" @@ -37,65 +32,52 @@ import ( // Options represents reserve-radius check options. type Options struct { - Mode string // "drive" (upload to force a change) or "observe" (monitor a change driven externally) - Duration time.Duration // observe mode: total monitor run length - RndSeed int64 - StakeAmount string // per-node stake to ensure before driving (wei, e.g. "100000000000000000"); empty/"0" = skip - StakeGroups []string // node groups to stake (empty = the observed/selected groups) - StakeConfirmWait time.Duration // max wait for a deposit to reflect on-chain (~10 blocks) - PostageTTL time.Duration - PostageDepth uint64 - PostageLabel string - UploadGroups []string // node groups to upload to / observe (empty = all full nodes) - BlobSize int64 // bytes per upload - MaxUploads int // cap on uploads during the increase phase - TargetRadius uint8 // drive mode: storageRadius any node must reach before stopping uploads - DisruptAtRadius uint8 // halt mode: storageRadius ALL observed nodes must reach before disruption - DisruptMechanism string // "node-churn" (default) | "batch-expiry" | "both" | "none" - DisruptNodeCount int // node-churn: full nodes to remove (randomly, seeded by RndSeed); 0 = skip - DisruptMethod string // node-churn: "stop" (scale-0, default) | "delete" - MinSurvivors int // refuse to disrupt below this many surviving nodes - Verdict string // halt mode: "report" (default, never fail on outcome) | "assert" (gate on ExpectRecovery) - ExpectRecovery bool // halt mode + verdict=assert: false ⇒ expect HALT, true ⇒ expect RECOVERED - WarmupWait time.Duration // max wait for nodes to leave warmup before staging - IncreaseTimeout time.Duration // max time to reach TargetRadius - SettleWait time.Duration // wait after uploads for pushsync overshoot to drain - DecreaseTimeout time.Duration // max time to observe a decrease after uploads stop - RecoveryWait time.Duration // observe mode: max wait for pull-sync recovery after each decrease - PollInterval time.Duration + RndSeed int64 + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string // this check's own batch label + UploadGroups []string // node groups to upload to / observe (empty = all full nodes) + BlobSize int64 // bytes per upload + MaxUploads int // cap on uploads during the increase phase + TargetRadius uint8 // storageRadius any node must reach before stopping uploads + ForceDecrease bool // after pull-sync idles, dilute a batch to expiry to force the decrease (vs waiting for the idle tick) + DiluteStep uint64 // depth increase applied per dilution (each step ~halves remaining TTL) + MaxDilutions int // cap on dilutions per batch when forcing expiry + GasPrice string // gas price for the dilute tx (empty = node default) + WarmupWait time.Duration // max wait for nodes to leave warmup before driving + IncreaseTimeout time.Duration // max time to reach TargetRadius + SyncSettleWait time.Duration // after uploads stop, max wait for pullsyncRate==0 on all nodes (the decrease gate) + DecreaseTimeout time.Duration // max time to observe a radius decrease after pull-sync idles + PollInterval time.Duration } // NewDefaultOptions returns new default options. func NewDefaultOptions() Options { return Options{ - Mode: ModeDrive, - Duration: 12 * time.Hour, - RndSeed: time.Now().UnixNano(), - StakeAmount: "", // skip staking unless set - StakeGroups: nil, - StakeConfirmWait: 2 * time.Minute, - PostageTTL: 24 * time.Hour, - PostageDepth: 22, - PostageLabel: "reserve-radius", - UploadGroups: []string{"bee"}, - BlobSize: 1 << 20, // 1 MiB - MaxUploads: 60, - TargetRadius: 1, - DisruptAtRadius: 3, - DisruptMechanism: DisruptNodeChurn, - DisruptNodeCount: 2, - DisruptMethod: RemoveStop, - MinSurvivors: 3, - Verdict: VerdictReport, - ExpectRecovery: false, - WarmupWait: 15 * time.Minute, - IncreaseTimeout: 5 * time.Minute, - SettleWait: time.Minute, - DecreaseTimeout: 20 * time.Minute, - RecoveryWait: 5 * time.Minute, - PollInterval: 15 * time.Second, - } -} + RndSeed: time.Now().UnixNano(), + PostageTTL: 48 * time.Hour, + PostageDepth: 22, + PostageLabel: "reserve-radius", + UploadGroups: []string{"bee"}, + BlobSize: 8 << 20, // 8 MiB + MaxUploads: 200, + TargetRadius: 3, + ForceDecrease: true, + DiluteStep: 1, // depth += 1 per dilution ~halves remaining TTL; larger jumps revert once they cross the min-validity floor + MaxDilutions: 20, + GasPrice: "", + WarmupWait: 15 * time.Minute, + IncreaseTimeout: 30 * time.Minute, + SyncSettleWait: 10 * time.Minute, + DecreaseTimeout: 6 * time.Hour, // testnet: batch expiry lands ~minimumValidityBlocks (~5h) after diluting to the floor + PollInterval: 15 * time.Second, + } +} + +// pullSyncIdleRate is the pullsyncRate at or below which pull-sync is treated as idle. +// The rate is a decaying windowed average that settles to a small residual rather than +// exactly 0, so an == 0 gate would never open; active sync is orders of magnitude higher. +const pullSyncIdleRate = 0.05 // compile check whether Check implements interface var _ beekeeper.Action = (*Check)(nil) @@ -114,173 +96,31 @@ func NewCheck(log logging.Logger) beekeeper.Action { } } -// Mode values for Options.Mode. -const ( - ModeDrive = "drive" // upload to force a radius change, then observe the decrease - ModeObserve = "observe" // monitor radius changes driven externally (e.g. by the load check) - ModeHalt = "halt" // self-driving: stake → drive all nodes → disrupt → observe outcome -) - -// Disruption mechanisms for Options.DisruptMechanism. -const ( - DisruptNodeChurn = "node-churn" // stop/delete random full nodes (default) - DisruptBatchExpiry = "batch-expiry" // expire a fill batch → commitment drop → radius decrease - DisruptBoth = "both" // batch-expiry + node-churn for a harsher merge - DisruptNone = "none" // monitor-only (no disruption) -) - -// Node-removal methods for Options.DisruptMethod. -const ( - RemoveStop = "stop" // scale statefulset to 0 (restorable) - RemoveDelete = "delete" // delete statefulset + services/ingress/secret/configmap -) - -// Run outcomes classified by the halt observe-outcome stage. -const ( - OutcomeMonitored = "MONITORED" // no disruption requested; state recorded only - OutcomeHalt = "HALT" // post-disruption sustained non-convergence and/or staked round-loss - OutcomeRecovered = "RECOVERED" // post-disruption, all de-synced survivors re-converged -) - -// Verdict policies for Options.Verdict. -const ( - VerdictReport = "report" // never fail on the outcome; only operational errors fail (default) - VerdictAssert = "assert" // fail iff the outcome contradicts expect-recovery (A/B regression gate) -) +// batchRef is a postage batch created during the increase phase, paired with the node it lives on. +type batchRef struct { + node *bee.Client + batchID string +} -// Run dispatches on Mode: drive (force a change and observe it), observe -// (monitor changes driven externally, e.g. by a parallel load check), or halt -// (self-driving stake → drive → disrupt → observe-outcome reproduction). +// Run drives the radius up by uploading random data to random nodes, waits for pull-sync to +// go idle (the reserve worker's decrease gate), optionally forces the decrease by diluting a +// batch to expiry, then verifies the radius ticks back down. func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { o, ok := opts.(Options) if !ok { return errors.New("invalid options type") } - switch o.Mode { - case "", ModeDrive: - return c.runDrive(ctx, cluster, o) - case ModeObserve: - return c.runObserve(ctx, cluster, o) - case ModeHalt: - return c.runHalt(ctx, cluster, o) - default: - return fmt.Errorf("invalid mode %q (want %q, %q or %q)", o.Mode, ModeDrive, ModeObserve, ModeHalt) + if o.TargetRadius == 0 { + return errors.New("target-radius must be > 0") } -} -// selectNodes returns the observed full-node clients (shuffled, optionally filtered to UploadGroups). -func (c *Check) selectNodes(ctx context.Context, cluster orchestration.Cluster, o Options) (orchestration.ClientList, error) { c.logger.Infof("random seed: %d", o.RndSeed) rnd := random.PseudoGenerator(o.RndSeed) - fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) + nodes, err := c.selectNodes(ctx, cluster, rnd, o) if err != nil { - return nil, fmt.Errorf("get shuffled full node clients: %w", err) - } - nodes := fullNodeClients - if len(o.UploadGroups) > 0 { - nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) - } - if len(nodes) < 1 { - return nil, fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(nodes)) - } - return nodes, nil -} - -// ensureStaked makes sure every node in the staking set has at least StakeAmount -// staked, depositing the shortfall and confirming on-chain. It is idempotent: a -// node already at/above the target is skipped, and "already staked" is tolerated. -// Skips entirely when StakeAmount is empty or "0". The staking set is StakeGroups -// (full nodes filtered) or, when unset, the observed nodes. -func (c *Check) ensureStaked(ctx context.Context, cluster orchestration.Cluster, observed orchestration.ClientList, o Options) error { - amount := strings.TrimSpace(o.StakeAmount) - if amount == "" || amount == "0" { - return nil // staking disabled - } - want, ok := new(big.Int).SetString(amount, 10) - if !ok || want.Sign() <= 0 { - return fmt.Errorf("invalid stake-amount %q (want a positive base-10 wei integer)", o.StakeAmount) - } - - nodes := observed - if len(o.StakeGroups) > 0 { - rnd := random.PseudoGenerator(o.RndSeed) - full, err := cluster.ShuffledFullNodeClients(ctx, rnd) - if err != nil { - return fmt.Errorf("get full node clients for staking: %w", err) - } - nodes = full.FilterByNodeGroups(o.StakeGroups) - } - if len(nodes) == 0 { - return errors.New("ensureStaked: no nodes selected for staking") - } - - c.logger.Infof("ensureStaked: ensuring >= %s wei staked on %d node(s)", want, len(nodes)) - for _, n := range nodes { - if err := c.ensureNodeStaked(ctx, n, want, o); err != nil { - return fmt.Errorf("ensureStaked %s: %w", n.Name(), err) - } - } - return nil -} - -// ensureNodeStaked tops a single node up to want (if below) and confirms the deposit. -func (c *Check) ensureNodeStaked(ctx context.Context, n *bee.Client, want *big.Int, o Options) error { - cur, err := n.GetStake(ctx) - if err != nil { - return fmt.Errorf("get stake: %w", err) - } - if cur.Cmp(want) >= 0 { - c.logger.Infof("ensureStaked: %s already staked %s wei (>= %s) — skip", n.Name(), cur, want) - return nil - } - c.logger.Infof("ensureStaked: %s staked %s wei < target %s — depositing", n.Name(), cur, want) - if _, err := n.DepositStake(ctx, want); err != nil { - return fmt.Errorf("deposit stake: %w", err) - } - // POST /stake returns a tx hash; the staked amount may not reflect until the tx - // mines (~a few blocks), so poll until it confirms (or already mined → immediate). - return c.waitStakeAtLeast(ctx, n, want, o) -} - -// waitStakeAtLeast polls a node's stake until it reaches want or the budget expires. -func (c *Check) waitStakeAtLeast(ctx context.Context, n *bee.Client, want *big.Int, o Options) error { - deadline := time.Now().Add(o.StakeConfirmWait) - var last error - for { - cur, err := n.GetStake(ctx) - last = err - if err == nil && cur.Cmp(want) >= 0 { - c.logger.Infof("ensureStaked: %s confirmed staked %s wei", n.Name(), cur) - return nil - } - if time.Now().After(deadline) { - if last != nil { - return fmt.Errorf("stake not confirmed within %s: %w", o.StakeConfirmWait, last) - } - return fmt.Errorf("stake not confirmed within %s (still below %s wei)", o.StakeConfirmWait, want) - } - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return err - } - } -} - -// runDrive uploads to force a radius increase, then observes the decrease + recovery. -func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o Options) error { - if o.TargetRadius == 0 { - return errors.New("target-radius must be > 0") - } - nodes, err := c.selectNodes(ctx, cluster, o) - if err != nil { - return err - } - uploader := nodes[0] // a random node, since the list is shuffled - c.logger.Infof("mode=drive uploader: %s, observing %d node(s)", uploader.Name(), len(nodes)) - - // 0. Stake (optional) — runs before warmup so deposits confirm while we wait. - if err := c.ensureStaked(ctx, cluster, nodes, o); err != nil { return err } + c.logger.Infof("observing %d node(s); uploading random data to random nodes under label %q", len(nodes), o.PostageLabel) // 1. Wait for warmup/stabilization — the decrease loop is gated on it. if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { @@ -288,556 +128,46 @@ func (c *Check) runDrive(ctx context.Context, cluster orchestration.Cluster, o O } c.snapshot(ctx, nodes, "baseline") - // 2. Drive the radius up by uploading. - batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) - if err != nil { - return fmt.Errorf("create batch on %s: %w", uploader.Name(), err) - } - c.logger.WithField("batch_id", batchID).Infof("node %s: using batch", uploader.Name()) + // 2. Drive the radius up by uploading to random nodes; peak tracks the per-node high-water. peak := make(map[string]uint8, len(nodes)) - if err := c.driveIncrease(ctx, uploader, nodes, batchID, o, peak); err != nil { - return err - } - - // 3. Keep raising the per-node high-water `peak` through the settle window. - // The decrease can begin during settle (on an already-stabilised node it is - // near-immediate), so peak must be the max seen — a single post-settle read - // would come back at baseline and make a decrease impossible to detect. - c.logger.Infof("uploads stopped; tracking peak for %s (pushsync settle)", o.SettleWait) - settleDeadline := time.Now().Add(o.SettleWait) - for time.Now().Before(settleDeadline) { - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return err - } - c.updatePeak(ctx, nodes, peak) - } - c.logger.Infof("peak storageRadius per node: %v", peak) - - // 4. Observe the decrease + pull-sync recovery. - return c.observeDecrease(ctx, nodes, peak, o) -} - -// runHalt self-drives the whole reproduction: stake (optional) → warmup → drive -// ALL observed nodes to DisruptAtRadius → settle → disrupt → observe the outcome. -// The disruption (Phase 3) and outcome classification/verdict (Phase 4) stages -// land in later cycles; for now it completes the stake/drive/settle prefix. -func (c *Check) runHalt(ctx context.Context, cluster orchestration.Cluster, o Options) error { - if o.DisruptAtRadius == 0 { - return errors.New("disrupt-at-radius must be > 0") - } - nodes, err := c.selectNodes(ctx, cluster, o) + batches, err := c.driveIncrease(ctx, nodes, rnd, peak, o) if err != nil { return err } - uploader := nodes[0] // a random node, since the list is shuffled - c.logger.Infof("mode=halt uploader: %s, %d observed node(s), disrupt-at-radius=%d", uploader.Name(), len(nodes), o.DisruptAtRadius) - - // 0. Stake (optional) — runs before warmup so deposits confirm while we wait. - if err := c.ensureStaked(ctx, cluster, nodes, o); err != nil { - return err - } - - // 1. Wait for warmup/stabilization. - if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { - return err - } - c.baselineSnapshot(ctx, nodes) - - // 2. Drive every observed node up to DisruptAtRadius (a populated neighbourhood - // is the precondition for the disruption to bite). - batchID, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) - if err != nil { - return fmt.Errorf("create batch on %s: %w", uploader.Name(), err) - } - c.logger.WithField("batch_id", batchID).Infof("node %s: using batch", uploader.Name()) - peak := make(map[string]uint8, len(nodes)) - if err := c.driveAllToRadius(ctx, uploader, nodes, batchID, o, peak); err != nil { - return err - } - - // 3. Settle window — let pushsync overshoot drain before disrupting. - c.logger.Infof("halt: all nodes at radius %d; settling for %s before disruption", o.DisruptAtRadius, o.SettleWait) - settleDeadline := time.Now().Add(o.SettleWait) - for time.Now().Before(settleDeadline) { - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return err - } - c.updatePeak(ctx, nodes, peak) - } - - // 4. Disrupt the neighbourhood; the survivor set is what the observe loop polls. - survivors, err := c.disrupt(ctx, cluster, nodes, uploader.Name(), o) - if err != nil { - return err - } - disrupted := len(survivors) < len(nodes) // node-churn removed nodes; none/count-0 = monitor-only - disruptedAt := time.Now() - c.snapshot(ctx, survivors, "post-disrupt") - - // 5. Observe the outcome on the survivors and classify it. - outcome, err := c.observeOutcome(ctx, survivors, disrupted, disruptedAt, o) - if err != nil { - return err - } - c.logger.Infof("halt: run outcome = %s", outcome) - return c.applyVerdict(outcome, o) -} - -// applyVerdict turns a classified outcome into a check result per the verdict policy. -// report (default) always succeeds on any outcome (only operational errors, raised -// earlier, fail). assert fails iff the outcome contradicts expect-recovery; MONITORED -// always passes (no disruption was requested). -func (c *Check) applyVerdict(outcome string, o Options) error { - switch o.Verdict { - case "", VerdictReport: - c.logger.Infof("verdict=report: outcome %s (reported, not gated)", outcome) - return nil - case VerdictAssert: - if outcome == OutcomeMonitored { - c.logger.Info("verdict=assert: outcome MONITORED (no disruption requested) — pass") - return nil - } - want := OutcomeHalt - if o.ExpectRecovery { - want = OutcomeRecovered - } - if outcome == want { - c.logger.Infof("verdict=assert: outcome %s matches expect-recovery=%t — pass", outcome, o.ExpectRecovery) - return nil - } - return fmt.Errorf("verdict=assert: outcome %s contradicts expect-recovery=%t (expected %s)", outcome, o.ExpectRecovery, want) - default: - return fmt.Errorf("invalid verdict %q (want %q or %q)", o.Verdict, VerdictReport, VerdictAssert) - } -} - -// disrupt applies the configured disruption mechanism and returns the survivor set -// the observe loop should poll. excludeName is kept out of node-churn selection (the -// uploader in halt mode); pass "" when there is nothing to protect. -func (c *Check) disrupt(ctx context.Context, cluster orchestration.Cluster, observed orchestration.ClientList, excludeName string, o Options) (orchestration.ClientList, error) { - switch o.DisruptMechanism { - case DisruptNone: - c.logger.Info("disrupt: mechanism=none (monitor-only); no nodes removed") - return observed, nil - case "", DisruptNodeChurn: - return c.disruptNodeChurn(ctx, cluster, observed, excludeName, o) - case DisruptBatchExpiry, DisruptBoth: - return nil, fmt.Errorf("disrupt-mechanism %q not yet implemented (Phase 3b)", o.DisruptMechanism) - default: - return nil, fmt.Errorf("invalid disrupt-mechanism %q (want %q, %q, %q or %q)", o.DisruptMechanism, DisruptNodeChurn, DisruptBatchExpiry, DisruptBoth, DisruptNone) - } -} - -// disruptNodeChurn randomly selects DisruptNodeCount nodes (seeded by RndSeed, -// reproducible), excluding excludeName, and removes them via Stop (scale-0) or -// Delete. The MinSurvivors guard is checked BEFORE any removal, so a too-aggressive -// count fails without touching the cluster. DisruptNodeCount <= 0 is monitor-only. -// Returns the survivor ClientList (observed minus removed). -func (c *Check) disruptNodeChurn(ctx context.Context, cluster orchestration.Cluster, observed orchestration.ClientList, excludeName string, o Options) (orchestration.ClientList, error) { - if o.DisruptNodeCount <= 0 { - c.logger.Info("disrupt: node-churn disrupt-node-count=0 (monitor-only); no nodes removed") - return observed, nil - } - switch o.DisruptMethod { - case "", RemoveStop, RemoveDelete: - default: - return nil, fmt.Errorf("invalid disrupt-method %q (want %q or %q)", o.DisruptMethod, RemoveStop, RemoveDelete) - } - - // Candidate pool excludes the protected node (the uploader in halt mode). - candidates := make(orchestration.ClientList, 0, len(observed)) - for _, n := range observed { - if n.Name() != excludeName { - candidates = append(candidates, n) - } - } - if o.DisruptNodeCount > len(candidates) { - return nil, fmt.Errorf("disrupt: cannot remove %d node(s), only %d candidate(s) available (excluding %q)", o.DisruptNodeCount, len(candidates), excludeName) - } - if survivors := len(observed) - o.DisruptNodeCount; survivors < o.MinSurvivors { - return nil, fmt.Errorf("disrupt: removing %d of %d node(s) would leave %d survivor(s), below min-survivors=%d", o.DisruptNodeCount, len(observed), survivors, o.MinSurvivors) - } - - // Reproducible random pick of DisruptNodeCount candidates. - rnd := random.PseudoGenerator(o.RndSeed) - pick := rnd.Perm(len(candidates))[:o.DisruptNodeCount] - removed := make(map[string]bool, len(pick)) - nodesByName := cluster.Nodes() - ns := cluster.Namespace() - for _, ci := range pick { - name := candidates[ci].Name() - node, ok := nodesByName[name] - if !ok { - return nil, fmt.Errorf("disrupt: node %q not found in cluster", name) - } - method := o.DisruptMethod - if method == "" { - method = RemoveStop - } - var rerr error - if method == RemoveDelete { - rerr = node.Delete(ctx, ns) - } else { - rerr = node.Stop(ctx, ns) - } - if rerr != nil { - return nil, fmt.Errorf("disrupt: %s node %q: %w", method, name, rerr) - } - removed[name] = true - c.metrics.Disruptions.WithLabelValues(DisruptNodeChurn).Inc() - c.logger.Warningf("disrupt: %s node %q (%d/%d) at %s", method, name, len(removed), o.DisruptNodeCount, time.Now().Format(time.RFC3339)) - } - - survivors := make(orchestration.ClientList, 0, len(observed)-len(removed)) - for _, n := range observed { - if !removed[n.Name()] { - survivors = append(survivors, n) - } - } - c.logger.Infof("disrupt: node-churn removed %d node(s) via %s; %d survivor(s) remain", len(removed), o.DisruptMethod, len(survivors)) - return survivors, nil -} - -// outcomeNode tracks a survivor's post-disruption trajectory for classification. -type outcomeNode struct { - haveRef bool - refSynced bool // was the node fullySynced at the first post-disruption reading - refRound uint64 // round participation reference (for staked round-loss) - refPlayed uint64 - refWon uint64 - - onset bool // lost sync (fullySynced true->false) after disruption - onsetAt time.Time // when the onset was first seen - recovered bool // returned to fullySynced WITHIN RecoveryWait after an onset - lateResync bool // re-synced, but only after RecoveryWait (too slow — a halt symptom) - recoverBy time.Time // recovery deadline after onset (onsetAt + RecoveryWait) - - haveLast bool - lastRound uint64 - lastPlayed uint64 - lastWon uint64 -} - -// observeOutcome polls the survivor set for Duration after disruption, tracking per -// node the onset of de-sync (fullySynced true->false), recovery (back to fullySynced -// within RecoveryWait), and staked round-loss (Round advancing while -// LastPlayedRound/LastWonRound stall). It classifies the run as MONITORED (no -// disruption), HALT (a survivor stuck de-synced past RecoveryWait and/or round-loss), -// or RECOVERED (all de-synced survivors re-converged), emits the outcome metric + a -// summary, and returns the outcome. The verdict policy is applied by the caller. -func (c *Check) observeOutcome(ctx context.Context, survivors orchestration.ClientList, disrupted bool, disruptedAt time.Time, o Options) (string, error) { - st := make(map[string]*outcomeNode, len(survivors)) - for _, n := range survivors { - st[n.Name()] = &outcomeNode{} - } - c.logger.Infof("observe-outcome: watching %d survivor(s) for %s (disrupted=%t, recovery-wait=%s)", len(survivors), o.Duration, disrupted, o.RecoveryWait) - - deadline := time.Now().Add(o.Duration) - for time.Now().Before(deadline) { - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return "", err // parent context cancelled (e.g. check timeout) - } - for _, n := range survivors { - name := n.Name() - ns := st[name] - - s, err := n.Status(ctx) - if err != nil { - continue - } - c.emit(name, s) - - r, rerr := n.RedistributionState(ctx) - if rerr != nil { - continue // the redistribution signal is the classifier; skip this poll for the node - } - c.emitRedist(name, r) - - if !ns.haveRef { - ns.haveRef = true - ns.refSynced = r.IsFullySynced - ns.refRound, ns.refPlayed, ns.refWon = r.Round, r.LastPlayedRound, r.LastWonRound - } - ns.haveLast = true - ns.lastRound, ns.lastPlayed, ns.lastWon = r.Round, r.LastPlayedRound, r.LastWonRound - - // onset: a node that was synced loses sync after disruption - if disrupted && ns.refSynced && !ns.onset && !r.IsFullySynced { - ns.onset = true - ns.onsetAt = time.Now() - ns.recoverBy = ns.onsetAt.Add(o.RecoveryWait) - c.metrics.OnsetSeconds.WithLabelValues(name).Set(ns.onsetAt.Sub(disruptedAt).Seconds()) - c.logger.Warningf("observe-outcome: %s de-synced (fullySynced=false) at round %d, %s after disruption — onset", name, r.Round, ns.onsetAt.Sub(disruptedAt).Round(time.Second)) - } - // recovery: back to fullySynced (and not frozen) WITHIN RecoveryWait. A - // re-sync after the deadline is too slow to count — it stays not-recovered - // so it classifies as stuck (the "puller gave up at SyncRate=0" false-sync - // is exactly this: it re-flags fullySynced late without truly converging). - if ns.onset && !ns.recovered && !ns.lateResync && r.IsFullySynced && !r.IsFrozen { - if time.Now().Before(ns.recoverBy) { - ns.recovered = true - c.logger.Infof("observe-outcome: %s re-converged %s after onset (within recovery-wait)", name, time.Since(ns.onsetAt).Round(time.Second)) - } else { - ns.lateResync = true - c.logger.Warningf("observe-outcome: %s re-synced %s after onset — PAST recovery-wait %s, too slow (counts as halt)", name, time.Since(ns.onsetAt).Round(time.Second), o.RecoveryWait) - } - } - } - } - - return c.classifyOutcome(st, disrupted, o), nil -} - -// classifyOutcome reduces the per-node trajectories to a single run outcome and emits it. -func (c *Check) classifyOutcome(st map[string]*outcomeNode, disrupted bool, o Options) string { - if !disrupted { - c.setOutcome(OutcomeMonitored) - c.logger.Infof("observe-outcome: MONITORED — no disruption requested; %d node(s) observed for %s", len(st), o.Duration) - return OutcomeMonitored - } - - desynced, recovered, stuck, roundLoss := 0, 0, 0, 0 - for name, ns := range st { - if ns.onset { - desynced++ - } - if ns.onset && ns.recovered { - recovered++ - } - // stuck: de-synced, never recovered, past its recovery deadline - if ns.onset && !ns.recovered && !ns.recoverBy.IsZero() && time.Now().After(ns.recoverBy) { - stuck++ - } - // staked round-loss: rounds advanced after onset but the node never played/won - if ns.onset && ns.haveLast && ns.lastRound > ns.refRound && ns.lastPlayed == ns.refPlayed && ns.lastWon == ns.refWon { - roundLoss++ - c.metrics.RoundLoss.WithLabelValues(name).Inc() - c.logger.Warningf("observe-outcome: %s round-loss — round %d->%d while lastPlayed/Won stalled (%d/%d) since de-sync", name, ns.refRound, ns.lastRound, ns.refPlayed, ns.refWon) - } - } - - if stuck > 0 || roundLoss > 0 { - c.setOutcome(OutcomeHalt) - c.logger.Warningf("observe-outcome: HALT — %d/%d survivor(s) stuck de-synced past %s, %d with staked round-loss (%d de-synced, %d recovered)", stuck, len(st), o.RecoveryWait, roundLoss, desynced, recovered) - return OutcomeHalt - } - c.setOutcome(OutcomeRecovered) - c.logger.Infof("observe-outcome: RECOVERED — all %d survivor(s) converged (%d de-synced then recovered)", len(st), recovered) - return OutcomeRecovered -} - -// setOutcome one-hot-encodes the classified outcome into the outcome gauge. -func (c *Check) setOutcome(outcome string) { - for _, name := range []string{OutcomeMonitored, OutcomeHalt, OutcomeRecovered} { - v := 0.0 - if name == outcome { - v = 1 - } - c.metrics.Outcome.WithLabelValues(name).Set(v) - } -} - -// obsNode is the per-node state the observe monitor tracks across polls. -type obsNode struct { - lastRadius uint8 - haveRadius bool - recoverBy time.Time // non-zero => awaiting recovery after a decrease - downAt time.Time // when the pending decrease started (for time-to-recovery) - frozen bool // currently inside a frozen episode -} - -// runObserve monitors a cluster for Duration without uploading; the radius is driven -// externally (e.g. by a parallel load check). It records every up/down radius -// transition and, after each decrease, asserts the node recovers — preferring the -// redistribution-game signal (isFullySynced, not frozen) over the weaker pullsyncRate -// when /redistributionstate is available. It also flags freeze episodes directly -// (a frozen node skips rounds — the October halt symptom). Halt indicators -// (un-recovered decreases or freezes) fail the check at the end. -// disruptionActive reports whether the configured mechanism will actually disrupt, so -// observe mode runs the staged reproduction rather than the plain soak monitor. -// Monitor-only = disrupt-mechanism none, or node-churn with disrupt-node-count 0. -func disruptionActive(o Options) bool { - switch o.DisruptMechanism { - case DisruptNone: - return false - case "", DisruptNodeChurn: - return o.DisruptNodeCount > 0 - default: // batch-expiry / both - return true - } -} - -// runObserveDisrupt is the parallel-with-load shape: the radius is driven externally -// (e.g. by a parallel load check), so this check waits for the neighbourhood to -// populate, snapshots a baseline, disrupts, then observes + classifies the outcome — -// the same stages as halt mode, minus the self-driving upload. -func (c *Check) runObserveDisrupt(ctx context.Context, cluster orchestration.Cluster, nodes orchestration.ClientList, o Options) error { - if o.DisruptAtRadius == 0 { - return errors.New("disrupt-at-radius must be > 0") - } - c.logger.Infof("mode=observe+disrupt: staged reproduction (mechanism=%s, count=%d); radius driven externally", o.DisruptMechanism, o.DisruptNodeCount) - - if err := c.waitAllReachRadius(ctx, nodes, o); err != nil { - return err - } - c.baselineSnapshot(ctx, nodes) - survivors, err := c.disrupt(ctx, cluster, nodes, "", o) // no uploader to protect in observe mode - if err != nil { - return err - } - disrupted := len(survivors) < len(nodes) - disruptedAt := time.Now() - c.snapshot(ctx, survivors, "post-disrupt") + // 3. Stop uploading and wait for pull-sync to go idle. The reserve worker only decreases + // the radius when SyncRate()==0 (== /status pullsyncRate), so the neighbourhood must + // finish syncing first. Keep raising peak in case a decrease begins during the wait. + c.waitPullSyncIdle(ctx, nodes, peak, o) - outcome, err := c.observeOutcome(ctx, survivors, disrupted, disruptedAt, o) - if err != nil { - return err + // 4. Optionally hasten the decrease: dilute one batch down to the contract's minimum + // validity so it expires within the check's budget (~minimumValidityBlocks, ~hours) + // instead of its full TTL. On expiry its chunks are evicted and reserve commitment + // drops, which — with idle pull-sync — lets the reserve worker decrease the radius. + // Diluting a single batch is a partial expiry: it drops commitment without evicting + // the whole reserve (which would halt the neighbourhood). + if o.ForceDecrease && len(batches) > 0 { + c.diluteToMinValidity(ctx, batches[0], o) } - c.logger.Infof("observe+disrupt: run outcome = %s", outcome) - return c.applyVerdict(outcome, o) -} -// waitAllReachRadius blocks until every node reports storageRadius >= DisruptAtRadius -// (driven externally, e.g. by a parallel load check) or IncreaseTimeout elapses. -func (c *Check) waitAllReachRadius(ctx context.Context, nodes orchestration.ClientList, o Options) error { - c.logger.Infof("observe+disrupt: waiting up to %s for all %d node(s) to reach storageRadius>=%d (driven externally)", o.IncreaseTimeout, len(nodes), o.DisruptAtRadius) - deadline := time.Now().Add(o.IncreaseTimeout) - peak := make(map[string]uint8, len(nodes)) - for { - if err := ctx.Err(); err != nil { - return err - } - mn, mx := c.updatePeak(ctx, nodes, peak) - if mn >= o.DisruptAtRadius { - c.logger.Infof("observe+disrupt: all nodes reached storageRadius %d (max %d)", o.DisruptAtRadius, mx) - return nil - } - if time.Now().After(deadline) { - return fmt.Errorf("not all nodes reached storageRadius %d within %s (min=%d) — is the radius being driven (e.g. the load check) and is the reserve patch active?", o.DisruptAtRadius, o.IncreaseTimeout, mn) - } - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return err - } - } + // 5. Observe the radius tick back down below its peak (via the idle tick, or batch expiry). + return c.observeDecrease(ctx, nodes, peak, o) } -func (c *Check) runObserve(ctx context.Context, cluster orchestration.Cluster, o Options) error { - nodes, err := c.selectNodes(ctx, cluster, o) +// selectNodes returns the observed full-node clients (shuffled, optionally filtered to UploadGroups). +func (c *Check) selectNodes(ctx context.Context, cluster orchestration.Cluster, rnd *rand.Rand, o Options) (orchestration.ClientList, error) { + fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) if err != nil { - return err - } - c.logger.Infof("mode=observe monitoring %d node(s) for %s (no uploads; drive the radius externally, e.g. the load check)", len(nodes), o.Duration) - - if err := c.ensureStaked(ctx, cluster, nodes, o); err != nil { - return err - } - - if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { - return err - } - - // observe + disruption configured → run the staged reproduction (parallel-with-load - // shape) instead of the plain soak monitor. Monitor-only requires disrupt-mechanism: - // none (or node-churn with disrupt-node-count: 0). - if disruptionActive(o) { - return c.runObserveDisrupt(ctx, cluster, nodes, o) - } - - st := make(map[string]*obsNode, len(nodes)) - for _, n := range nodes { - st[n.Name()] = &obsNode{} + return nil, fmt.Errorf("get shuffled full node clients: %w", err) } - redistAvailable := true // flips false on first error; selects which recovery signal to use - unrecovered, freezes := 0, 0 - c.logger.Info("observe: watching radius transitions, freezes, and redistribution liveness") - - deadline := time.Now().Add(o.Duration) - for time.Now().Before(deadline) { - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return err // parent context cancelled (e.g. check timeout) - } - for _, n := range nodes { - name := n.Name() - ns := st[name] - - s, err := n.Status(ctx) - if err != nil { - continue - } - c.emit(name, s) - - // redistribution-game state (full-mode only; best-effort) - var rs *api.RedistributionState - if redistAvailable { - if r, rerr := n.RedistributionState(ctx); rerr == nil { - rs = r - c.emitRedist(name, r) - } else { - redistAvailable = false - c.logger.Warningf("observe: /redistributionstate unavailable (%v); using pullsyncRate for recovery, no freeze detection", rerr) - } - } - - // radius transition - cur := s.StorageRadius - if ns.haveRadius && cur != ns.lastRadius { - dir := "up" - if cur < ns.lastRadius { - dir = "down" - } - c.metrics.RadiusTransitions.WithLabelValues(name, dir).Inc() - c.logger.Infof("observe: %s storageRadius %d -> %d (%s) within=%d pullsyncRate=%.4f", name, ns.lastRadius, cur, dir, s.ReserveSizeWithinRadius, s.PullsyncRate) - if dir == "down" { - ns.recoverBy = time.Now().Add(o.RecoveryWait) - ns.downAt = time.Now() - } - } - ns.lastRadius = cur - ns.haveRadius = true - - // freeze detection — a frozen node skips redistribution rounds (halt symptom) - if rs != nil { - switch { - case rs.IsFrozen && !ns.frozen: - ns.frozen = true - freezes++ - c.logger.Warningf("observe: %s is FROZEN at round %d (halt symptom; lastFrozenRound=%d)", name, rs.Round, rs.LastFrozenRound) - case !rs.IsFrozen: - ns.frozen = false - } - } - - // recovery tracking after a decrease - if !ns.recoverBy.IsZero() { - recovered, reason := false, "" - switch { - case rs != nil && rs.IsFullySynced && !rs.IsFrozen: - recovered, reason = true, "fullySynced" - case rs == nil && s.PullsyncRate > 0: - recovered, reason = true, "pullsyncRate(fallback)" - } - switch { - case recovered: - c.metrics.RecoveryObserved.WithLabelValues(name, "recovered").Inc() - c.metrics.TimeToFullySynced.Set(time.Since(ns.downAt).Seconds()) - c.logger.Infof("observe: %s recovered after decrease in %s (%s)", name, time.Since(ns.downAt).Round(time.Second), reason) - ns.recoverBy = time.Time{} - case time.Now().After(ns.recoverBy): - unrecovered++ - c.metrics.RecoveryObserved.WithLabelValues(name, "timeout").Inc() - c.logger.Warningf("observe: %s did NOT recover within %s after decrease (not fullySynced / no pull-sync) — pull-sync halt symptom (cf. PR #581)", name, o.RecoveryWait) - ns.recoverBy = time.Time{} - } - } - } + nodes := fullNodeClients + if len(o.UploadGroups) > 0 { + nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) } - - if unrecovered > 0 || freezes > 0 { - return fmt.Errorf("observe: halt indicators over %s — %d decrease(s) without recovery, %d freeze episode(s)", o.Duration, unrecovered, freezes) + if len(nodes) < 1 { + return nil, fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(nodes)) } - c.logger.Infof("observe: completed %s monitor — no halt indicators (all decreases recovered, no freezes)", o.Duration) - return nil + return nodes, nil } // waitForWarmupDone blocks until every observed node reports isWarmingUp=false. @@ -868,118 +198,167 @@ func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.Clien } } -// driveIncrease uploads blobs to the uploader until any observed node reaches TargetRadius. -func (c *Check) driveIncrease(ctx context.Context, uploader *bee.Client, nodes orchestration.ClientList, batchID string, o Options, peak map[string]uint8) error { - c.logger.Infof("driving increase: %d-byte blobs to %s until storageRadius>=%d (max %d uploads, timeout %s)", - o.BlobSize, uploader.Name(), o.TargetRadius, o.MaxUploads, o.IncreaseTimeout) +// driveIncrease uploads BlobSize random blobs to randomly-chosen nodes until any observed +// node reaches TargetRadius. Each node uploads under its own mutable batch (this check's +// label). It raises the per-node high-water peak as it goes and returns the batches it +// created (in creation order) for the optional force-decrease step. +func (c *Check) driveIncrease(ctx context.Context, nodes orchestration.ClientList, rnd *rand.Rand, peak map[string]uint8, o Options) ([]batchRef, error) { + c.logger.Infof("driving increase: %d-byte blobs to random nodes until storageRadius>=%d (max %d uploads, timeout %s)", + o.BlobSize, o.TargetRadius, o.MaxUploads, o.IncreaseTimeout) start := time.Now() deadline := start.Add(o.IncreaseTimeout) t := test.NewTest(c.logger) data := make([]byte, o.BlobSize) + batchByNode := make(map[string]string, len(nodes)) + var batches []batchRef for i := 1; i <= o.MaxUploads; i++ { if err := ctx.Err(); err != nil { - return err + return nil, err } if time.Now().After(deadline) { - return fmt.Errorf("storageRadius did not reach %d within %s (%d uploads) — is the bee reserve patch active and is there enough data?", o.TargetRadius, o.IncreaseTimeout, i-1) + return nil, fmt.Errorf("storageRadius did not reach %d within %s (%d uploads) — is the bee reserve patch active and is there enough data?", o.TargetRadius, o.IncreaseTimeout, i-1) } + + uploader := nodes[rnd.Intn(len(nodes))] + batchID, ok := batchByNode[uploader.Name()] + if !ok { + id, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + // Transient per-node error (e.g. a 503 from chainstate); another random + // node is picked next iteration, so skip rather than fail the whole run. + c.logger.Warningf("create batch on %s failed, skipping upload #%d: %v", uploader.Name(), i, err) + continue + } + batchID = id + batchByNode[uploader.Name()] = id + batches = append(batches, batchRef{node: uploader, batchID: id}) + c.logger.WithField("batch_id", id).Infof("node %s: using batch", uploader.Name()) + } + if _, err := crand.Read(data); err != nil { - return fmt.Errorf("generate random data: %w", err) + return nil, fmt.Errorf("generate random data: %w", err) } if _, _, err := t.Upload(ctx, uploader, data, batchID, nil); err != nil { - c.logger.Errorf("upload #%d failed: %v", i, err) + c.logger.Errorf("upload #%d to %s failed: %v", i, uploader.Name(), err) continue } + _, mx := c.updatePeak(ctx, nodes, peak) - c.logger.Infof("increase: upload #%d, max storageRadius=%d (target %d)", i, mx, o.TargetRadius) + c.logger.Infof("increase: upload #%d to %s, max storageRadius=%d (target %d)", i, uploader.Name(), mx, o.TargetRadius) if mx >= o.TargetRadius { c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) - c.logger.Infof("reached storageRadius %d after %d uploads (~%.1f MiB) in %s", - mx, i, float64(int64(i)*o.BlobSize)/(1<<20), time.Since(start).Round(time.Second)) - return nil + c.logger.Infof("reached storageRadius %d after %d uploads (~%.1f MiB) across %d batch(es) in %s", + mx, i, float64(int64(i)*o.BlobSize)/(1<<20), len(batches), time.Since(start).Round(time.Second)) + return batches, nil } } - return fmt.Errorf("storageRadius did not reach %d after %d uploads", o.TargetRadius, o.MaxUploads) + return nil, fmt.Errorf("storageRadius did not reach %d after %d uploads", o.TargetRadius, o.MaxUploads) } -// driveAllToRadius uploads blobs to the uploader until EVERY observed node reaches -// DisruptAtRadius (gates on the min, not the max — the whole neighbourhood must be -// populated before disruption). Otherwise it mirrors driveIncrease. -func (c *Check) driveAllToRadius(ctx context.Context, uploader *bee.Client, nodes orchestration.ClientList, batchID string, o Options, peak map[string]uint8) error { - c.logger.Infof("halt drive: %d-byte blobs to %s until ALL %d node(s) storageRadius>=%d (max %d uploads, timeout %s)", - o.BlobSize, uploader.Name(), len(nodes), o.DisruptAtRadius, o.MaxUploads, o.IncreaseTimeout) - start := time.Now() - deadline := start.Add(o.IncreaseTimeout) - t := test.NewTest(c.logger) - data := make([]byte, o.BlobSize) - - for i := 1; i <= o.MaxUploads; i++ { - if err := ctx.Err(); err != nil { - return err +// diluteToMinValidity dilutes a batch as far down as the contract allows — each +DiluteStep +// (default 1) roughly halves its remaining TTL — stopping when a dilution is rejected because +// it would drop the batch's validity below the chain's minimumValidityBlocks floor. That floor +// makes expiry itself unreachable by dilution, so this only *hastens* the natural expiry: it +// leaves the batch at its minimum TTL (~minimumValidityBlocks) so it expires within the check's +// budget rather than its full original TTL. On expiry the batch's chunks are evicted and reserve +// commitment drops. Best-effort — a read/dilute error just stops early (the batch keeps whatever +// TTL it reached), so this never fails the check on its own. +func (c *Check) diluteToMinValidity(ctx context.Context, b batchRef, o Options) { + c.logger.Infof("diluting batch %s on %s toward minimum validity to hasten expiry", b.batchID, b.node.Name()) + for k := 0; k < o.MaxDilutions; k++ { + if ctx.Err() != nil { + return } - if time.Now().After(deadline) { - return fmt.Errorf("not all nodes reached storageRadius %d within %s (%d uploads) — is the bee reserve patch active and is there enough data?", o.DisruptAtRadius, o.IncreaseTimeout, i-1) + stamp, err := b.node.PostageStamp(ctx, b.batchID) + if err != nil { + c.logger.Warningf("read batch %s on %s: %v; stopping dilution", b.batchID, b.node.Name(), err) + return + } + if !stamp.Exists || stamp.BatchTTL <= 0 { + c.logger.Infof("batch %s on %s already expired (ttl=%ds)", b.batchID, b.node.Name(), stamp.BatchTTL) + return + } + newDepth := uint64(stamp.Depth) + o.DiluteStep + if err := b.node.DilutePostageBatch(ctx, b.batchID, newDepth, o.GasPrice); err != nil { + // Rejected: the resulting validity would fall below minimumValidityBlocks — the + // batch is at its minimum TTL, the expected stop condition. + c.logger.Infof("batch %s on %s at minimum validity: ttl ~%ds (~%.1fh); dilute to depth %d rejected (%v)", + b.batchID, b.node.Name(), stamp.BatchTTL, float64(stamp.BatchTTL)/3600, newDepth, err) + return + } + c.metrics.Dilutions.Inc() + c.logger.Infof("diluted batch %s on %s to depth %d (ttl was %ds ~%.1fh)", b.batchID, b.node.Name(), newDepth, stamp.BatchTTL, float64(stamp.BatchTTL)/3600) + } + c.logger.Warningf("batch %s on %s: reached max %d dilutions without hitting the validity floor", b.batchID, b.node.Name(), o.MaxDilutions) +} + +// waitPullSyncIdle blocks until every node reports pullsyncRate==0 (pull-sync idle) or +// SyncSettleWait elapses. Idle is the reserve worker's decrease precondition, so this is a +// best-effort gate: on timeout it logs and proceeds (observeDecrease will fail if the radius +// never drops). It keeps raising peak, since a decrease can begin as soon as sync idles. +func (c *Check) waitPullSyncIdle(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8, o Options) { + c.logger.Infof("uploads stopped; waiting up to %s for pull-sync to go idle (pullsyncRate<=%.2f on all %d node(s))", o.SyncSettleWait, pullSyncIdleRate, len(nodes)) + deadline := time.Now().Add(o.SyncSettleWait) + for { + idle, maxRate := true, 0.0 + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + idle = false + continue + } + c.emit(n.Name(), s) + if s.StorageRadius > peak[n.Name()] { + peak[n.Name()] = s.StorageRadius + } + if s.PullsyncRate > pullSyncIdleRate { + idle = false + } + if s.PullsyncRate > maxRate { + maxRate = s.PullsyncRate + } } - if _, err := crand.Read(data); err != nil { - return fmt.Errorf("generate random data: %w", err) + if idle { + c.logger.Infof("pull-sync idle on all nodes (pullsyncRate<=%.2f) — decrease gate open", pullSyncIdleRate) + return } - if _, _, err := t.Upload(ctx, uploader, data, batchID, nil); err != nil { - c.logger.Errorf("upload #%d failed: %v", i, err) - continue + if time.Now().After(deadline) { + c.logger.Warningf("pull-sync did not go idle within %s (max pullsyncRate=%.4f); proceeding to observe anyway", o.SyncSettleWait, maxRate) + return } - mn, mx := c.updatePeak(ctx, nodes, peak) - c.logger.Infof("halt drive: upload #%d, min storageRadius=%d max=%d (target %d)", i, mn, mx, o.DisruptAtRadius) - if mn >= o.DisruptAtRadius { - c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) - c.logger.Infof("all nodes reached storageRadius %d after %d uploads (~%.1f MiB) in %s", - o.DisruptAtRadius, i, float64(int64(i)*o.BlobSize)/(1<<20), time.Since(start).Round(time.Second)) - return nil + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return // context cancelled (e.g. check timeout) — stop waiting } } - return fmt.Errorf("not all nodes reached storageRadius %d after %d uploads", o.DisruptAtRadius, o.MaxUploads) } -// observeDecrease watches for any node's radius to fall below its peak, and for -// pull-sync to recover (pullsyncRate>0). No decrease within the timeout is the -// failure signal (cf. PR #581 puller manage()/disconnectPeer() stall). +// observeDecrease watches for any node's storageRadius to fall below its peak — the reserve +// worker's decrease once pull-sync is idle. No decrease within DecreaseTimeout is the failure +// signal (a stuck puller never lets SyncRate reach 0, so the radius never decreases). func (c *Check) observeDecrease(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8, o Options) error { - c.logger.Infof("watching for radius decrease + pull-sync recovery (timeout %s)", o.DecreaseTimeout) + c.logger.Infof("watching for a radius decrease below the peak %v (timeout %s)", peak, o.DecreaseTimeout) start := time.Now() deadline := start.Add(o.DecreaseTimeout) - recovered := false - for { - if err := ctx.Err(); err != nil { - return err - } - decreasedOn := "" for _, n := range nodes { s, err := n.Status(ctx) if err != nil { continue } c.emit(n.Name(), s) - if s.PullsyncRate > 0 { - recovered = true + if s.StorageRadius > peak[n.Name()] { + peak[n.Name()] = s.StorageRadius } - if p, ok := peak[n.Name()]; ok && s.StorageRadius < p { - decreasedOn = n.Name() - } - } - if decreasedOn != "" { - c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) - c.logger.Infof("radius decrease observed on %s after %s (pull-sync recovery seen: %t)", - decreasedOn, time.Since(start).Round(time.Second), recovered) - // TODO(reserve-radius): make pull-sync recovery a hard gate once the - // expected rate floor is characterised on a data-heavy cluster (Phase 3). - if !recovered { - c.logger.Warning("decrease observed but pullsyncRate stayed 0 — puller may not have resumed; investigate (cf. PR #581)") + if s.StorageRadius < peak[n.Name()] { + c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) + c.logger.Infof("radius decrease observed on %s (%d -> %d) after %s — done", + n.Name(), peak[n.Name()], s.StorageRadius, time.Since(start).Round(time.Second)) + return nil } - return nil } if time.Now().After(deadline) { - return fmt.Errorf("no radius decrease within %s after uploads stopped — puller manage()/disconnectPeer() may be stalled (cf. PR #581), or the decrease gate (synced + count0, cf. PR #581)", peak, o.DecreaseTimeout) } if err := sleepCtx(ctx, o.PollInterval); err != nil { return err @@ -987,69 +366,8 @@ func (c *Check) observeDecrease(ctx context.Context, nodes orchestration.ClientL } } -// snapshot polls every node, emits metrics, logs a line, and returns the max storageRadius. -func (c *Check) snapshot(ctx context.Context, nodes orchestration.ClientList, phase string) uint8 { - var mx uint8 - for _, n := range nodes { - s, err := n.Status(ctx) - if err != nil { - c.logger.Debugf("%s: status error: %v", n.Name(), err) - continue - } - c.emit(n.Name(), s) - if s.StorageRadius > mx { - mx = s.StorageRadius - } - c.logger.Infof("[%s] %s: storageRadius=%d reserveSize=%d withinR=%d pullsyncRate=%.4f warmingUp=%t", - phase, n.Name(), s.StorageRadius, s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate, s.IsWarmingUp) - } - return mx -} - -// baselineSnapshot logs and emits the pre-disruption reference state for every node: -// storage radius + reserveSizeWithinRadius (/status), isFullySynced + round -// participation (/redistributionstate), and stake. Best-effort per node — a failed -// read shows as "?" in the log. Phase 4 will extend this to return the captured -// values for onset and staked-round-loss comparison. -func (c *Check) baselineSnapshot(ctx context.Context, nodes orchestration.ClientList) { - for _, n := range nodes { - name := n.Name() - - radius, within := "?", "?" - if s, err := n.Status(ctx); err == nil { - c.emit(name, s) - radius = strconv.Itoa(int(s.StorageRadius)) - within = strconv.FormatUint(s.ReserveSizeWithinRadius, 10) - } else { - c.logger.Debugf("baseline %s: status error: %v", name, err) - } - - synced, round, played, won := "?", "?", "?", "?" - if r, err := n.RedistributionState(ctx); err == nil { - c.emitRedist(name, r) - synced = strconv.FormatBool(r.IsFullySynced) - round = strconv.FormatUint(r.Round, 10) - played = strconv.FormatUint(r.LastPlayedRound, 10) - won = strconv.FormatUint(r.LastWonRound, 10) - } else { - c.logger.Debugf("baseline %s: redistributionstate error: %v", name, err) - } - - stake := "?" - if st, err := n.GetStake(ctx); err == nil { - stake = st.String() - } else { - c.logger.Debugf("baseline %s: stake error: %v", name, err) - } - - c.logger.Infof("[baseline] %s: storageRadius=%s withinR=%s fullySynced=%s round=%s lastPlayed=%s lastWon=%s stake=%s", - name, radius, within, synced, round, played, won, stake) - } -} - -// updatePeak polls each node, emits metrics, raises the per-node high-water peak, -// logs a line, and returns the min and max current storageRadius across the nodes -// that responded (min is 0 if none responded). +// updatePeak polls each node, emits metrics, raises the per-node high-water peak, and +// returns the min and max current storageRadius across the nodes that responded. func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8) (minR, maxR uint8) { seen := false for _, n := range nodes { @@ -1069,12 +387,24 @@ func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, minR = s.StorageRadius seen = true } - c.logger.Infof("%s: storageRadius=%d (peak %d) reserveSize=%d withinR=%d pullsyncRate=%.4f", - n.Name(), s.StorageRadius, peak[n.Name()], s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate) } return minR, maxR } +// snapshot polls every node, emits metrics, and logs a per-node line for the given phase. +func (c *Check) snapshot(ctx context.Context, nodes orchestration.ClientList, phase string) { + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + c.logger.Debugf("%s: status error: %v", n.Name(), err) + continue + } + c.emit(n.Name(), s) + c.logger.Infof("[%s] %s: storageRadius=%d reserveSize=%d withinR=%d pullsyncRate=%.4f warmingUp=%t", + phase, n.Name(), s.StorageRadius, s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate, s.IsWarmingUp) + } +} + func (c *Check) emit(node string, s *api.StatusResponse) { c.metrics.StorageRadius.WithLabelValues(node).Set(float64(s.StorageRadius)) c.metrics.ReserveSize.WithLabelValues(node).Set(float64(s.ReserveSize)) @@ -1082,20 +412,6 @@ func (c *Check) emit(node string, s *api.StatusResponse) { c.metrics.PullsyncRate.WithLabelValues(node).Set(s.PullsyncRate) } -func (c *Check) emitRedist(node string, r *api.RedistributionState) { - c.metrics.FullySynced.WithLabelValues(node).Set(b2f(r.IsFullySynced)) - c.metrics.Frozen.WithLabelValues(node).Set(b2f(r.IsFrozen)) - c.metrics.RedistRound.WithLabelValues(node).Set(float64(r.Round)) - c.metrics.LastSampleDuration.WithLabelValues(node).Set(r.LastSampleDurationSeconds) -} - -func b2f(b bool) float64 { - if b { - return 1 - } - return 0 -} - func sleepCtx(ctx context.Context, d time.Duration) error { select { case <-ctx.Done(): diff --git a/pkg/config/check.go b/pkg/config/check.go index 177f8f1f..9c38e0c7 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -501,32 +501,23 @@ var Checks = map[string]CheckType{ NewAction: reserveradius.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { - Mode *string `yaml:"mode"` - Duration *time.Duration `yaml:"duration"` - RndSeed *int64 `yaml:"rnd-seed"` - StakeAmount *string `yaml:"stake-amount"` - StakeGroups *[]string `yaml:"stake-groups"` - StakeConfirmWait *time.Duration `yaml:"stake-confirm-wait"` - PostageTTL *time.Duration `yaml:"postage-ttl"` - PostageDepth *uint64 `yaml:"postage-depth"` - PostageLabel *string `yaml:"postage-label"` - UploadGroups *[]string `yaml:"upload-groups"` - BlobSize *int64 `yaml:"blob-size"` - MaxUploads *int `yaml:"max-uploads"` - TargetRadius *uint8 `yaml:"target-radius"` - DisruptAtRadius *uint8 `yaml:"disrupt-at-radius"` - DisruptMechanism *string `yaml:"disrupt-mechanism"` - DisruptNodeCount *int `yaml:"disrupt-node-count"` - DisruptMethod *string `yaml:"disrupt-method"` - MinSurvivors *int `yaml:"min-survivors"` - Verdict *string `yaml:"verdict"` - ExpectRecovery *bool `yaml:"expect-recovery"` - WarmupWait *time.Duration `yaml:"warmup-wait"` - IncreaseTimeout *time.Duration `yaml:"increase-timeout"` - SettleWait *time.Duration `yaml:"settle-wait"` - DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` - RecoveryWait *time.Duration `yaml:"recovery-wait"` - PollInterval *time.Duration `yaml:"poll-interval"` + RndSeed *int64 `yaml:"rnd-seed"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + UploadGroups *[]string `yaml:"upload-groups"` + BlobSize *int64 `yaml:"blob-size"` + MaxUploads *int `yaml:"max-uploads"` + TargetRadius *uint8 `yaml:"target-radius"` + ForceDecrease *bool `yaml:"force-decrease"` + DiluteStep *uint64 `yaml:"dilute-step"` + MaxDilutions *int `yaml:"max-dilutions"` + GasPrice *string `yaml:"gas-price"` + WarmupWait *time.Duration `yaml:"warmup-wait"` + IncreaseTimeout *time.Duration `yaml:"increase-timeout"` + SyncSettleWait *time.Duration `yaml:"sync-settle-wait"` + DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + PollInterval *time.Duration `yaml:"poll-interval"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From f70aa9d162713ca3f6658f20f89dbe0102048f78 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Sat, 11 Jul 2026 21:36:25 +0200 Subject: [PATCH 28/37] chore: remove docs --- docs/radius-check-plan.md | 181 ------------------- docs/radius-halt-check-plan.md | 317 --------------------------------- docs/radius-halt.md | 164 ----------------- 3 files changed, 662 deletions(-) delete mode 100644 docs/radius-check-plan.md delete mode 100644 docs/radius-halt-check-plan.md delete mode 100644 docs/radius-halt.md diff --git a/docs/radius-check-plan.md b/docs/radius-check-plan.md deleted file mode 100644 index 4731eca9..00000000 --- a/docs/radius-check-plan.md +++ /dev/null @@ -1,181 +0,0 @@ -# Reserve-radius change check — plan - -Status: **Local check done (single idle-driven flow); Phase 3 + A/B pending.** Branch `ljubisa-radius-soak-v2`. - -A repeatable, automated way to stage a **storage-radius change** across a Bee cluster and -verify pull-sync recovers afterwards. End product: a beekeeper check -(`pkg/check/reserveradius`) plus the metrics it emits. Operating know-how lives in the -`radius-testing` skill; this doc is the design and build checklist. - -## Pull-sync validation (the real target) - -The narrow goal is "force a radius decrease, confirm the puller doesn't deadlock" (PR #581). -The broader goal — why @sig/@marios want this — is a **reproducible test of the October -network halt** and validation of the pull-sync redesign (SWIP-25 / `pullsync-optimal-design`). - -**The October mechanism.** A large operator went offline → commitment dropped → **storage -radius decreased**, a neighbourhood **merge**: `k` jumps to ~8, so every node must re-sync a -much larger reserve from many more peers. Today's pull-sync is "correctness through -redundancy" — up to **k× redundant deliveries** per chunk, no shared dedup, no -failover-with-exclude; on a depth change bins are cancelled/restarted repeatedly (unbounded -delay) and stalls livelock. Result: incomplete reserves → wrong `ReserveSample` → nodes -**freeze, skip rounds, go unresponsive** (slashing risk). - -**What the test must show** on a staged decrease: - -- **Completeness** — reserves catch up (`reserveSizeWithinRadius` recovers). -- **Liveness** — no freezes, `round` keeps advancing, `isFullySynced` returns within a bound. -- **Bounded convergence** — resync finishes in bounded time; no stuck `STALLED` bins. -- **Efficiency (SWIP-25)** — redundant deliveries drop ~k×→1× (`offered/delivered` ratio). - -**Implemented so far:** a single self-driving check (`pkg/check/reserveradius`) drives the radius -up by uploading random data to random nodes to a target (≥3), stops, waits for pull-sync to go idle -(`pullsyncRate<=0.05` — the reserve worker only decreases while `SyncRate()==0`), dilutes one batch -to the chain's min-validity floor to hasten its expiry (force-decrease), then asserts the radius -ticks back down below its peak within a timeout. Metrics per node: `storage_radius`, `reserve_size`, -`reserve_within_radius`, `pullsync_rate`, plus `time_to_increase_seconds` / `time_to_decrease_seconds` -/ `dilution_total`. Validated live 2026-07-11 on the patched 6-node local-dns: 0→3 in ~64 MiB / -~6 min, batch diluted 48h→6h (floored), radius 3→2 at ~8 min. **Locally the decrease is pull-sync- -idle-gated** (threshold patched to 100% ⇒ commitment/expiry irrelevant); **on testnet it is -expiry-driven** — dilution hastens the batch expiry that drops commitment. The deeper liveness -signals (`/redistributionstate` freeze / round-loss) -are documented in the `radius-testing` skill for follow-up. - -**Still needs @marios + scale:** exact pass/fail thresholds; the per-`(peer,bin)` `STALLED` / -`MaxStallsPerBin` metrics (only @sig's fixed PR exposes them); the **~20-node cluster** (the -merge needs scale); the **A/B harness** (below). - -## Prior art - -- **PR #581** (`radiusdecrease`) — forces a decrease (overflow tiny reserve, 0→1→0), polls - `/status` for `pullsyncRate>0` recovery. Needs a patched bee image. The real radius tryout. -- **PR #591** (`stampexpiry`) — patch-free, observes radius via stamp expiry. Review feedback: - use `PostageTTL`, random node, periodic/RC cadence, not standard CI. -- **GC check** (`pkg/check/gc/reserve.go`) — precedent for a patch-dependent check; patches live - in `bee/.github/patches/`, applied by bee's beekeeper workflow. - -## Signals - -Per node: `/status` (`storageRadius`, `pullsyncRate`, `reserveSize`, `reserveSizeWithinRadius`, -`committedDepth`, `isWarmingUp`), `/reservestate` (network `radius`), `/redistributionstate` -(`isFullySynced`, `isFrozen`, `round`). Prometheus: `bee_localstore_*`, `bee_pullsync_*`, -`bee_storageincentives_*`. Full table in the `radius-testing` skill. - -## Approach - -Patched-local first (deterministic, fast transitions) → unpatched ephemeral k8s (realistic -timing, periodic/RC cadence). Same check, different environment and timeouts. - -## Phase 0 — Local scaffolding (done) - -- [x] `/cluster-verify`, `/cluster-up`, `/cluster-down` commands. -- [x] `radius-testing` skill + `scripts/radius-poll.sh` monitor. - -## Staging the change (reuse the `load` check) - -Don't pre-compute byte counts — upload in a loop, watch the signal, stop at a target. `load` -(`pkg/check/load/load.go`) already does this: `MaxCommittedDepth` gates uploads via -`checkCommittedDepth` (`committedDepth = storageRadius + capacityDoubling`). - -**Increase and decrease are not symmetric.** Increase is upload-driven (fill past capacity → -eviction → radius up). Decrease is not — per `bee/pkg/storer/reserve.go`, radius drops only when -the reserve is under-utilized AND fully synced (`SyncRate()==0`) above `minimumRadius`. Trigger -it by inflating first, then letting the patched reserve worker re-evaluate (0→1→0 cascade). - -## Phase 1 — Spike: force a change with a patched bee (done) - -- [x] **Patch created** in `bee/.github/patches/` (GC-check pattern; all three symbols are - unreachable by config) — what and why: - - `radius_reserve.patch`: `DefaultReserveCapacity` 1<<22→**200** (so ~2 MB moves the radius — - stock is too large to budge in CI-time) and `ReserveWakeUpDuration` 30m→**10s** (the reserve - worker re-evaluates in seconds, not 30 min). - - `radius_threshold.patch`: decrease `threshold` 50%→**100%** (makes the 1→0 decrease reachable). - gofmt-clean, applies cleanly. -- [ ] **Apply, build, push, redeploy** (revert source after build): - -```sh -cd -patch pkg/storer/reserve.go .github/patches/radius_threshold.patch -patch pkg/storer/storer.go .github/patches/radius_reserve.patch -make docker-build PLATFORM=linux/arm64 BEE_IMAGE=k3d-registry.localhost:5000/ethersphere/bee:latest \ - REACHABILITY_OVERRIDE_PUBLIC=true BATCHFACTOR_OVERRIDE_PUBLIC=2 -docker push k3d-registry.localhost:5000/ethersphere/bee:latest -git checkout pkg/storer/reserve.go pkg/storer/storer.go -``` - -Then `/cluster-down local-dns` → `/cluster-up local-dns`. To wire into CI, add the two `patch` -lines to bee's `.github/workflows/beekeeper.yml` "Apply patches and build" step. - -### Why the check is shaped this way (spike, 2026-06-17) - -The decrease is slow and non-obvious; each fact below maps to a check decision: - -- **Lags ~10 min, stabilization-gated** (the reserve worker's decrease loop waits on the - startup-stabilizer) → long timeouts (decrease ≥15 min), gate on `isWarmingUp==false`. -- **Overshoots, then settles in a staircase** (in-flight pushsync keeps filling after uploads stop; - each decrease triggers pull-sync that blocks the next) → wait for pushsync drain and track the - high-water peak before treating a value as settled. -- **No fixed floor** (radius held at 1 for 30+ min) → assert *"a decrease occurred AND pull-sync - recovered"*, not *"radius returned to N"*. -- **Puller drives it** — `node/puller "radius decrease"` + a `syncWorker context cancelled` storm = - the PR #581 `manage()`/`disconnectPeer()` path (the liveness risk). - -## Phase 2 — The check (done) - -`pkg/check/reserveradius/`. `Run` = `selectNodes` → `waitForWarmupDone` → baseline → -`driveIncrease` (upload random data to random nodes under this check's own postage label, to -`target-radius`) → `waitPullSyncIdle` (wait `pullsyncRate<=0.05` on all nodes) → -`diluteToMinValidity` (force-decrease: dilute one batch to the min-validity floor to hasten expiry) -→ `observeDecrease` (assert the radius ticks below its peak within `decrease-timeout`). Registered -as type `reserve-radius` (`pkg/config/check.go`); `ci-reserve-radius` in `config/local.yaml`; added -`IsWarmingUp` to `StatusResponse`. build/vet/lint green. - -- [x] **Validated end-to-end (2026-06-17)**: increase 0→1 in 3s, decrease observed at 13m16s. -- [x] **Re-validated (2026-07-11, target 3, dilute-to-min-validity + drop)**: 0→3 in ~64 MiB / - ~6 min, pull-sync idled, diluted a batch 48h→24h→12h→6h (floored at the min-validity), radius - 3→2 at ~8 min, check passed. Needs the patched image (the default registry image is unpatched - — the radius won't climb). - -## Phase 2.5 — the decrease lever: pull-sync idle (local) / batch expiry (testnet) (done) - -Two paths reach the decrease, both gated on `SyncRate()==0`: - -- **Local (patched cluster)** — the reserve worker's gate (`reserve.go`: - `countWithinRadius < threshold && SyncRate()==0 && radius > minimumRadius`) reduces to just - **`SyncRate()==0`** (`threshold` patched to 100%, `countWithinRadius` well below it), so once - pull-sync idles the radius ticks down on its own — measured ~8–13 min after idle. `waitPullSyncIdle` - (`pullsyncRate<=0.05`; the rate settles to a small residual, never exactly 0) opens that gate. -- **Testnet (real reserve)** — the decrease is **commitment-driven**: it needs a batch to expire so - its chunks are evicted. `force-decrease` **dilutes one batch to the chain's `minimumValidityBlocks` - floor** (each `+1` ~halves TTL; the contract reverts any dilution below the floor, so expiry itself - is unreachable — dilution only *hastens* it) so the batch expires in ~a few hours instead of its - full TTL. Requires a non-zero chain price (see the `change-storage-price` skill); with price 0 - batches have infinite TTL. Full mechanics: [`docs/radius-halt.md`] and the radius-testing notes. - -The general `load` primitives added along the way remain available but are not wired into the radius -check: the `--parallel` check flag (`check.go` + `runner.go`) and `load`'s re-arming `decrease-hold`. - -Run: `beekeeper check --checks=ci-reserve-radius --timeout=45m` (local; raise `decrease-timeout`/`timeout` on testnet). - -## Phase 3 — Real ephemeral cluster - -- [ ] Run the check against an unpatched ~20-node ephemeral cluster (realistic timing). -- [ ] Wire as a periodic / public-testnet RC check (per #591), not standard PR CI. - -## A/B testing procedure (current vs fixed bee) - -Run the *identical* radius-decrease scenario against two builds and diff them: - -- **A = current bee** (stock `master`) — expected to **reproduce** the halt. -- **B = fixed bee** (@sig's SWIP-25 PR) — expected to **survive** it. - -## Open questions - -- Patch vs config: can the small-reserve setup come from `bee-config` knobs alone? -- Does `/redistributionstate` need full-mode + incentives locally, or assert on `/status` + - `/reservestate` only? -- Metrics sink for local runs: pushgateway vs scrape vs `radius-poll.sh` CSV. - -## References - -- Skill: `.claude/skills/radius-testing/SKILL.md` (+ `scripts/radius-poll.sh`) diff --git a/docs/radius-halt-check-plan.md b/docs/radius-halt-check-plan.md deleted file mode 100644 index d3c0c28b..00000000 --- a/docs/radius-halt-check-plan.md +++ /dev/null @@ -1,317 +0,0 @@ -# Reserve-radius check — halt-reproduction mode (plan) - -> **Superseded (2026-07).** The `reserveradius` check was simplified to a **single -> self-driving flow with one disruption mechanism: batch-expiry via dilution** (see -> `docs/radius-check-plan.md` and `pkg/check/reserveradius/README.md`). The `mode: halt` -> design below — modes, **node churn**, staking, verdict/assert, and HALT/RECOVERED -> outcome classification — was implemented on this branch and then removed. This doc is -> kept for the background it captures (the bee-side batch-expiry → commitment-drop → -> gated-radius-decrease path, and the empirical halt findings that motivated the check); -> it no longer describes the current code. - -Status: **Superseded — see the banner above.** Branch `ljubisa-radius-soak`. - -Goal: fold the now-working **manual** pull-sync-halt reproduction (see `docs/radius-halt.md`) -into the `reserveradius` check so the whole flow runs **unattended in one longer run** — -stake → drive radius to N → disrupt the neighbourhood (**node churn and/or batch expiry**) → observe -convergence/halt + the staked redistribution round-loss — and so the same check can serve the -**A/B harness** (stock master vs -`pullsync-optimal-design`). Optionally run alongside the `load` check in parallel. - -This plan only covers the **check code** (`pkg/check/reserveradius`). Environment setup — -building the patched bee image, deploying + funding the cluster — stays manual / in the -`cluster-up` flow and is out of scope here. - -## What the manual flow does today (the target to automate) - -From `docs/radius-halt.md` "Staked run", the reproduced recipe is: - -1. Stake every full node (`POST /stake/1e17`, verify `stakedAmount`). -2. Drive the radius to **3** with `ci-load-soak` (`max-committed-depth: 3`); stop the load once - all nodes report `storageRadius=3`. -3. **Remove 2 of 6 full nodes** (`kubectl delete statefulset/pod`) to disrupt the populated - neighbourhood. -4. Observe the 4 survivors ≥25 min: delayed onset (~8 min) → `within_radius` 0→~2200, - `fullySynced→false`, `pullsyncRate` decays toward 0, split-brain end-state, never re-converges. -5. Confirm the staked round-loss: `is_playing_errors` cluster-wide; un-synced nodes can't produce a - `ReserveSample`, so they win nothing while rounds keep advancing. - -The check already does step 2's driving (`mode: drive`) and a generic observe loop (`mode: observe`), -but has **no** staking, **no** node removal, and asserts only a *radius decrease + recovery* — not the -disruption-driven non-convergence we actually reproduce. - -## Capabilities confirmed (no blockers) - -- **Node removal — supported.** `orchestration.Node.Delete(ctx, namespace)` (deletes statefulset + - services/ingress/secret/configmap) and `Node.Stop(ctx, namespace)` (scales statefulset to 0); also - `NodeGroup.DeleteNode(ctx, name)`. The check reaches them via `cluster.Nodes()` / - `cluster.NodeGroup(name)`, and the namespace via `cluster.Namespace()`. -- **Staking — supported.** `bee.Client.DepositStake(ctx, *big.Int)` → `POST /stake/{amount}`, - `GetStake(ctx)` → `GET /stake`. The `stake` check already uses these via `cluster.NodesClients`. -- **Batch expiry — supported (the original October lever).** `CreatePostageBatch(ctx, amount, depth, - label)` (explicit amount → controllable expiry; batches are mutable) and `GetOrCreateMutableBatch` - (TTL→amount via chainstate price). Read expiry via `PostageStamp(batchID).BatchTTL`/`Exists` and - `GetChainState().TotalAmount`. Bee side (confirmed): expiry → `EvictBatch` → `evictExpiredBatches` - → commitment drop → gated radius decrease in `pkg/storer/reserve.go`. No in-repo `stampexpiry` - check yet (PR #591 is external); `pkg/check/gc/reserve.go` is the tiny-amount-batch precedent. -- **Redistribution round-loss — assertable WITHOUT `/metrics` scraping.** Checks can't scrape - Prometheus, but `/redistributionstate` already returns `Round`, `LastPlayedRound`, `LastWonRound`, - `LastSelectedRound`, `LastSampleDurationSeconds`, `IsFullySynced`, `IsFrozen`. Round-loss = - `Round` advances while `LastPlayedRound`/`LastWonRound` stalls (and `IsFullySynced=false`). The - check reads `/redistributionstate` today but ignores these fields. -- **Not available:** the `bee_pullsync_chunks_delivered` "delivered-plateau" smoking gun (needs a new - `/metrics` scrape on the bee client). The structured equivalents — `pullsyncRate` decay + - `reserveSizeWithinRadius` plateau + `fullySynced` stuck false — are enough; treat `/metrics` - scraping as a deferred nice-to-have (Phase 7). - -## Design - -A new self-driving **`mode: halt`** that composes the existing pieces plus two new stages, so one -run reproduces the whole recipe: - -``` -stake(optional) → waitWarmup → driveIncrease(to DisruptAtRadius) → settle - → disrupt(node-churn | batch-expiry) → observeOutcome(on survivors) -``` - -The disruption + staking + outcome-classification are written as **composable stages gated by -options**, so the parallel-with-load shape reuses them: `mode: observe` + a disruption mechanism means -"don't upload — let the `load` check drive the radius — but still stake, disrupt at the target, and -classify the outcome." - -**Both shapes are first-class (decided).** Write the stages composable so the radius can be driven -either by the check itself (`mode: halt`) **or** by a parallel `load` check (`mode: observe` + -disrupt). To avoid a fragile cross-check coordination channel, the observer **tolerates ongoing -uploads**: with `node-churn` the halt is a neighbourhood-redundancy drop, not a committed-depth effect, -so the assertions (`fullySynced`, round-loss) hold whether or not `load` keeps running. The manual run -*stopped* load only for a cleaner CSV; it is not required for the node-churn halt. **Caveat for -`batch-expiry`:** that lever works by *dropping* commitment, so ongoing `load` (which re-adds -commitment) counteracts it — in parallel-with-load, prefer `node-churn`, or have `load` wind down -around the expiry. Single-mode remains the most faithful reproduction; parallel-mode is the -soak/realism shape. - -**Disruption mechanisms (pluggable: node-churn and/or batch-expiry).** `disrupt-mechanism` chooses how -the neighbourhood is disrupted, both feeding the same observe/classify/verdict stage: -- `node-churn` (default) — stop/delete randomly-chosen full nodes. The reliable redundancy-drop lever - (what the manual repro used): removes neighbourhood replicas so survivors must re-replicate. -- `batch-expiry` — the **original October mechanism**. Create a short-lived postage batch, fill the - reserve under it, then let it expire: bee evicts its chunks cluster-wide (`evictExpiredBatches`), - commitment drops, and on a synced node below the decrease threshold the **storage radius decreases** - → neighbourhood merge → pull-sync re-sync. More faithful to October than node-churn, but the - decrease is **gated** (`SyncRate==0` + `countWithinRadius < threshold`) so it fires intermittently — - the `radius_threshold.patch` (threshold = capacity) makes it reachable on a small cluster. -- `both` — expire a batch *and* churn nodes for a harsher merge; `none` — monitor-only. - -**Tunable disruption + a non-failing verdict (decided).** Each mechanism's intensity is tunable -(`disrupt-node-count` for node-churn, **randomly selected** seeded by `rnd-seed`; `expiring-batch-ttl` -for batch-expiry) and the whole disruption can be turned off (`disrupt-mechanism: none`, or -`disrupt-node-count: 0`) so the check just monitors radius / sync / round state for `duration` (watch a -healthy cluster, or build a baseline). Either way the check **reports the outcome rather than failing -on it** — so the *same* check can reproduce the halt, deliberately *not* reproduce it (low intensity), -or just monitor, all without a red result. - -**Outcomes** the observe stage classifies and emits (a metric + an end-of-run summary line): -- `MONITORED` — `disrupt-node-count: 0`; no disruption, state recorded over `duration`. -- `HALT` — after disruption, sustained non-convergence (survivors stuck `fullySynced=false` past a - bound) and/or staked round-loss. -- `RECOVERED` — after disruption, all survivors returned to `fullySynced` (and resumed playing rounds) - within `recovery-wait`. - -**Verdict policy** (`verdict`): -- `report` (**default**) — always return success on any observed outcome; emit the outcome + metrics + - a clear summary. Fail **only** on operational errors (warmup timeout, stake/removal failure, all - nodes unreachable). This is the explore / soak / reproduce-or-not mode. -- `assert` — gate on the expectation: fail iff the observed outcome contradicts `expect-recovery` - (`false` ⇒ expect `HALT`, `true` ⇒ expect `RECOVERED`). For the A/B regression gate. `MONITORED` - (no disruption requested) is always a pass. - -## Phases - -### Phase 0 — Shape decisions (RESOLVED) -- [x] **Both shapes first-class**: `mode: halt` (self-driving) **and** `mode: observe` + disrupt - (parallel with `load`). Stages written composable to serve both. -- [x] **Parallel contract**: observer **tolerates ongoing uploads** (no cross-check IPC); assertions - are upload-robust. Stopping `load` is optional, not required. -- [x] **Node-removal method default = `stop`** (scale statefulset to 0; restorable; emptyDir means a - restart is a fresh node so restore-mid-run needs re-funding — a Phase-7 nicety). `delete` stays - available as an option. -- [x] **A-side semantics = PASS when halt reproduced** (`expect-recovery: false` is the default → - reproduction gate, not a regression detector). -- [x] **Tunable disruption + non-failing default**: `disrupt-node-count` randomly selects nodes - (`0` = monitor-only), and `verdict: report` (default) records `HALT`/`RECOVERED`/`MONITORED` - and only fails on operational errors. `verdict: assert` opts into A/B gating via `expect-recovery`. - -### Phase 1 — Staking pre-step -- [x] Add options: `StakeAmount` (config `stake-amount` string→parsed `*big.Int` in the stage, - e.g. `"100000000000000000"`; empty/`"0"` = skip), `StakeGroups` (`stake-groups`, default = - observed groups). Held as a `string` in `Options` (parsed at use-time) so config mapping and the - empty=skip sentinel stay trivial. -- [x] New stage `ensureStaked` (+`ensureNodeStaked`): for each node in the staking set (`StakeGroups`, - or the observed nodes), `GetStake`; if `< StakeAmount`, `DepositStake`; re-read to verify. - Idempotent (already-at-target nodes skipped), logged per node. Wired before warmup in both - `runDrive` and `runObserve`, gated on `stake-amount` set. -- [x] Budget the ~10-block usable wait via `waitStakeAtLeast` + `stake-confirm-wait` (default `2m`): - `POST /stake` returns a tx hash, so poll `GetStake` until the deposit confirms on-chain (or the - budget expires). Already-mined deposits pass immediately. - -### Phase 2 — Drive (or wait) to the disruption radius -- [x] Add `DisruptAtRadius` (default 3, `disrupt-at-radius`) distinct from `TargetRadius`. -- [x] `mode: halt` (`runHalt`): stake → warmup → `driveAllToRadius` (gates on the **min** node radius, - not the max, so ALL observed nodes reach `DisruptAtRadius`) → settle window. `updatePeak` now - returns `(min, max)`. Disruption (Phase 3) + outcome observation (Phase 4) are stubbed with TODOs. -- [x] `mode: observe`+disrupt: poll until every observed node reports - `storageRadius >= DisruptAtRadius` (load drives), with a timeout. - **(Resequenced → done in Phase 3.0 as `waitAllReachRadius`.)** -- [x] Record a pre-disruption baseline snapshot (`baselineSnapshot`): per node, storageRadius + - reserveSizeWithinRadius (/status), isFullySynced + round/lastPlayed/lastWon - (/redistributionstate), and stake — emitted + logged. Wired into `runHalt` (replaces the plain - `snapshot("baseline")`). Phase 4 will extend it to return the values for onset/round-loss compare. - -### Phase 3 — Disruption mechanisms (`disrupt-mechanism`) -Shared: a `disruption_total` metric + timestamp marks the onset reference. `disrupt-mechanism: none` -(or node-churn with `disrupt-node-count: 0`) skips straight to observe (monitor-only). - -**3.0 — observe+disrupt dispatch + staging** (resequenced from Phase 2; built once 3a + Phase 4 landed) -- [x] `disrupt-mechanism` dispatch (`disruptionActive` + `runObserveDisrupt`): `mode: observe` with an - active mechanism runs the staged reproduction (wait-to-radius → baseline → disrupt → observe → - verdict) instead of the plain soak monitor. **Behaviour change:** with the default - `node-churn`/count-2, a bare `mode: observe` now disrupts; monitor-only requires - `disrupt-mechanism: none` (or count 0). The existing `ci-reserve-radius-observe` was set to `none`. -- [x] `mode: observe`+disrupt staging (`waitAllReachRadius`): poll until every observed node reports - `storageRadius >= DisruptAtRadius` (load drives) within `IncreaseTimeout`; then `baselineSnapshot`. - -**3a — node-churn** -- [x] Add the disruption option surface: `DisruptMechanism` (`disrupt-mechanism`, default `node-churn`), - `DisruptNodeCount` (default 2; `0` = skip), `DisruptMethod` (`stop` default | `delete`), and - `MinSurvivors` (default 3) — with mechanism (`DisruptNodeChurn`/`DisruptBatchExpiry`/`DisruptBoth`/ - `DisruptNone`) and method (`RemoveStop`/`RemoveDelete`) constants, config mapping, defaults, README. - *(MinSurvivors + DisruptMechanism folded in here since the node-churn stage consumes all of them.)* -- [x] **Randomly** select `DisruptNodeCount` nodes, seeded by `rnd-seed` (`rnd.Perm`, reproducible); - excludes `excludeName` (the uploader in `halt` mode). Builds the **survivor set** (observed minus - removed) returned for the observe loop. `DisruptNodeCount <= 0` = monitor-only (returns all). -- [x] Remove via `node.Stop(ctx, namespace)` / `node.Delete(...)` (looked up from `cluster.Nodes()` by - name), selected by `DisruptMethod`. Logs + RFC3339-timestamps each removal; emits `disruption_total` - (labelled by mechanism). Implemented as `disruptNodeChurn`, behind a `disrupt` mechanism dispatcher - (node-churn/none done; batch-expiry/both error as Phase-3b). Wired into `runHalt`. -- [x] Guard: `len(observed) - DisruptNodeCount >= MinSurvivors`, plus a candidate-count check — both - error out **before** any removal touches the cluster. - -**3b — batch-expiry** *(DEFERRED to live co-development in Phase 6 — decided 2026-06-29)* -> **Decision (user, 2026-06-29):** defer batch-expiry to live co-development. The whole primary -> deliverable (node-churn halt, end-to-end, both shapes, config, docs) is complete and verified. -> Batch-expiry is the **secondary** mechanism and its defining behaviour — the *gated* radius decrease -> (`SyncRate==0` + `count patched bee image + a live `local-dns` cluster. Rather than ship speculative, rework-prone code, it is -> implemented **during Phase 6**, when the cluster is up and the eviction → gated-decrease path can be -> observed and tuned against real signals. The autonomous code loop ended here. -- [ ] Create a soon-to-expire mutable batch — `CreatePostageBatch(ctx, amount, depth, label)` with a - small explicit `amount`, or `GetOrCreateMutableBatch` with a short `expiring-batch-ttl`. Drive the - radius (Phase 2) by uploading the fill **under this batch** so its chunks dominate the reserve. -- [ ] Detect expiry: poll `PostageStamp(batchID)` for `BatchTTL`→0 / `Exists=false`, cross-checking - `GetChainState().TotalAmount` against the batch `Amount`; timestamp it as onset. -- [ ] Observe eviction: `reserveSize` / `reserveSizeWithinRadius` drop as `evictExpiredBatches` runs; - then watch for the **gated** radius decrease (synced + `count minimumRadius`. Stock `threshold = capacity*50%`; `radius_threshold.patch` raises it to - `capacity`. `SyncRate()==0` rarely holds on a busy cluster, so decreases are slow/intermittent — - node scale-down triggered one only 1 of 4 attempts; stop-load and `dilute` never did (`dilute` is the - *wrong* lever — it raises committed capacity). The faithful decrease lever is a commitment drop - (batch expiry), which is slow/hard to control locally. - -## Prerequisites - -- beelocal/k3d substrate + `geth-swap` + metrics stack (pushgateway 9091 / prometheus 9090 / grafana 3000). -- Patched image `k3d-registry.localhost:5000/ethersphere/bee:latest` = `radius_reserve.patch` - (DefaultReserveCapacity 4000, ReserveWakeUpDuration 10s) + `radius_threshold.patch` (threshold = capacity). - Build it (apply both patches to the bee source, build, push, revert source): - - ``` - patch pkg/storer/storer.go .github/patches/radius_reserve.patch - patch pkg/storer/reserve.go .github/patches/radius_threshold.patch - make docker-build PLATFORM=linux/arm64 BEE_IMAGE=k3d-registry.localhost:5000/ethersphere/bee:latest \ - REACHABILITY_OVERRIDE_PUBLIC=true BATCHFACTOR_OVERRIDE_PUBLIC=2 - docker push k3d-registry.localhost:5000/ethersphere/bee:latest - git checkout pkg/storer/storer.go pkg/storer/reserve.go - ``` - -- `config/local.yaml`: `local-dns` bee `count: 6`, light `count: 0`; `ci-load-soak` `max-committed-depth: 3`. -- Geth **chain price must be non-zero** (e.g. 24000); price 0 logs `invalid chain price` and breaks postage. -- Env prefix for every beekeeper command (local chain; `~/.beekeeper.yaml` is Sepolia): - `BEEKEEPER_GETH_URL=http://geth-swap.localhost BEEKEEPER_BZZ_TOKEN_ADDRESS=0x6aab14fe9cccd64a502d23842d916eb5321c26e7 BEEKEEPER_ETH_ACCOUNT=0x62cab2b3b55f341f10348720ca18063cdb779ad5 BEEKEEPER_WALLET_KEY=4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d` - -## Reproducible steps - -1. **Recreate a clean cluster** (NEVER `kubectl rollout restart` — storage is emptyDir; a restart wipes - reserve+statestore+chequebook and needs re-funding). Geth survives (deployed separately): - - ``` - beekeeper delete bee-cluster --cluster-name=local-dns --geth-url http://geth-swap.localhost - beekeeper create bee-cluster --cluster-name=local-dns --geth-url http://geth-swap.localhost --wallet-key - ``` - -2. **Stake every node** via the API (minimum `1e17`; height 0 because `reserve-capacity-doubling=0`). - Staking is not needed for the *sync* signals, but it is needed to measure the redistribution - round-loss. NOTE: `ci-stake` does NOT stake the cluster — it is a single-node contract test that - deposits then withdraws to 0. - - ``` - for n in bee-0 bee-1 bee-2 bee-3 bee-4 bee-5; do - curl -s -XPOST http://$n.localhost/stake/100000000000000000 - done - # verify: curl -s http://bee-0.localhost/stake | jq .stakedAmount → 100000000000000000 on all six - ``` - -3. **Drive the radius to 3** (stop the load once every node reports `storageRadius=3`): - - ``` - beekeeper check --cluster-name=local-dns --checks=ci-load-soak --metrics-enabled=true --metrics-pusher-address=localhost:9091 - pkill -f 'beekeeper check' # reserveSizeWithinRadius is now REAL (~2000+), not 0 - ``` - -4. **Trigger the halt — remove 2 of the 6 nodes** (disrupt the populated neighbourhood): - - ``` - kubectl delete statefulset bee-4 bee-5 -n local - kubectl delete pod bee-4-0 bee-5-0 -n local --grace-period=3 - ``` - -5. **Observe the 4 survivors for ≥25 min** (onset is delayed ~3–9 min — short windows miss it): - - ``` - for n in bee-0 bee-1 bee-2 bee-3; do - curl -s http://$n.localhost/status | jq '{r:.storageRadius,w:.reserveSizeWithinRadius,ps:.pullsyncRate}' - curl -s http://$n.localhost/redistributionstate | jq '{fs:.isFullySynced,fr:.isFrozen,rnd:.round}' - curl -s http://$n.localhost/metrics | grep '^bee_pullsync_chunks_delivered ' - done - ``` - -6. **Cleanup:** `beekeeper delete bee-cluster --cluster-name=local-dns --geth-url http://geth-swap.localhost`. - -## What happens (measured; consistent across runs, numbers from the 31-min staked run) - -| Phase | t after removal | Observation | -| --- | --- | --- | -| Quiet | 0–~8 min | radius 3, `within_radius=0`, `fullySynced=true`, rounds advance. Delayed, variable onset (~3–9 min). | -| Onset | ~8 min | `within_radius` jumps 0→~2200 on all survivors; some radius 3→2; `pullsyncRate` spikes (0.7–0.9); `fullySynced→false`. | -| Stall | ~9–29 min | all survivors stuck `fullySynced=false`; `pullsyncRate` decays monotonically toward 0; `bee_pullsync_chunks_delivered` **plateaus** after one onset burst while `offered` ran 24k–34k; rounds advance but stuck nodes can't participate. | -| End | ~30 min | **split-brain, never re-converged** — nodes at *different* radii (3/2/2/3) and four `within_radius` plateaus. Some flip *back* to `fullySynced=true` once `pullsyncRate→0`, but `delivered` stayed flat — they "synced" by the **puller giving up** (SyncRate=0) at inconsistent radii, not by completing. No hard `isFrozen`. No recovery in 30 min. | - -**Smoking gun:** `bee_pullsync_chunks_delivered` goes flat while `fullySynced=false` — the puller gives -up before completing the re-sync. `offered/delivered` ran ~2.4×–17.7× — SWIP-25's k×-redundancy problem. - -## Redistribution round-loss (staked run, 2026-06-26 — the economic impact) - -With all six nodes staked, `bee_storageincentives_*` at the stall showed: - -- `is_playing_errors = 3` on **all four** survivors — every node errors when it tries to play. -- A node that never finishes the re-sync has `reserve_sample_duration = 0` and `winner = 0` (e.g. - bee-2) — it **cannot produce a valid `ReserveSample`, so it cannot win any round**. -- Rounds keep advancing (no hard freeze at this scale), so the chain does not pause — staked survivors - simply **lose every round**. At network scale, with frozen rounds and slashing, this is the October halt. - -## Findings - -- The radius **increase** is deterministic (committed-depth lever); the **decrease** is not reliably - stageable on a small cluster (see Mechanism). The halt does not require a decrease. -- On **A** the cluster **never re-converges** after disruption (persistent ≥30 min, no recovery). -- The small-cluster signature is a **soft stall / non-convergence**, not a hard `isFrozen` freeze — but - staked un-synced survivors still lose every redistribution round, which is the halt at scale. - -## Gotchas - -- **Radius 3 is required** — radius 1–2 is degenerate (`within_radius=0`, nothing to stall on). -- **Ephemeral storage (emptyDir)** — recreate, don't restart; re-stake after every recreate. If a node - was restarted and lost its chequebook (`/status` 503, "no chequebook found"), recover with - `beekeeper node-funder --namespace local --min-native 1 --min-swarm 100`. -- **Staking** — use `POST /stake/` per node; `ci-stake` does not stake the cluster. -- **Pull-sync must be healthy** — if node logs show `pushsync … context deadline exceeded` cluster-wide, - chunks won't propagate, the reserve won't fill, and the radius stays 0. Restart the beelocal substrate - (a cluster recreate alone does not fix it). -- **Chain price must be non-zero** (price 0 breaks postage/uploads). -- **Observe ≥25 min** — the delayed onset means short windows wrongly conclude "no effect". - -## A/B proof (next) - -Run this identical recipe against **B = `pullsync-optimal-design`** (built with the same `radius_*` -patches) and confirm the survivors re-converge (`fullySynced→true`) where A stays stuck. The staked run -gives the decisive B-side metric: survivors should keep `is_playing_errors`/`winner` healthy because -they finish the re-sync and can still sample. That is the proof the redesign fixes the halt. - -Artifacts from the staked run: `radius-drive.csv` (radius 0→3) and `halt-timeline.csv` (survivors, 30 s, -full signal set incl. stake + round + delivered). From 9e0cb35c50931c3468e04e47064c925a66ed18b6 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Mon, 13 Jul 2026 08:09:33 +0200 Subject: [PATCH 29/37] fix: add warning if chain price is 0 --- pkg/check/reserveradius/README.md | 24 +++++++++++++----------- pkg/check/reserveradius/reserveradius.go | 8 +++++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md index 3ceb8b30..fcb55ac0 100644 --- a/pkg/check/reserveradius/README.md +++ b/pkg/check/reserveradius/README.md @@ -92,13 +92,16 @@ check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem - **Patched bee image.** On stock capacity the reserve is far too large to move. The check needs a node built with `bee/.github/patches/radius_reserve.patch` (`DefaultReserveCapacity`→4000, `ReserveWakeUpDuration`→10s) and - `radius_threshold.patch` (decrease `threshold`→100%). See - `docs/radius-check-plan.md` and the `radius-testing` skill. The default local-cluster - image is **not** patched — build and push it first, or the radius never climbs. -- **A non-zero postage price is optional.** The decrease is driven by pull-sync idling - (`SyncRate()==0`), not by batch expiry, so it works at price 0 (batches never expire). - If a price *is* set (see the `change-storage-price` skill), the contract enforces a 24h - minimum batch validity, so `postage-ttl` must exceed 24h — hence the `48h` default. + `radius_threshold.patch` (decrease `threshold`→100%). See the `radius-testing` skill. The + default local-cluster image is **not** patched — build and push it first, or the radius never + climbs. +- **Postage price — optional locally, required for the testnet (expiry-driven) path.** Locally + the decrease is driven by pull-sync idling (`SyncRate()==0`), so the check passes at price 0 — + but at price 0 batches have infinite TTL (`batchTTL -1`), so the `force-decrease` dilute step + has nothing to hasten and skips itself. On testnet the decrease is **expiry-driven**, so a + **non-zero price is required** (set it with the `change-storage-price` skill) — otherwise no + batch ever expires. Once a price is set the contract enforces a minimum batch validity, so + `postage-ttl` must exceed 24h — hence the `48h` default. - **`isWarmingUp` in `StatusResponse`** (`pkg/bee/api/status.go`) — the node returns it; the struct field was added for this check's warmup gate. @@ -122,13 +125,12 @@ Against a patched local cluster (`local-dns`): batch below `minimumValidityBlocks` (~a few hours here), so a batch floors at ~6h validity and can't be diluted to 0. `force-decrease` therefore *hastens* the natural expiry (dilutes to the floor) rather than triggering it directly. Locally this is redundant with the idle tick (which - fires first); it matters on testnet, where the decrease is expiry-driven. See - `docs/radius-check-plan.md`. + fires first); it matters on testnet, where the decrease is expiry-driven. See the + `radius-testing` skill for the full mechanics. ## Related -- `docs/radius-check-plan.md` — design, phases, spike findings. -- `.claude/skills/radius-testing/` — operating know-how, `radius-poll.sh`, the metrics stack. +- `.claude/skills/radius-testing/` — design, mechanics, operating know-how, `radius-poll.sh`, the metrics stack. - `.claude/skills/change-storage-price/` — activating a non-zero postage price on the local chain. - Prior art: PR #581 (`radiusdecrease`), PR #591 (`stampexpiry`), `pkg/check/gc`, `pkg/check/load`, `pkg/check/smoke`. diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go index 4b2078fe..e857b24d 100644 --- a/pkg/check/reserveradius/reserveradius.go +++ b/pkg/check/reserveradius/reserveradius.go @@ -275,7 +275,13 @@ func (c *Check) diluteToMinValidity(ctx context.Context, b batchRef, o Options) c.logger.Warningf("read batch %s on %s: %v; stopping dilution", b.batchID, b.node.Name(), err) return } - if !stamp.Exists || stamp.BatchTTL <= 0 { + if stamp.BatchTTL < 0 { + // batchTTL -1 means "never expires" — the chain postage price is 0, so there is no + // expiry to hasten. Skip (the local idle tick still drives the decrease). + c.logger.Warningf("batch %s on %s has infinite TTL (chain price is 0) — nothing to hasten; skipping dilution", b.batchID, b.node.Name()) + return + } + if !stamp.Exists || stamp.BatchTTL == 0 { c.logger.Infof("batch %s on %s already expired (ttl=%ds)", b.batchID, b.node.Name(), stamp.BatchTTL) return } From 487f2bad7188c235529b9b19bc1c4ab296149b13 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Thu, 16 Jul 2026 12:35:34 +0200 Subject: [PATCH 30/37] feat: add reserve-radius-v2 check (acud suggestions) --- pkg/check/reserveradiusv2/README.md | 67 ++++ pkg/check/reserveradiusv2/metrics.go | 61 +++ pkg/check/reserveradiusv2/reserveradiusv2.go | 377 +++++++++++++++++++ pkg/config/check.go | 36 ++ 4 files changed, 541 insertions(+) create mode 100644 pkg/check/reserveradiusv2/README.md create mode 100644 pkg/check/reserveradiusv2/metrics.go create mode 100644 pkg/check/reserveradiusv2/reserveradiusv2.go diff --git a/pkg/check/reserveradiusv2/README.md b/pkg/check/reserveradiusv2/README.md new file mode 100644 index 00000000..e90c691d --- /dev/null +++ b/pkg/check/reserveradiusv2/README.md @@ -0,0 +1,67 @@ +# reserve-radius-v2 check + +A **simplified, reproducible** reserve-radius scenario built to capture pull-sync issues +around storage-radius changes and postage-batch dilution. + +Differences from v1 (`reserve-radius`): + +- **Granular fill control** — instead of one large depth-22 batch per node, it creates + **many small batches** (default depth 18) and uploads a **fixed amount of data per batch** + (default 4 MiB ≈ 1024 chunks), so the reserve fill is known and reproducible. +- **Capacity-threshold targeting** — it fills until any node's `reserveSizeWithinRadius` + is **just over 50%** of the configured reserve capacity (default target 55% of 4000 + chunks), the trigger point for the radius decrease. +- **Dilutions as distinct events** — after pull-sync settles, batches are diluted **one at + a time** (default one per minute), each incrementing a plain `dilutions_total` counter so + a Grafana time series shows exactly when dilutions occur relative to radius changes. + +## Flow + +1. Wait for all observed nodes to finish warmup. +2. **Fill**: create batches round-robin across nodes (label `-`), upload + `data-per-batch` bytes under each, stop when max fill ≥ `target-fill-percent`. +3. Wait for pull-sync to go idle (`pullsyncRate <= 0.05`, best-effort). +4. **Decrease**: dilute one batch every `dilute-interval` while polling; pass on the first + `storageRadius` drop below its peak, fail after `decrease-timeout` (stuck pull-sync). + +## Options + +Defaults in `NewDefaultOptions()`; YAML keys wired in `pkg/config/check.go` under +`reserve-radius-v2`. Key ones: + +| yaml | default | purpose | +| --- | --- | --- | +| `batch-depth` | `18` | depth of each small batch | +| `blob-size` | `1048576` (1 MiB) | bytes per upload | +| `data-per-batch` | `4194304` (4 MiB) | bytes uploaded under each batch | +| `max-batches` | `50` | cap on batches in the fill phase | +| `reserve-capacity` | `4000` | reserve capacity in chunks (must match the bee patch) | +| `target-fill-percent` | `55` | stop filling when any node crosses this % of capacity | +| `dilute` | `true` | dilute batches to drive the decrease | +| `dilute-interval` | `1m` | spacing between dilutions (distinct chart events) | +| `max-dilution-rounds` | `5` | each round dilutes every batch once | +| `decrease-timeout` | `30m` | max wait for a radius decrease (raise to ~6h on testnet) | + +## Metrics + +Namespace `beekeeper`, subsystem `check_reserve_radius_v2`: + +- `…_storage_radius{node}`, `…_reserve_size{node}`, `…_reserve_within_radius{node}`, + `…_reserve_fill_percent{node}`, `…_pullsync_rate{node}` — gauges polled every `poll-interval` +- `…_radius_events_total{node,direction}` — counter of observed radius increases/decreases +- `…_batches_created_total`, `…_uploaded_bytes_total`, `…_dilutions_total` — counters +- `…_time_to_decrease_seconds` — gauge, decrease-phase start → first observed decrease + +## Requirements + +Same as v1: a **patched bee image** (`radius_reserve.patch`: capacity 4000, fast wakeup; +`radius_threshold.patch`) — on stock capacity the radius never moves. `reserve-capacity` +must match the patched value. A non-zero chain postage price is required for dilution to +have an effect (at price 0 batches have infinite TTL and dilution is skipped). + +## Running it + +```sh +./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius-v2 \ + --timeout=90m --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info +``` diff --git a/pkg/check/reserveradiusv2/metrics.go b/pkg/check/reserveradiusv2/metrics.go new file mode 100644 index 00000000..d6772f4a --- /dev/null +++ b/pkg/check/reserveradiusv2/metrics.go @@ -0,0 +1,61 @@ +package reserveradiusv2 + +import ( + m "github.com/ethersphere/beekeeper/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type metrics struct { + StorageRadius *prometheus.GaugeVec + ReserveSize *prometheus.GaugeVec + ReserveWithinRadius *prometheus.GaugeVec + ReserveFillPercent *prometheus.GaugeVec + PullsyncRate *prometheus.GaugeVec + RadiusEvents *prometheus.CounterVec + BatchesCreated prometheus.Counter + UploadedBytes prometheus.Counter + Dilutions prometheus.Counter + TimeToDecrease prometheus.Gauge +} + +func newMetrics(subsystem string) metrics { + return metrics{ + StorageRadius: nodeGauge(subsystem, "storage_radius", "Storage radius reported by /status, per node."), + ReserveSize: nodeGauge(subsystem, "reserve_size", "Reserve size (chunks) reported by /status, per node."), + ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node."), + ReserveFillPercent: nodeGauge(subsystem, "reserve_fill_percent", "Reserve fill as percent of configured capacity, per node."), + PullsyncRate: nodeGauge(subsystem, "pullsync_rate", "Pull-sync rate reported by /status, per node."), + RadiusEvents: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "radius_events_total", Help: "Storage radius changes observed, per node and direction (increase/decrease).", + }, []string{"node", "direction"}), + BatchesCreated: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "batches_created_total", Help: "Postage batches created during the fill phase.", + }), + UploadedBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "uploaded_bytes_total", Help: "Bytes uploaded during the fill phase.", + }), + Dilutions: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "dilutions_total", Help: "Postage-batch dilutions applied during the decrease phase.", + }), + TimeToDecrease: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "time_to_decrease_seconds", Help: "Seconds from the start of the decrease phase to the first observed radius decrease.", + }), + } +} + +func nodeGauge(subsystem, name, help string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{Namespace: m.Namespace, Subsystem: subsystem, Name: name, Help: help}, + []string{"node"}, + ) +} + +// Report implements the metrics.Reporter interface. +func (c *Check) Report() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(c.metrics) +} diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go new file mode 100644 index 00000000..2ecde9fb --- /dev/null +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -0,0 +1,377 @@ +// Package reserveradiusv2 is a simplified, reproducible reserve-radius scenario built to +// capture pull-sync issues around radius changes and batch dilution. Instead of blindly +// uploading until a target radius (v1), it fills the reserve with a known amount of data +// per small postage batch until just over the 50% capacity threshold, then dilutes the +// batches one at a time and verifies the storage radius decreases while pull-sync resyncs. +package reserveradiusv2 + +import ( + "context" + crand "crypto/rand" + "errors" + "fmt" + "time" + + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/bee/api" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + "github.com/ethersphere/beekeeper/pkg/random" + "github.com/ethersphere/beekeeper/pkg/test" +) + +// Options represents reserve-radius-v2 check options. +type Options struct { + RndSeed int64 + PostageTTL time.Duration + BatchDepth uint64 // depth of each small batch + PostageLabel string // label prefix; each batch gets label- + UploadGroups []string // node groups to upload to / observe (empty = all full nodes) + BlobSize int64 // bytes per upload + DataPerBatch int64 // bytes uploaded under each batch + MaxBatches int // cap on batches during the fill phase + ReserveCapacity uint64 // reserve capacity in chunks (patched bee: 4000) + TargetFillPercent float64 // stop filling when any node's reserve reaches this % of capacity + Dilute bool // dilute batches after the fill to trigger the decrease + DiluteStep uint64 // depth increase per dilution + DiluteInterval time.Duration // spacing between dilutions so they show as distinct events + MaxDilutionRounds int // cap on dilution rounds (each round dilutes every batch once) + GasPrice string + WarmupWait time.Duration + FillTimeout time.Duration + SyncSettleWait time.Duration // after filling, max wait for pull-sync to go idle + DecreaseTimeout time.Duration // max time to observe a radius decrease + PollInterval time.Duration +} + +// NewDefaultOptions returns new default options. +func NewDefaultOptions() Options { + return Options{ + RndSeed: time.Now().UnixNano(), + PostageTTL: 48 * time.Hour, + BatchDepth: 18, + PostageLabel: "reserve-radius-v2", + UploadGroups: []string{"bee"}, + BlobSize: 1 << 20, // 1 MiB + DataPerBatch: 4 << 20, // 4 MiB = ~1024 chunks per batch + MaxBatches: 50, + ReserveCapacity: 4000, // patched bee reserve; 50% = 2000 chunks + TargetFillPercent: 55, // just over the 50% decrease threshold + Dilute: true, + DiluteStep: 1, + DiluteInterval: time.Minute, + MaxDilutionRounds: 5, + GasPrice: "", + WarmupWait: 15 * time.Minute, + FillTimeout: 30 * time.Minute, + SyncSettleWait: 10 * time.Minute, + DecreaseTimeout: 30 * time.Minute, + PollInterval: 10 * time.Second, + } +} + +// pullSyncIdleRate is the pullsyncRate at or below which pull-sync is treated as idle +// (the rate decays to a small residual, never exactly 0). +const pullSyncIdleRate = 0.05 + +var _ beekeeper.Action = (*Check)(nil) + +// Check instance. +type Check struct { + metrics metrics + logger logging.Logger +} + +// NewCheck returns a new reserve-radius-v2 check. +func NewCheck(log logging.Logger) beekeeper.Action { + return &Check{ + metrics: newMetrics("check_reserve_radius_v2"), + logger: log, + } +} + +type batchRef struct { + node *bee.Client + batchID string + done bool // reached the validity floor or errored; skip in later rounds +} + +// state tracks per-node radius history across polls. +type state struct { + peak map[string]uint8 + last map[string]uint8 +} + +// Run fills the reserve just over the 50% capacity threshold using small batches with a +// fixed amount of data each, waits for pull-sync to settle, then dilutes the batches one +// at a time and verifies the storage radius decreases. +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return errors.New("invalid options type") + } + if o.ReserveCapacity == 0 { + return errors.New("reserve-capacity must be > 0") + } + if o.TargetFillPercent <= 50 { + c.logger.Warningf("target-fill-percent %.1f is not over 50%% — the decrease threshold may already be crossed before dilution", o.TargetFillPercent) + } + + c.logger.Infof("random seed: %d", o.RndSeed) + rnd := random.PseudoGenerator(o.RndSeed) + + fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) + if err != nil { + return fmt.Errorf("get shuffled full node clients: %w", err) + } + nodes := fullNodeClients + if len(o.UploadGroups) > 0 { + nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) + } + if len(nodes) < 1 { + return fmt.Errorf("reserve-radius-v2 check requires at least 1 full node, got %d", len(nodes)) + } + c.logger.Infof("observing %d node(s); reserve capacity %d chunks, fill target %.1f%%", len(nodes), o.ReserveCapacity, o.TargetFillPercent) + + if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { + return err + } + + st := &state{peak: make(map[string]uint8), last: make(map[string]uint8)} + c.poll(ctx, nodes, st, o) + + batches, err := c.fill(ctx, nodes, st, o) + if err != nil { + return err + } + + c.waitPullSyncIdle(ctx, nodes, st, o) + + return c.driveDecrease(ctx, nodes, batches, st, o) +} + +// waitForWarmupDone blocks until every observed node reports isWarmingUp=false. +func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.ClientList, o Options) error { + c.logger.Infof("waiting up to %s for nodes to finish warmup", o.WarmupWait) + deadline := time.Now().Add(o.WarmupWait) + for { + if err := ctx.Err(); err != nil { + return err + } + ready := true + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil || s.IsWarmingUp { + ready = false + } + } + if ready { + c.logger.Info("all observed nodes finished warmup") + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("nodes still warming up after %s", o.WarmupWait) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + +// fill creates small batches round-robin across nodes and uploads exactly DataPerBatch +// bytes under each, stopping once any node's reserve crosses TargetFillPercent. +func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) ([]*batchRef, error) { + uploadsPerBatch := int((o.DataPerBatch + o.BlobSize - 1) / o.BlobSize) + c.logger.Infof("filling reserve to %.1f%% of %d chunks: %d bytes per batch (depth %d, %d x %d-byte blobs), max %d batches", + o.TargetFillPercent, o.ReserveCapacity, o.DataPerBatch, o.BatchDepth, uploadsPerBatch, o.BlobSize, o.MaxBatches) + + start := time.Now() + deadline := start.Add(o.FillTimeout) + t := test.NewTest(c.logger) + data := make([]byte, o.BlobSize) + var batches []*batchRef + + for b := 1; b <= o.MaxBatches; b++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("reserve did not reach %.1f%% within %s (%d batches) — is the bee reserve patch active?", o.TargetFillPercent, o.FillTimeout, b-1) + } + + node := nodes[(b-1)%len(nodes)] + label := fmt.Sprintf("%s-%d", o.PostageLabel, b) + batchID, err := node.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.BatchDepth, label) + if err != nil { + c.logger.Warningf("create batch %s on %s failed, skipping: %v", label, node.Name(), err) + continue + } + batches = append(batches, &batchRef{node: node, batchID: batchID}) + c.metrics.BatchesCreated.Inc() + c.logger.WithField("batch_id", batchID).Infof("batch #%d created on %s (label %s)", b, node.Name(), label) + + for u := 0; u < uploadsPerBatch; u++ { + if _, err := crand.Read(data); err != nil { + return nil, fmt.Errorf("generate random data: %w", err) + } + if _, _, err := t.Upload(ctx, node, data, batchID, nil); err != nil { + c.logger.Errorf("upload %d/%d to %s failed: %v", u+1, uploadsPerBatch, node.Name(), err) + continue + } + c.metrics.UploadedBytes.Add(float64(o.BlobSize)) + } + + fill, _ := c.poll(ctx, nodes, st, o) + c.logger.Infof("fill: batch #%d done (%d bytes), max reserve fill %.1f%% (target %.1f%%)", b, o.DataPerBatch, fill, o.TargetFillPercent) + if fill >= o.TargetFillPercent { + c.logger.Infof("reached %.1f%% fill with %d batches (~%.1f MiB) in %s", + fill, len(batches), float64(int64(len(batches))*o.DataPerBatch)/(1<<20), time.Since(start).Round(time.Second)) + return batches, nil + } + } + return nil, fmt.Errorf("reserve did not reach %.1f%% after %d batches", o.TargetFillPercent, o.MaxBatches) +} + +// waitPullSyncIdle blocks until every node reports pull-sync idle or SyncSettleWait +// elapses (best-effort: on timeout it logs and proceeds). +func (c *Check) waitPullSyncIdle(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) { + c.logger.Infof("uploads stopped; waiting up to %s for pull-sync to go idle (pullsyncRate<=%.2f on all nodes)", o.SyncSettleWait, pullSyncIdleRate) + deadline := time.Now().Add(o.SyncSettleWait) + for { + _, maxRate := c.poll(ctx, nodes, st, o) + if maxRate <= pullSyncIdleRate { + c.logger.Info("pull-sync idle on all nodes") + return + } + if time.Now().After(deadline) { + c.logger.Warningf("pull-sync did not go idle within %s (max pullsyncRate=%.4f); proceeding anyway", o.SyncSettleWait, maxRate) + return + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return + } + } +} + +// driveDecrease dilutes the batches one at a time (one dilution per DiluteInterval) while +// watching for any node's storageRadius to fall below its peak. Interleaving dilutions +// with polling makes each dilution a distinct event next to the radius/pull-sync series. +func (c *Check) driveDecrease(ctx context.Context, nodes orchestration.ClientList, batches []*batchRef, st *state, o Options) error { + c.logger.Infof("watching for a radius decrease below peak %v (timeout %s, dilute=%t)", st.peak, o.DecreaseTimeout, o.Dilute) + start := time.Now() + deadline := start.Add(o.DecreaseTimeout) + nextDilute := time.Now() + dilutions, maxDilutions := 0, len(batches)*o.MaxDilutionRounds + + for { + if err := ctx.Err(); err != nil { + return err + } + c.poll(ctx, nodes, st, o) + for _, n := range nodes { + if st.last[n.Name()] < st.peak[n.Name()] { + c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) + c.logger.Infof("radius decrease observed on %s (%d -> %d) after %s and %d dilution(s) — done", + n.Name(), st.peak[n.Name()], st.last[n.Name()], time.Since(start).Round(time.Second), dilutions) + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("no radius decrease below peak %v within %s (%d dilutions applied) — possible stuck pull-sync", st.peak, o.DecreaseTimeout, dilutions) + } + if o.Dilute && dilutions < maxDilutions && time.Now().After(nextDilute) { + if c.diluteNext(ctx, batches, dilutions, o) { + dilutions++ + } + nextDilute = time.Now().Add(o.DiluteInterval) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + +// diluteNext dilutes the next not-done batch (round-robin) by DiluteStep and reports +// whether a dilution was applied. A rejected dilution (validity floor) marks the batch done. +func (c *Check) diluteNext(ctx context.Context, batches []*batchRef, round int, o Options) bool { + for i := 0; i < len(batches); i++ { + b := batches[(round+i)%len(batches)] + if b.done { + continue + } + stamp, err := b.node.PostageStamp(ctx, b.batchID) + if err != nil { + c.logger.Warningf("read batch %s on %s: %v; skipping", b.batchID, b.node.Name(), err) + b.done = true + continue + } + if stamp.BatchTTL < 0 { + c.logger.Warningf("batch %s has infinite TTL (chain price 0) — dilution cannot hasten expiry; skipping", b.batchID) + b.done = true + continue + } + newDepth := uint64(stamp.Depth) + o.DiluteStep + if err := b.node.DilutePostageBatch(ctx, b.batchID, newDepth, o.GasPrice); err != nil { + c.logger.Infof("batch %s at validity floor (ttl ~%ds); dilute to depth %d rejected: %v", b.batchID, stamp.BatchTTL, newDepth, err) + b.done = true + continue + } + c.metrics.Dilutions.Inc() + c.logger.Infof("diluted batch %s on %s to depth %d (ttl was %ds)", b.batchID, b.node.Name(), newDepth, stamp.BatchTTL) + return true + } + return false +} + +// poll reads /status on every node, emits metrics, updates peak/last radius, and returns +// the max reserve fill percent and max pullsyncRate across nodes. +func (c *Check) poll(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) (maxFill, maxRate float64) { + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + c.logger.Debugf("%s: status error: %v", n.Name(), err) + continue + } + name := n.Name() + fill := float64(s.ReserveSizeWithinRadius) / float64(o.ReserveCapacity) * 100 + c.emit(name, s, fill) + + if prev, seen := st.last[name]; seen && s.StorageRadius != prev { + dir := "increase" + if s.StorageRadius < prev { + dir = "decrease" + } + c.metrics.RadiusEvents.WithLabelValues(name, dir).Inc() + c.logger.Infof("%s: storageRadius %d -> %d (%s), fill %.1f%%, pullsyncRate %.4f", name, prev, s.StorageRadius, dir, fill, s.PullsyncRate) + } + st.last[name] = s.StorageRadius + if s.StorageRadius > st.peak[name] { + st.peak[name] = s.StorageRadius + } + if fill > maxFill { + maxFill = fill + } + if s.PullsyncRate > maxRate { + maxRate = s.PullsyncRate + } + } + return maxFill, maxRate +} + +func (c *Check) emit(node string, s *api.StatusResponse, fill float64) { + c.metrics.StorageRadius.WithLabelValues(node).Set(float64(s.StorageRadius)) + c.metrics.ReserveSize.WithLabelValues(node).Set(float64(s.ReserveSize)) + c.metrics.ReserveWithinRadius.WithLabelValues(node).Set(float64(s.ReserveSizeWithinRadius)) + c.metrics.ReserveFillPercent.WithLabelValues(node).Set(fill) + c.metrics.PullsyncRate.WithLabelValues(node).Set(s.PullsyncRate) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} diff --git a/pkg/config/check.go b/pkg/config/check.go index 9c38e0c7..05b4acc1 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -31,6 +31,7 @@ import ( "github.com/ethersphere/beekeeper/pkg/check/pushsync" "github.com/ethersphere/beekeeper/pkg/check/redundancy" "github.com/ethersphere/beekeeper/pkg/check/reserveradius" + "github.com/ethersphere/beekeeper/pkg/check/reserveradiusv2" "github.com/ethersphere/beekeeper/pkg/check/retrieval" "github.com/ethersphere/beekeeper/pkg/check/settlements" "github.com/ethersphere/beekeeper/pkg/check/smoke" @@ -529,6 +530,41 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "reserve-radius-v2": { + NewAction: reserveradiusv2.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + RndSeed *int64 `yaml:"rnd-seed"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + BatchDepth *uint64 `yaml:"batch-depth"` + PostageLabel *string `yaml:"postage-label"` + UploadGroups *[]string `yaml:"upload-groups"` + BlobSize *int64 `yaml:"blob-size"` + DataPerBatch *int64 `yaml:"data-per-batch"` + MaxBatches *int `yaml:"max-batches"` + ReserveCapacity *uint64 `yaml:"reserve-capacity"` + TargetFillPercent *float64 `yaml:"target-fill-percent"` + Dilute *bool `yaml:"dilute"` + DiluteStep *uint64 `yaml:"dilute-step"` + DiluteInterval *time.Duration `yaml:"dilute-interval"` + MaxDilutionRounds *int `yaml:"max-dilution-rounds"` + GasPrice *string `yaml:"gas-price"` + WarmupWait *time.Duration `yaml:"warmup-wait"` + FillTimeout *time.Duration `yaml:"fill-timeout"` + SyncSettleWait *time.Duration `yaml:"sync-settle-wait"` + DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + PollInterval *time.Duration `yaml:"poll-interval"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := reserveradiusv2.NewDefaultOptions() + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, "soc": { NewAction: soc.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { From 97ea0ef9d66dcbd2899cb0b1946efbb705b86ad7 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Thu, 16 Jul 2026 12:42:13 +0200 Subject: [PATCH 31/37] refactor: remove waitForWarmupDone method from Run function --- pkg/check/reserveradiusv2/reserveradiusv2.go | 30 -------------------- 1 file changed, 30 deletions(-) diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index 2ecde9fb..80101a93 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -134,9 +134,6 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any } c.logger.Infof("observing %d node(s); reserve capacity %d chunks, fill target %.1f%%", len(nodes), o.ReserveCapacity, o.TargetFillPercent) - if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { - return err - } st := &state{peak: make(map[string]uint8), last: make(map[string]uint8)} c.poll(ctx, nodes, st, o) @@ -151,33 +148,6 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return c.driveDecrease(ctx, nodes, batches, st, o) } -// waitForWarmupDone blocks until every observed node reports isWarmingUp=false. -func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.ClientList, o Options) error { - c.logger.Infof("waiting up to %s for nodes to finish warmup", o.WarmupWait) - deadline := time.Now().Add(o.WarmupWait) - for { - if err := ctx.Err(); err != nil { - return err - } - ready := true - for _, n := range nodes { - s, err := n.Status(ctx) - if err != nil || s.IsWarmingUp { - ready = false - } - } - if ready { - c.logger.Info("all observed nodes finished warmup") - return nil - } - if time.Now().After(deadline) { - return fmt.Errorf("nodes still warming up after %s", o.WarmupWait) - } - if err := sleepCtx(ctx, o.PollInterval); err != nil { - return err - } - } -} // fill creates small batches round-robin across nodes and uploads exactly DataPerBatch // bytes under each, stopping once any node's reserve crosses TargetFillPercent. From 0b5cab40b478e96d68a9f177dd907d22e34a09b8 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Thu, 16 Jul 2026 12:48:35 +0200 Subject: [PATCH 32/37] feat: add postage-amount option for batch configuration --- pkg/check/reserveradiusv2/README.md | 1 + pkg/check/reserveradiusv2/reserveradiusv2.go | 8 +++----- pkg/config/check.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/check/reserveradiusv2/README.md b/pkg/check/reserveradiusv2/README.md index e90c691d..3a8660ae 100644 --- a/pkg/check/reserveradiusv2/README.md +++ b/pkg/check/reserveradiusv2/README.md @@ -31,6 +31,7 @@ Defaults in `NewDefaultOptions()`; YAML keys wired in `pkg/config/check.go` unde | yaml | default | purpose | | --- | --- | --- | +| `postage-amount` | `1000` | amount per batch (each batch is created fresh via `CreatePostageBatch`, never reused) | | `batch-depth` | `18` | depth of each small batch | | `blob-size` | `1048576` (1 MiB) | bytes per upload | | `data-per-batch` | `4194304` (4 MiB) | bytes uploaded under each batch | diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index 80101a93..75dcd707 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -24,7 +24,7 @@ import ( // Options represents reserve-radius-v2 check options. type Options struct { RndSeed int64 - PostageTTL time.Duration + PostageAmount int64 // amount per batch (each batch is created fresh, never reused) BatchDepth uint64 // depth of each small batch PostageLabel string // label prefix; each batch gets label- UploadGroups []string // node groups to upload to / observe (empty = all full nodes) @@ -49,7 +49,7 @@ type Options struct { func NewDefaultOptions() Options { return Options{ RndSeed: time.Now().UnixNano(), - PostageTTL: 48 * time.Hour, + PostageAmount: 1000, BatchDepth: 18, PostageLabel: "reserve-radius-v2", UploadGroups: []string{"bee"}, @@ -134,7 +134,6 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any } c.logger.Infof("observing %d node(s); reserve capacity %d chunks, fill target %.1f%%", len(nodes), o.ReserveCapacity, o.TargetFillPercent) - st := &state{peak: make(map[string]uint8), last: make(map[string]uint8)} c.poll(ctx, nodes, st, o) @@ -148,7 +147,6 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any return c.driveDecrease(ctx, nodes, batches, st, o) } - // fill creates small batches round-robin across nodes and uploads exactly DataPerBatch // bytes under each, stopping once any node's reserve crosses TargetFillPercent. func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) ([]*batchRef, error) { @@ -172,7 +170,7 @@ func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *st node := nodes[(b-1)%len(nodes)] label := fmt.Sprintf("%s-%d", o.PostageLabel, b) - batchID, err := node.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.BatchDepth, label) + batchID, err := node.CreatePostageBatch(ctx, o.PostageAmount, o.BatchDepth, label, false) if err != nil { c.logger.Warningf("create batch %s on %s failed, skipping: %v", label, node.Name(), err) continue diff --git a/pkg/config/check.go b/pkg/config/check.go index 05b4acc1..fca43df9 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -535,7 +535,7 @@ var Checks = map[string]CheckType{ NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { checkOpts := new(struct { RndSeed *int64 `yaml:"rnd-seed"` - PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageAmount *int64 `yaml:"postage-amount"` BatchDepth *uint64 `yaml:"batch-depth"` PostageLabel *string `yaml:"postage-label"` UploadGroups *[]string `yaml:"upload-groups"` From 04d600fa57933062a95eeea916b68778161640c1 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:05:25 -0600 Subject: [PATCH 33/37] simplify check, fill only for now, manual batch dilution needed --- pkg/check/reserveradiusv2/reserveradiusv2.go | 57 ++++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index 75dcd707..59e8b29a 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -10,6 +10,7 @@ import ( crand "crypto/rand" "errors" "fmt" + "math" "time" "github.com/ethersphere/beekeeper/pkg/bee" @@ -57,7 +58,7 @@ func NewDefaultOptions() Options { DataPerBatch: 4 << 20, // 4 MiB = ~1024 chunks per batch MaxBatches: 50, ReserveCapacity: 4000, // patched bee reserve; 50% = 2000 chunks - TargetFillPercent: 55, // just over the 50% decrease threshold + TargetFillPercent: 60, // just over the 50% decrease threshold Dilute: true, DiluteStep: 1, DiluteInterval: time.Minute, @@ -137,30 +138,42 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any st := &state{peak: make(map[string]uint8), last: make(map[string]uint8)} c.poll(ctx, nodes, st, o) - batches, err := c.fill(ctx, nodes, st, o) + _, err = c.fill(ctx, nodes, st, o) if err != nil { return err } + c.logger.Infof("radius check fill done. waiting for pull sync to go idle") + c.waitPullSyncIdle(ctx, nodes, st, o) - return c.driveDecrease(ctx, nodes, batches, st, o) + c.logger.Infof("radius check fill done. pull sync idle, driving decrease") + + // return c.driveDecrease(ctx, nodes, batches, st, o) + return nil } // fill creates small batches round-robin across nodes and uploads exactly DataPerBatch // bytes under each, stopping once any node's reserve crosses TargetFillPercent. func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) ([]*batchRef, error) { - uploadsPerBatch := int((o.DataPerBatch + o.BlobSize - 1) / o.BlobSize) - c.logger.Infof("filling reserve to %.1f%% of %d chunks: %d bytes per batch (depth %d, %d x %d-byte blobs), max %d batches", - o.TargetFillPercent, o.ReserveCapacity, o.DataPerBatch, o.BatchDepth, uploadsPerBatch, o.BlobSize, o.MaxBatches) + clusterSize := len(nodes) + neighborhoods := math.Log2(float64(clusterSize)) + nodeReserve := float64(4000) + totalStorageToFill := o.TargetFillPercent * nodeReserve * neighborhoods + stamps := 5 + sizePerStamp := int(totalStorageToFill / float64(stamps)) + + c.logger.Infof("fill: clusterSize %d, neighborhoods %d, nodeReserve %d, totalStorageToFill %d, stamps %d,sizePerStamp(chunks) %d", clusterSize, neighborhoods, nodeReserve, totalStorageToFill, stamps, sizePerStamp) start := time.Now() deadline := start.Add(o.FillTimeout) t := test.NewTest(c.logger) - data := make([]byte, o.BlobSize) + data := make([]byte, sizePerStamp*4096) var batches []*batchRef - for b := 1; b <= o.MaxBatches; b++ { + // caveat: if there's an error, this won't work very well because + // the errors have a continue statement + for b := 0; b < stamps; b++ { if err := ctx.Err(); err != nil { return nil, err } @@ -179,16 +192,14 @@ func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *st c.metrics.BatchesCreated.Inc() c.logger.WithField("batch_id", batchID).Infof("batch #%d created on %s (label %s)", b, node.Name(), label) - for u := 0; u < uploadsPerBatch; u++ { - if _, err := crand.Read(data); err != nil { - return nil, fmt.Errorf("generate random data: %w", err) - } - if _, _, err := t.Upload(ctx, node, data, batchID, nil); err != nil { - c.logger.Errorf("upload %d/%d to %s failed: %v", u+1, uploadsPerBatch, node.Name(), err) - continue - } - c.metrics.UploadedBytes.Add(float64(o.BlobSize)) + if _, err := crand.Read(data); err != nil { + return nil, fmt.Errorf("generate random data: %w", err) + } + if _, _, err := t.Upload(ctx, node, data, batchID, nil); err != nil { + c.logger.Errorf("upload to %s failed: %v", node.Name(), err) + continue } + c.metrics.UploadedBytes.Add(float64(o.BlobSize)) fill, _ := c.poll(ctx, nodes, st, o) c.logger.Infof("fill: batch #%d done (%d bytes), max reserve fill %.1f%% (target %.1f%%)", b, o.DataPerBatch, fill, o.TargetFillPercent) @@ -225,8 +236,8 @@ func (c *Check) waitPullSyncIdle(ctx context.Context, nodes orchestration.Client // driveDecrease dilutes the batches one at a time (one dilution per DiluteInterval) while // watching for any node's storageRadius to fall below its peak. Interleaving dilutions // with polling makes each dilution a distinct event next to the radius/pull-sync series. -func (c *Check) driveDecrease(ctx context.Context, nodes orchestration.ClientList, batches []*batchRef, st *state, o Options) error { - c.logger.Infof("watching for a radius decrease below peak %v (timeout %s, dilute=%t)", st.peak, o.DecreaseTimeout, o.Dilute) +func (c *Check) driveDecrease(ctx context.Context, nodes orchestration.ClientList, batches []*batchRef, radiusState *state, o Options) error { + c.logger.Infof("watching for a radius decrease below peak %v (timeout %s, dilute=%t)", radiusState.peak, o.DecreaseTimeout, o.Dilute) start := time.Now() deadline := start.Add(o.DecreaseTimeout) nextDilute := time.Now() @@ -236,17 +247,17 @@ func (c *Check) driveDecrease(ctx context.Context, nodes orchestration.ClientLis if err := ctx.Err(); err != nil { return err } - c.poll(ctx, nodes, st, o) + c.poll(ctx, nodes, radiusState, o) for _, n := range nodes { - if st.last[n.Name()] < st.peak[n.Name()] { + if radiusState.last[n.Name()] < radiusState.peak[n.Name()] { c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) c.logger.Infof("radius decrease observed on %s (%d -> %d) after %s and %d dilution(s) — done", - n.Name(), st.peak[n.Name()], st.last[n.Name()], time.Since(start).Round(time.Second), dilutions) + n.Name(), radiusState.peak[n.Name()], radiusState.last[n.Name()], time.Since(start).Round(time.Second), dilutions) return nil } } if time.Now().After(deadline) { - return fmt.Errorf("no radius decrease below peak %v within %s (%d dilutions applied) — possible stuck pull-sync", st.peak, o.DecreaseTimeout, dilutions) + return fmt.Errorf("no radius decrease below peak %v within %s (%d dilutions applied) — possible stuck pull-sync", radiusState.peak, o.DecreaseTimeout, dilutions) } if o.Dilute && dilutions < maxDilutions && time.Now().After(nextDilute) { if c.diluteNext(ctx, batches, dilutions, o) { From 0cafbaadae2a7c917115455ec3e228eadaf5b3ff Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:35:53 -0600 Subject: [PATCH 34/37] percentage --- pkg/check/reserveradiusv2/reserveradiusv2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index 59e8b29a..ddfb3358 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -58,7 +58,7 @@ func NewDefaultOptions() Options { DataPerBatch: 4 << 20, // 4 MiB = ~1024 chunks per batch MaxBatches: 50, ReserveCapacity: 4000, // patched bee reserve; 50% = 2000 chunks - TargetFillPercent: 60, // just over the 50% decrease threshold + TargetFillPercent: 0.6, // just over the 50% decrease threshold Dilute: true, DiluteStep: 1, DiluteInterval: time.Minute, From 8bede51d2866815f3557f9717c71ee818578e426 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:58:21 -0600 Subject: [PATCH 35/37] percentage fix --- pkg/check/reserveradiusv2/reserveradiusv2.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index ddfb3358..775308e2 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -144,10 +144,12 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any } c.logger.Infof("radius check fill done. waiting for pull sync to go idle") + c.poll(ctx, nodes, st, o) c.waitPullSyncIdle(ctx, nodes, st, o) c.logger.Infof("radius check fill done. pull sync idle, driving decrease") + c.poll(ctx, nodes, st, o) // return c.driveDecrease(ctx, nodes, batches, st, o) return nil @@ -181,7 +183,7 @@ func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *st return nil, fmt.Errorf("reserve did not reach %.1f%% within %s (%d batches) — is the bee reserve patch active?", o.TargetFillPercent, o.FillTimeout, b-1) } - node := nodes[(b-1)%len(nodes)] + node := nodes[b] label := fmt.Sprintf("%s-%d", o.PostageLabel, b) batchID, err := node.CreatePostageBatch(ctx, o.PostageAmount, o.BatchDepth, label, false) if err != nil { @@ -203,7 +205,7 @@ func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *st fill, _ := c.poll(ctx, nodes, st, o) c.logger.Infof("fill: batch #%d done (%d bytes), max reserve fill %.1f%% (target %.1f%%)", b, o.DataPerBatch, fill, o.TargetFillPercent) - if fill >= o.TargetFillPercent { + if fill >= (o.TargetFillPercent * 100) { c.logger.Infof("reached %.1f%% fill with %d batches (~%.1f MiB) in %s", fill, len(batches), float64(int64(len(batches))*o.DataPerBatch)/(1<<20), time.Since(start).Round(time.Second)) return batches, nil From 054ec7d85103ab7bfc5a00bfa07d577671c9a08b Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 21 Jul 2026 11:53:33 +0200 Subject: [PATCH 36/37] feat: update reserve-radius configuration and enhance fill logic --- config/local.yaml | 31 ++++++++++++++++++-- pkg/check/reserveradiusv2/reserveradiusv2.go | 29 +++++++++++------- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/config/local.yaml b/config/local.yaml index 7e661679..ff0faf56 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -40,7 +40,7 @@ clusters: bee: bee-config: bee-local-dns config: local-dns - count: 6 + count: 8 mode: node light: bee-config: bee-local-light @@ -101,7 +101,7 @@ clusters: node-groups: local: _inherit: "" - image: k3d-registry.localhost:5000/ethersphere/bee:latest + image: k3d-registry.localhost:5002/ethersphere/bee:latest image-pull-policy: Always ingress-annotations: nginx.ingress.kubernetes.io/affinity: "cookie" @@ -402,6 +402,33 @@ checks: - bee timeout: 45m type: reserve-radius + # Simplified reserve-radius scenario (v2, WIP): fill the reserve with small, fixed-size + # postage batches until fill-percent crosses target-fill-percent, wait for pull-sync to + # settle. NOTE: as of this writing fill()/driveDecrease in reserveradiusv2.go are under + # active rework — keep this config in sync with pkg/check/reserveradiusv2/reserveradiusv2.go. + ci-reserve-radius-v2: + options: + postage-amount: 1000000000 + batch-depth: 18 + postage-label: reserve-radius-v2 + blob-size: 1048576 # 1 MiB per upload + data-per-batch: 4194304 # 4 MiB = ~1024 chunks per batch — granular fill control + max-batches: 50 + reserve-capacity: 4000 # patched bee reserve capacity in chunks + target-fill-percent: 0.6 + dilute: true + dilute-step: 1 + dilute-interval: 1m # one dilution per minute — distinct events next to radius changes + max-dilution-rounds: 5 + warmup-wait: 15m + fill-timeout: 30m + sync-settle-wait: 10m + decrease-timeout: 30m + poll-interval: 10s + upload-groups: + - bee + timeout: 90m + type: reserve-radius-v2 ci-soc: options: postage-ttl: 24h diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index 775308e2..8c68d214 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -10,7 +10,6 @@ import ( crand "crypto/rand" "errors" "fmt" - "math" "time" "github.com/ethersphere/beekeeper/pkg/bee" @@ -138,6 +137,11 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any st := &state{peak: make(map[string]uint8), last: make(map[string]uint8)} c.poll(ctx, nodes, st, o) + + // for { + // c.poll(ctx, nodes, st, o) + // time.Sleep(5 * time.Second) + // } _, err = c.fill(ctx, nodes, st, o) if err != nil { return err @@ -159,13 +163,12 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any // bytes under each, stopping once any node's reserve crosses TargetFillPercent. func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) ([]*batchRef, error) { clusterSize := len(nodes) - neighborhoods := math.Log2(float64(clusterSize)) - nodeReserve := float64(4000) - totalStorageToFill := o.TargetFillPercent * nodeReserve * neighborhoods stamps := 5 - sizePerStamp := int(totalStorageToFill / float64(stamps)) + // each stamp alone must push a single node over the target, since poll() reports the + // max fill across nodes and every stamp's data lands on just one node. + sizePerStamp := int(o.TargetFillPercent * float64(o.ReserveCapacity)) - c.logger.Infof("fill: clusterSize %d, neighborhoods %d, nodeReserve %d, totalStorageToFill %d, stamps %d,sizePerStamp(chunks) %d", clusterSize, neighborhoods, nodeReserve, totalStorageToFill, stamps, sizePerStamp) + c.logger.Infof("fill: clusterSize %d, reserveCapacity %d, stamps %d, sizePerStamp(chunks) %d", clusterSize, o.ReserveCapacity, stamps, sizePerStamp) start := time.Now() deadline := start.Add(o.FillTimeout) @@ -203,15 +206,19 @@ func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *st } c.metrics.UploadedBytes.Add(float64(o.BlobSize)) - fill, _ := c.poll(ctx, nodes, st, o) - c.logger.Infof("fill: batch #%d done (%d bytes), max reserve fill %.1f%% (target %.1f%%)", b, o.DataPerBatch, fill, o.TargetFillPercent) - if fill >= (o.TargetFillPercent * 100) { + // give the node time to account the upload into its reserve before polling status, + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return nil, err + } + maxFill, _ := c.poll(ctx, nodes, st, o) + c.logger.Infof("fill: batch #%d done (%d bytes), max reserve fill %.1f%% (target %.1f%%)", b, o.DataPerBatch, maxFill, o.TargetFillPercent) + if maxFill >= (o.TargetFillPercent * 100) { c.logger.Infof("reached %.1f%% fill with %d batches (~%.1f MiB) in %s", - fill, len(batches), float64(int64(len(batches))*o.DataPerBatch)/(1<<20), time.Since(start).Round(time.Second)) + maxFill, len(batches), float64(int64(len(batches))*o.DataPerBatch)/(1<<20), time.Since(start).Round(time.Second)) return batches, nil } } - return nil, fmt.Errorf("reserve did not reach %.1f%% after %d batches", o.TargetFillPercent, o.MaxBatches) + return nil, fmt.Errorf("reserve did not reach %.1f%% after %d batches", o.TargetFillPercent, stamps) } // waitPullSyncIdle blocks until every node reports pull-sync idle or SyncSettleWait From d22e00186daefe72161016b81d9a3c4b6f85b3b5 Mon Sep 17 00:00:00 2001 From: akrem-chabchoub Date: Tue, 21 Jul 2026 13:48:47 +0200 Subject: [PATCH 37/37] feat: add Neighborhoods API for per-neighborhood reserve stats and update polling logic --- pkg/bee/api/status.go | 18 +++++++++++++++ pkg/bee/client.go | 6 +++++ pkg/check/reserveradiusv2/reserveradiusv2.go | 23 ++++++++++++++++---- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/pkg/bee/api/status.go b/pkg/bee/api/status.go index de0028f0..f5cfd03d 100644 --- a/pkg/bee/api/status.go +++ b/pkg/bee/api/status.go @@ -30,3 +30,21 @@ func (s *StatusService) Status(ctx context.Context) (resp *StatusResponse, err e err = s.client.requestJSON(ctx, http.MethodGet, "/status", nil, &resp) return resp, err } + +type NeighborhoodStat struct { + Neighborhood string `json:"neighborhood"` + ReserveSizeWithinRadius int `json:"reserveSizeWithinRadius"` + Proximity uint8 `json:"proximity"` +} + +type NeighborhoodsResponse struct { + Neighborhoods []NeighborhoodStat `json:"neighborhoods"` +} + +// Neighborhoods returns per-neighborhood reserve stats, computed synchronously (uncached) +// on every call — unlike Status's ReserveSizeWithinRadius, which is refreshed on a fixed +// interval and can lag a just-written chunk by that interval. +func (s *StatusService) Neighborhoods(ctx context.Context) (resp *NeighborhoodsResponse, err error) { + err = s.client.requestJSON(ctx, http.MethodGet, "/status/neighborhoods", nil, &resp) + return resp, err +} diff --git a/pkg/bee/client.go b/pkg/bee/client.go index a39b1121..83c3febe 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -1090,6 +1090,12 @@ func (c *Client) Status(ctx context.Context) (*api.StatusResponse, error) { return c.api.Status.Status(ctx) } +// Neighborhoods returns per-neighborhood reserve stats, computed synchronously (uncached) +// on every call — use this instead of Status's ReserveSizeWithinRadius when freshness matters. +func (c *Client) Neighborhoods(ctx context.Context) (*api.NeighborhoodsResponse, error) { + return c.api.Status.Neighborhoods(ctx) +} + // RedistributionState returns the node's redistribution-game state (full-mode only). func (c *Client) RedistributionState(ctx context.Context) (*api.RedistributionState, error) { return c.api.Redistribution.RedistributionState(ctx) diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go index 8c68d214..7f6c4ff4 100644 --- a/pkg/check/reserveradiusv2/reserveradiusv2.go +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -206,7 +206,9 @@ func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *st } c.metrics.UploadedBytes.Add(float64(o.BlobSize)) - // give the node time to account the upload into its reserve before polling status, + // brief buffer for the upload to land in the local reserve before scanning it; + // poll() reads /status/neighborhoods, which is computed fresh on every call, so + // this is just upload-commit latency, not a wait on any cached/ticked stat. if err := sleepCtx(ctx, o.PollInterval); err != nil { return nil, err } @@ -312,8 +314,11 @@ func (c *Check) diluteNext(ctx context.Context, batches []*batchRef, round int, return false } -// poll reads /status on every node, emits metrics, updates peak/last radius, and returns -// the max reserve fill percent and max pullsyncRate across nodes. +// poll reads /status and /status/neighborhoods on every node, emits metrics, updates +// peak/last radius, and returns the max reserve fill percent and max pullsyncRate across +// nodes. Fill is computed from /status/neighborhoods (summed across neighborhoods) rather +// than /status's ReserveSizeWithinRadius, which bee only refreshes on a fixed interval +// (15 minutes in production) and can lag a just-written chunk by that long. func (c *Check) poll(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) (maxFill, maxRate float64) { for _, n := range nodes { s, err := n.Status(ctx) @@ -322,7 +327,17 @@ func (c *Check) poll(ctx context.Context, nodes orchestration.ClientList, st *st continue } name := n.Name() - fill := float64(s.ReserveSizeWithinRadius) / float64(o.ReserveCapacity) * 100 + + nh, err := n.Neighborhoods(ctx) + if err != nil { + c.logger.Debugf("%s: neighborhoods error: %v", n.Name(), err) + continue + } + var reserveSize uint64 + for _, stat := range nh.Neighborhoods { + reserveSize += uint64(stat.ReserveSizeWithinRadius) + } + fill := float64(reserveSize) / float64(o.ReserveCapacity) * 100 c.emit(name, s, fill) if prev, seen := st.last[name]; seen && s.StorageRadius != prev {