diff --git a/cmd/beekeeper/cmd/check.go b/cmd/beekeeper/cmd/check.go index 6cc818f5..d91df1aa 100644 --- a/cmd/beekeeper/cmd/check.go +++ b/cmd/beekeeper/cmd/check.go @@ -22,6 +22,7 @@ func (c *command) initCheckCmd() error { optionNameSeed = "seed" optionNameTimeout = "timeout" optionNameMetricsPusherAddress = "metrics-pusher-address" + optionNameParallel = "parallel" ) cmd := &cobra.Command{ @@ -103,7 +104,7 @@ Use --metrics-enabled to collect and push metrics to Prometheus.`, checkRunner := check.NewCheckRunner(checkGlobalConfig, c.config.Checks, cluster, metricsPusher, tracer, c.log) - return checkRunner.Run(ctx, checks) + return checkRunner.Run(ctx, checks, c.globalConfig.GetBool(optionNameParallel)) }) }, PreRunE: c.preRunE, @@ -113,6 +114,7 @@ Use --metrics-enabled to collect and push metrics to Prometheus.`, cmd.Flags().String(optionNameMetricsPusherAddress, "pushgateway.staging.internal", "prometheus metrics pusher address") cmd.Flags().Bool(optionNameCreateCluster, false, "creates cluster before executing checks") cmd.Flags().StringSlice(optionNameChecks, []string{"pingpong"}, "list of checks to execute") + cmd.Flags().Bool(optionNameParallel, false, "run the --checks list concurrently instead of sequentially (default sequential)") cmd.Flags().Bool(optionNameMetricsEnabled, true, "enable metrics") cmd.Flags().Int64(optionNameSeed, -1, "seed, -1 for random") cmd.Flags().Duration(optionNameTimeout, 30*time.Minute, "timeout") diff --git a/config/local.yaml b/config/local.yaml index d389ee2b..ff0faf56 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -40,12 +40,12 @@ clusters: bee: bee-config: bee-local-dns config: local-dns - count: 3 + count: 8 mode: node light: bee-config: bee-local-light config: local-light - count: 2 + count: 0 mode: node local-dns-autotls: _inherit: "local" @@ -101,7 +101,7 @@ clusters: node-groups: local: _inherit: "" - image: k3d-registry.localhost:5000/ethersphere/bee:latest + image: k3d-registry.localhost:5002/ethersphere/bee:latest image-pull-policy: Always ingress-annotations: nginx.ingress.kubernetes.io/affinity: "cookie" @@ -374,6 +374,61 @@ checks: - light timeout: 30m type: load + # Self-driving radius check: upload random data to random full nodes (own postage label) to + # drive the storage radius up to target-radius (3); then wait for pull-sync to go idle + # (pullsyncRate==0 — the reserve worker only decreases while SyncRate()==0), dilute one batch + # down to the chain's minimum validity so it expires soon (commitment drop), and verify the + # radius ticks back down below its peak (3 -> 2). Needs the patched bee image (small reserve). + # Locally the reserve-worker idle tick drops the radius in ~10 min (before any expiry); on + # testnet the drop is expiry-driven (~minimumValidityBlocks after diluting, a few hours) — + # there, raise decrease-timeout to ~6h and timeout to ~8h (a chain postage price must be set). + ci-reserve-radius: + options: + postage-ttl: 48h # must exceed the 24h min-validity floor (24h lands on the boundary and is rejected) + postage-depth: 22 + postage-label: reserve-radius + blob-size: 8388608 # 8 MiB — bigger blobs reach the higher target radius faster + max-uploads: 200 + target-radius: 3 + force-decrease: true # dilute one batch to minimum validity to hasten its expiry (commitment drop) + dilute-step: 1 # +1 per dilution ~halves TTL; larger jumps revert at the min-validity floor + max-dilutions: 20 + warmup-wait: 15m + increase-timeout: 30m + sync-settle-wait: 10m # wait for pullsyncRate==0 before diluting/observing (the decrease gate) + decrease-timeout: 30m # local: the idle tick drops it in ~10min. testnet (expiry-driven): ~6h + poll-interval: 15s + upload-groups: + - bee + timeout: 45m + type: reserve-radius + # Simplified reserve-radius scenario (v2, WIP): fill the reserve with small, fixed-size + # postage batches until fill-percent crosses target-fill-percent, wait for pull-sync to + # settle. NOTE: as of this writing fill()/driveDecrease in reserveradiusv2.go are under + # active rework — keep this config in sync with pkg/check/reserveradiusv2/reserveradiusv2.go. + ci-reserve-radius-v2: + options: + postage-amount: 1000000000 + batch-depth: 18 + postage-label: reserve-radius-v2 + blob-size: 1048576 # 1 MiB per upload + data-per-batch: 4194304 # 4 MiB = ~1024 chunks per batch — granular fill control + max-batches: 50 + reserve-capacity: 4000 # patched bee reserve capacity in chunks + target-fill-percent: 0.6 + dilute: true + dilute-step: 1 + dilute-interval: 1m # one dilution per minute — distinct events next to radius changes + max-dilution-rounds: 5 + warmup-wait: 15m + fill-timeout: 30m + sync-settle-wait: 10m + decrease-timeout: 30m + poll-interval: 10s + upload-groups: + - bee + timeout: 90m + type: reserve-radius-v2 ci-soc: options: postage-ttl: 24h diff --git a/pkg/bee/api/api.go b/pkg/bee/api/api.go index f943fc33..5450b17f 100644 --- a/pkg/bee/api/api.go +++ b/pkg/bee/api/api.go @@ -42,22 +42,23 @@ type Client struct { service service // Reuse a single struct instead of allocating one for each service on the heap. // Services that API provides. - Act *ActService - Bytes *BytesService - Chunks *ChunksService - Dirs *DirsService - Feed *FeedService - Files *FilesService - Node *NodeService - PingPong *PingPongService - Pinning *PinningService - Postage *PostageService - PSS *PSSService - SOC *SOCService - Stake *StakingService - Status *StatusService - Stewardship *StewardshipService - Tags *TagsService + Act *ActService + Bytes *BytesService + Chunks *ChunksService + Dirs *DirsService + Feed *FeedService + Files *FilesService + Node *NodeService + PingPong *PingPongService + Pinning *PinningService + Postage *PostageService + PSS *PSSService + Redistribution *RedistributionService + SOC *SOCService + Stake *StakingService + Status *StatusService + Stewardship *StewardshipService + Tags *TagsService } // NewClient constructs a new Client. @@ -103,6 +104,7 @@ func newClient(apiURL *url.URL, httpClient *http.Client) (c *Client) { c.Pinning = (*PinningService)(&c.service) c.Postage = (*PostageService)(&c.service) c.PSS = (*PSSService)(&c.service) + c.Redistribution = (*RedistributionService)(&c.service) c.SOC = (*SOCService)(&c.service) c.Stake = (*StakingService)(&c.service) c.Status = (*StatusService)(&c.service) diff --git a/pkg/bee/api/redistribution.go b/pkg/bee/api/redistribution.go new file mode 100644 index 00000000..de7f6417 --- /dev/null +++ b/pkg/bee/api/redistribution.go @@ -0,0 +1,35 @@ +package api + +import ( + "context" + "net/http" +) + +type RedistributionService service + +// RedistributionState is the subset of the node's /redistributionstate response +// relevant to radius / pull-sync validation: whether the node is frozen, fully +// synced, healthy, and how far the redistribution round has progressed. These are +// the signals that distinguish a recovering node from a halted one after a radius +// change (a node that cannot reach fullySynced or that freezes skips rounds). +type RedistributionState struct { + IsFrozen bool `json:"isFrozen"` + IsFullySynced bool `json:"isFullySynced"` + IsHealthy bool `json:"isHealthy"` + HasSufficientFunds bool `json:"hasSufficientFunds"` + Phase string `json:"phase"` + Round uint64 `json:"round"` + LastWonRound uint64 `json:"lastWonRound"` + LastPlayedRound uint64 `json:"lastPlayedRound"` + LastFrozenRound uint64 `json:"lastFrozenRound"` + LastSelectedRound uint64 `json:"lastSelectedRound"` + LastSampleDurationSeconds float64 `json:"lastSampleDurationSeconds"` + Block uint64 `json:"block"` +} + +// RedistributionState returns the node's redistribution-game state. This endpoint +// is full-mode only; it errors on nodes without the redistribution agent enabled. +func (s *RedistributionService) RedistributionState(ctx context.Context) (resp *RedistributionState, err error) { + err = s.client.requestJSON(ctx, http.MethodGet, "/redistributionstate", nil, &resp) + return resp, err +} diff --git a/pkg/bee/api/status.go b/pkg/bee/api/status.go index 32e98a16..f5cfd03d 100644 --- a/pkg/bee/api/status.go +++ b/pkg/bee/api/status.go @@ -22,6 +22,7 @@ type StatusResponse struct { IsReachable bool `json:"isReachable"` LastSyncedBlock uint64 `json:"lastSyncedBlock"` CommittedDepth uint8 `json:"committedDepth"` + IsWarmingUp bool `json:"isWarmingUp"` } // Ping pings given node @@ -29,3 +30,21 @@ func (s *StatusService) Status(ctx context.Context) (resp *StatusResponse, err e err = s.client.requestJSON(ctx, http.MethodGet, "/status", nil, &resp) return resp, err } + +type NeighborhoodStat struct { + Neighborhood string `json:"neighborhood"` + ReserveSizeWithinRadius int `json:"reserveSizeWithinRadius"` + Proximity uint8 `json:"proximity"` +} + +type NeighborhoodsResponse struct { + Neighborhoods []NeighborhoodStat `json:"neighborhoods"` +} + +// Neighborhoods returns per-neighborhood reserve stats, computed synchronously (uncached) +// on every call — unlike Status's ReserveSizeWithinRadius, which is refreshed on a fixed +// interval and can lag a just-written chunk by that interval. +func (s *StatusService) Neighborhoods(ctx context.Context) (resp *NeighborhoodsResponse, err error) { + err = s.client.requestJSON(ctx, http.MethodGet, "/status/neighborhoods", nil, &resp) + return resp, err +} diff --git a/pkg/bee/client.go b/pkg/bee/client.go index 711ffeb6..83c3febe 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -1089,3 +1089,14 @@ func (c *Client) FindFeedUpdate(ctx context.Context, signer crypto.Signer, topic func (c *Client) Status(ctx context.Context) (*api.StatusResponse, error) { return c.api.Status.Status(ctx) } + +// Neighborhoods returns per-neighborhood reserve stats, computed synchronously (uncached) +// on every call — use this instead of Status's ReserveSizeWithinRadius when freshness matters. +func (c *Client) Neighborhoods(ctx context.Context) (*api.NeighborhoodsResponse, error) { + return c.api.Status.Neighborhoods(ctx) +} + +// RedistributionState returns the node's redistribution-game state (full-mode only). +func (c *Client) RedistributionState(ctx context.Context) (*api.RedistributionState, error) { + return c.api.Redistribution.RedistributionState(ctx) +} diff --git a/pkg/check/load/load.go b/pkg/check/load/load.go index 71e29b2c..3228f7f3 100644 --- a/pkg/check/load/load.go +++ b/pkg/check/load/load.go @@ -37,6 +37,7 @@ type Options struct { DownloadGroups []string MaxCommittedDepth uint8 CommittedDepthCheckWait time.Duration + DecreaseHold time.Duration // once target reached and committedDepth drops, pause uploads this long before re-filling (0 = disabled) IterationWait time.Duration } @@ -57,6 +58,7 @@ func NewDefaultOptions() Options { DownloadGroups: []string{}, MaxCommittedDepth: 2, CommittedDepthCheckWait: 5 * time.Minute, + DecreaseHold: 0, IterationWait: 5 * time.Minute, } } @@ -70,6 +72,9 @@ var _ beekeeper.Action = (*Check)(nil) type Check struct { metrics metrics logger logging.Logger + + mu sync.Mutex + reachedTarget bool // set once committedDepth reaches max; consumed on the following decrease } func NewCheck(log logging.Logger) beekeeper.Action { @@ -185,7 +190,7 @@ func (c *Check) run(ctx context.Context, cluster orchestration.Cluster, o Option default: } - if !c.checkCommittedDepth(ctx, uploader, o.MaxCommittedDepth, o.CommittedDepthCheckWait) { + if !c.checkCommittedDepth(ctx, uploader, o.MaxCommittedDepth, o.CommittedDepthCheckWait, o.DecreaseHold) { return } @@ -299,7 +304,7 @@ func (c *Check) run(ctx context.Context, cluster orchestration.Cluster, o Option return nil } -func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, maxDepth uint8, wait time.Duration) bool { +func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, maxDepth uint8, wait, decreaseHold time.Duration) bool { for { statusResp, err := client.Status(ctx) if err != nil { @@ -308,8 +313,21 @@ func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, max } if statusResp.CommittedDepth < maxDepth { + // Below target. If committedDepth just dropped from the target and a + // decrease-hold is set, pause uploads for that window before re-filling + // so the radius decrease + resync can play out without interference. + if decreaseHold > 0 && c.consumeReachedTarget() { + c.logger.Infof("committedDepth dropped to %d (max %d); holding uploads for %v before re-filling", statusResp.CommittedDepth, maxDepth, decreaseHold) + select { + case <-ctx.Done(): + return false + case <-time.After(decreaseHold): + } + } return true } + + c.markReachedTarget() c.logger.Infof("waiting %v for CommittedDepth to decrease. Current: %d, Max: %d", wait, statusResp.CommittedDepth, maxDepth) select { @@ -321,6 +339,24 @@ func (c *Check) checkCommittedDepth(ctx context.Context, client *bee.Client, max } } +func (c *Check) markReachedTarget() { + c.mu.Lock() + c.reachedTarget = true + c.mu.Unlock() +} + +// consumeReachedTarget reports whether the target was reached since the last call, +// resetting the flag so the decrease-hold fires once per fill→drop cycle (re-arming). +func (c *Check) consumeReachedTarget() bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.reachedTarget { + c.reachedTarget = false + return true + } + return false +} + func (c *Check) Report() []prometheus.Collector { return c.metrics.Report() } diff --git a/pkg/check/reserveradius/README.md b/pkg/check/reserveradius/README.md new file mode 100644 index 00000000..fcb55ac0 --- /dev/null +++ b/pkg/check/reserveradius/README.md @@ -0,0 +1,136 @@ +# reserve-radius check + +Drives a **storage-radius change** on a Bee cluster and verifies the radius comes +back **down** once pull-sync settles — exercising the reserve worker's decrease path +(the puller reconfiguration that caused the liveness bug in beekeeper PR #581). + +One self-driving run pushes the radius **up** by uploading, then stops and waits for +pull-sync to go idle — the reserve worker only decreases the radius while +`SyncRate()==0` (== `/status pullsyncRate`) — optionally dilutes a batch down to the chain's +minimum validity so it expires soon (hastening the commitment drop), and confirms the radius +ticks back down. + +> Keep this file in sync with the code. If you change `Options`, the `Run` flow, the +> emitted metrics, the registration, or the bee-patch requirement, update the matching +> section below in the same change. + +## What it does (`Run` flow) + +A single flow — there are no modes. + +1. **`selectNodes`** — shuffle the full-node clients (seeded by `rnd-seed`), optionally + filtered to `upload-groups`. +2. **`waitForWarmupDone`** — block until every observed node reports `isWarmingUp == false`. + The reserve worker's decrease loop is gated on stabilization, so this is a precondition + for the radius ever dropping. +3. **baseline** — snapshot `/status` for all observed nodes. +4. **`driveIncrease`** — upload `BlobSize` random blobs to **randomly-chosen** nodes (each + under its own mutable batch tagged with `postage-label`) until any node's `storageRadius` + reaches `TargetRadius`, tracking a per-node high-water `peak`. A transient per-node batch + error is logged and skipped (another random node is picked next). +5. **`waitPullSyncIdle`** — stop uploading and wait until every node reports + `pullsyncRate <= 0.05` (the rate settles to a small residual, never exactly 0), or + `SyncSettleWait` elapses (then it proceeds anyway). This opens the decrease gate. +6. **`diluteToMinValidity`** (when `force-decrease`) — dilute one batch, `+DiluteStep` at a + time (each step ~halves its remaining TTL), until a dilution is rejected because it would + drop validity below the chain's `minimumValidityBlocks` floor. Dilution **cannot reach + expiry** (the floor stops it, ~a few hours of validity), so this only *hastens* the natural + expiry — the batch now expires within the check's budget instead of its full TTL. On expiry + its chunks are evicted and reserve commitment drops. Diluting a single batch is a partial + expiry (drops commitment without evicting the whole reserve, which would halt). +7. **`observeDecrease`** — watch for any node's `storageRadius` to fall below its `peak`. + The first decrease passes the check; no decrease within `DecreaseTimeout` fails it (a + stuck puller keeps `SyncRate>0`, so the radius never decreases — cf. PR #581). + +**Two decrease paths.** On the patched local cluster the reserve worker's idle tick drops the +radius ~10 min after pull-sync idles (before any expiry) — validated live: 0→3 in ~64 MiB, then +3→2 at ~8 min. On testnet (real reserve, a chain price set) the decrease is **expiry-driven**: +the diluted batch expires ~`minimumValidityBlocks` after the dilution (a few hours), evicting +its chunks; raise `decrease-timeout` (~6h) and `timeout` (~8h) accordingly. + +## Options + +Defaults in `NewDefaultOptions()`; YAML keys (kebab-case) are wired in +`pkg/config/check.go` under the `reserve-radius` entry. + +| field | yaml | default | purpose | +| --- | --- | --- | --- | +| `RndSeed` | `rnd-seed` | `time.Now().UnixNano()` | seed for `random.PseudoGenerator` → shuffled node pick + random uploader choice | +| `PostageTTL` | `postage-ttl` | `48h` | batch TTL; must exceed the chain's 24h minimum validity once a price is set (24h lands exactly on the boundary and is rejected) | +| `PostageDepth` | `postage-depth` | `22` | batch depth | +| `PostageLabel` | `postage-label` | `reserve-radius` | this check's batch label | +| `UploadGroups` | `upload-groups` | `[bee]` | node groups to observe and upload to | +| `BlobSize` | `blob-size` | `8388608` (8 MiB) | bytes per upload (bigger blobs reach a higher target radius faster) | +| `MaxUploads` | `max-uploads` | `200` | cap on uploads in the increase phase | +| `TargetRadius` | `target-radius` | `3` | storageRadius any node must reach before stopping uploads (≥3 leaves headroom above the effective floor) | +| `ForceDecrease` | `force-decrease` | `true` | dilute one batch to minimum validity to hasten its expiry; `false` = idle-tick only | +| `DiluteStep` | `dilute-step` | `1` | depth increase per dilution (~halves TTL); larger jumps revert once they cross the min-validity floor | +| `MaxDilutions` | `max-dilutions` | `20` | cap on dilutions when driving a batch to the validity floor | +| `GasPrice` | `gas-price` | `""` | gas price for the dilute tx (empty = node default) | +| `WarmupWait` | `warmup-wait` | `15m` | max wait for nodes to finish warmup | +| `IncreaseTimeout` | `increase-timeout` | `30m` | max time to reach `TargetRadius` | +| `SyncSettleWait` | `sync-settle-wait` | `10m` | max wait for `pullsyncRate<=0.05` on all nodes (the decrease gate); proceeds anyway on timeout | +| `DecreaseTimeout` | `decrease-timeout` | `6h` | max time to observe a radius decrease (local idle tick ~10min; testnet expiry ~a few hours) | +| `PollInterval` | `poll-interval` | `15s` | poll cadence | + +## Metrics (`metrics.go`) + +Emitted via the `metrics.Reporter` interface (pushed to the pushgateway when the +check runs with `--metrics-enabled`). Namespace `beekeeper`, subsystem +`check_reserve_radius`: + +- `…_storage_radius{node}` — gauge, storageRadius per node +- `…_reserve_size{node}` — gauge, reserve size (chunks) per node +- `…_reserve_within_radius{node}` — gauge, reserve chunks within the storage radius (completeness signal) +- `…_pullsync_rate{node}` — gauge, `/status` pullsyncRate per node +- `…_time_to_increase_seconds` — gauge, first upload → `TargetRadius` +- `…_time_to_decrease_seconds` — gauge, pull-sync idle → first observed radius decrease +- `…_dilution_total` — counter, batch dilutions applied when hastening expiry (force-decrease) + +## Requirements + +- **Patched bee image.** On stock capacity the reserve is far too large to move. + The check needs a node built with `bee/.github/patches/radius_reserve.patch` + (`DefaultReserveCapacity`→4000, `ReserveWakeUpDuration`→10s) and + `radius_threshold.patch` (decrease `threshold`→100%). See the `radius-testing` skill. The + default local-cluster image is **not** patched — build and push it first, or the radius never + climbs. +- **Postage price — optional locally, required for the testnet (expiry-driven) path.** Locally + the decrease is driven by pull-sync idling (`SyncRate()==0`), so the check passes at price 0 — + but at price 0 batches have infinite TTL (`batchTTL -1`), so the `force-decrease` dilute step + has nothing to hasten and skips itself. On testnet the decrease is **expiry-driven**, so a + **non-zero price is required** (set it with the `change-storage-price` skill) — otherwise no + batch ever expires. Once a price is set the contract enforces a minimum batch validity, so + `postage-ttl` must exceed 24h — hence the `48h` default. +- **`isWarmingUp` in `StatusResponse`** (`pkg/bee/api/status.go`) — the node returns it; + the struct field was added for this check's warmup gate. + +## Running it + +Against a patched local cluster (`local-dns`): + +```sh +./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius \ + --timeout=45m --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info +``` + +## Observed behavior (local, patched cluster) + +- **Increase**: ~48–64 MiB moves the max `storageRadius` 0→3 in ~4–6 min (random uploads + distribute by proximity, so all nodes climb). +- **Local decrease is pull-sync-idle-gated**: once uploads stop and `pullsyncRate` settles, the + reserve worker ticks the radius down (3→2) after ~8–13 min; it can then oscillate 2↔3. There + is an effective floor (~2), so drive to `target-radius` ≥3 to leave headroom. +- **Dilution cannot reach expiry**: the postage contract reverts any dilution that would drop a + batch below `minimumValidityBlocks` (~a few hours here), so a batch floors at ~6h validity and + can't be diluted to 0. `force-decrease` therefore *hastens* the natural expiry (dilutes to the + floor) rather than triggering it directly. Locally this is redundant with the idle tick (which + fires first); it matters on testnet, where the decrease is expiry-driven. See the + `radius-testing` skill for the full mechanics. + +## Related + +- `.claude/skills/radius-testing/` — design, mechanics, operating know-how, `radius-poll.sh`, the metrics stack. +- `.claude/skills/change-storage-price/` — activating a non-zero postage price on the local chain. +- Prior art: PR #581 (`radiusdecrease`), PR #591 (`stampexpiry`), `pkg/check/gc`, + `pkg/check/load`, `pkg/check/smoke`. diff --git a/pkg/check/reserveradius/metrics.go b/pkg/check/reserveradius/metrics.go new file mode 100644 index 00000000..3cf56edd --- /dev/null +++ b/pkg/check/reserveradius/metrics.go @@ -0,0 +1,50 @@ +package reserveradius + +import ( + m "github.com/ethersphere/beekeeper/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type metrics struct { + StorageRadius *prometheus.GaugeVec + ReserveSize *prometheus.GaugeVec + ReserveWithinRadius *prometheus.GaugeVec + PullsyncRate *prometheus.GaugeVec + TimeToIncrease prometheus.Gauge + TimeToDecrease prometheus.Gauge + Dilutions prometheus.Counter +} + +func newMetrics(subsystem string) metrics { + return metrics{ + StorageRadius: nodeGauge(subsystem, "storage_radius", "Storage radius reported by /status, per node."), + ReserveSize: nodeGauge(subsystem, "reserve_size", "Reserve size (chunks) reported by /status, per node."), + ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node (completeness signal)."), + PullsyncRate: nodeGauge(subsystem, "pullsync_rate", "Pull-sync rate reported by /status, per node."), + TimeToIncrease: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "time_to_increase_seconds", Help: "Seconds to reach the target storage radius from the first upload.", + }), + TimeToDecrease: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "time_to_decrease_seconds", Help: "Seconds from pull-sync going idle to the first observed radius decrease.", + }), + Dilutions: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "dilution_total", Help: "Postage-batch dilutions applied when forcing the decrease via expiry.", + }), + } +} + +// nodeGauge builds a per-node GaugeVec in the check namespace/subsystem. +func nodeGauge(subsystem, name, help string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{Namespace: m.Namespace, Subsystem: subsystem, Name: name, Help: help}, + []string{"node"}, + ) +} + +// Report implements the metrics.Reporter interface. +func (c *Check) Report() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(c.metrics) +} diff --git a/pkg/check/reserveradius/reserveradius.go b/pkg/check/reserveradius/reserveradius.go new file mode 100644 index 00000000..e857b24d --- /dev/null +++ b/pkg/check/reserveradius/reserveradius.go @@ -0,0 +1,428 @@ +// Package reserveradius drives a storage-radius change on a Bee cluster and verifies the +// radius comes back down once pull-sync settles. It runs a single self-driving flow: +// +// 1. wait for warmup/stabilization — the reserve worker gates its decrease loop on it; +// 2. drive the radius UP by uploading random data to random full nodes until a target is reached; +// 3. stop uploading and wait for pull-sync to go idle (pullsyncRate==0) — the reserve worker +// only decreases the radius while SyncRate()==0 (== /status pullsyncRate), so the whole +// neighbourhood must finish syncing first; +// 4. observe the radius tick back down below its peak (the reserve worker's decrease), failing +// if no decrease appears within the timeout — a stuck puller shows as no decrease. +// +// This check requires a bee node patched for a small reserve (see +// bee/.github/patches/radius_*.patch); on stock capacity the radius never moves. +package reserveradius + +import ( + "context" + crand "crypto/rand" + "errors" + "fmt" + "math/rand" + "time" + + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/bee/api" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + "github.com/ethersphere/beekeeper/pkg/random" + "github.com/ethersphere/beekeeper/pkg/test" +) + +// Options represents reserve-radius check options. +type Options struct { + RndSeed int64 + PostageTTL time.Duration + PostageDepth uint64 + PostageLabel string // this check's own batch label + UploadGroups []string // node groups to upload to / observe (empty = all full nodes) + BlobSize int64 // bytes per upload + MaxUploads int // cap on uploads during the increase phase + TargetRadius uint8 // storageRadius any node must reach before stopping uploads + ForceDecrease bool // after pull-sync idles, dilute a batch to expiry to force the decrease (vs waiting for the idle tick) + DiluteStep uint64 // depth increase applied per dilution (each step ~halves remaining TTL) + MaxDilutions int // cap on dilutions per batch when forcing expiry + GasPrice string // gas price for the dilute tx (empty = node default) + WarmupWait time.Duration // max wait for nodes to leave warmup before driving + IncreaseTimeout time.Duration // max time to reach TargetRadius + SyncSettleWait time.Duration // after uploads stop, max wait for pullsyncRate==0 on all nodes (the decrease gate) + DecreaseTimeout time.Duration // max time to observe a radius decrease after pull-sync idles + PollInterval time.Duration +} + +// NewDefaultOptions returns new default options. +func NewDefaultOptions() Options { + return Options{ + RndSeed: time.Now().UnixNano(), + PostageTTL: 48 * time.Hour, + PostageDepth: 22, + PostageLabel: "reserve-radius", + UploadGroups: []string{"bee"}, + BlobSize: 8 << 20, // 8 MiB + MaxUploads: 200, + TargetRadius: 3, + ForceDecrease: true, + DiluteStep: 1, // depth += 1 per dilution ~halves remaining TTL; larger jumps revert once they cross the min-validity floor + MaxDilutions: 20, + GasPrice: "", + WarmupWait: 15 * time.Minute, + IncreaseTimeout: 30 * time.Minute, + SyncSettleWait: 10 * time.Minute, + DecreaseTimeout: 6 * time.Hour, // testnet: batch expiry lands ~minimumValidityBlocks (~5h) after diluting to the floor + PollInterval: 15 * time.Second, + } +} + +// pullSyncIdleRate is the pullsyncRate at or below which pull-sync is treated as idle. +// The rate is a decaying windowed average that settles to a small residual rather than +// exactly 0, so an == 0 gate would never open; active sync is orders of magnitude higher. +const pullSyncIdleRate = 0.05 + +// compile check whether Check implements interface +var _ beekeeper.Action = (*Check)(nil) + +// Check instance. +type Check struct { + metrics metrics + logger logging.Logger +} + +// NewCheck returns a new reserve-radius check. +func NewCheck(log logging.Logger) beekeeper.Action { + return &Check{ + metrics: newMetrics("check_reserve_radius"), + logger: log, + } +} + +// batchRef is a postage batch created during the increase phase, paired with the node it lives on. +type batchRef struct { + node *bee.Client + batchID string +} + +// Run drives the radius up by uploading random data to random nodes, waits for pull-sync to +// go idle (the reserve worker's decrease gate), optionally forces the decrease by diluting a +// batch to expiry, then verifies the radius ticks back down. +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return errors.New("invalid options type") + } + if o.TargetRadius == 0 { + return errors.New("target-radius must be > 0") + } + + c.logger.Infof("random seed: %d", o.RndSeed) + rnd := random.PseudoGenerator(o.RndSeed) + nodes, err := c.selectNodes(ctx, cluster, rnd, o) + if err != nil { + return err + } + c.logger.Infof("observing %d node(s); uploading random data to random nodes under label %q", len(nodes), o.PostageLabel) + + // 1. Wait for warmup/stabilization — the decrease loop is gated on it. + if err := c.waitForWarmupDone(ctx, nodes, o); err != nil { + return err + } + c.snapshot(ctx, nodes, "baseline") + + // 2. Drive the radius up by uploading to random nodes; peak tracks the per-node high-water. + peak := make(map[string]uint8, len(nodes)) + batches, err := c.driveIncrease(ctx, nodes, rnd, peak, o) + if err != nil { + return err + } + + // 3. Stop uploading and wait for pull-sync to go idle. The reserve worker only decreases + // the radius when SyncRate()==0 (== /status pullsyncRate), so the neighbourhood must + // finish syncing first. Keep raising peak in case a decrease begins during the wait. + c.waitPullSyncIdle(ctx, nodes, peak, o) + + // 4. Optionally hasten the decrease: dilute one batch down to the contract's minimum + // validity so it expires within the check's budget (~minimumValidityBlocks, ~hours) + // instead of its full TTL. On expiry its chunks are evicted and reserve commitment + // drops, which — with idle pull-sync — lets the reserve worker decrease the radius. + // Diluting a single batch is a partial expiry: it drops commitment without evicting + // the whole reserve (which would halt the neighbourhood). + if o.ForceDecrease && len(batches) > 0 { + c.diluteToMinValidity(ctx, batches[0], o) + } + + // 5. Observe the radius tick back down below its peak (via the idle tick, or batch expiry). + return c.observeDecrease(ctx, nodes, peak, o) +} + +// selectNodes returns the observed full-node clients (shuffled, optionally filtered to UploadGroups). +func (c *Check) selectNodes(ctx context.Context, cluster orchestration.Cluster, rnd *rand.Rand, o Options) (orchestration.ClientList, error) { + fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) + if err != nil { + return nil, fmt.Errorf("get shuffled full node clients: %w", err) + } + nodes := fullNodeClients + if len(o.UploadGroups) > 0 { + nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) + } + if len(nodes) < 1 { + return nil, fmt.Errorf("reserve-radius check requires at least 1 full node, got %d", len(nodes)) + } + return nodes, nil +} + +// waitForWarmupDone blocks until every observed node reports isWarmingUp=false. +func (c *Check) waitForWarmupDone(ctx context.Context, nodes orchestration.ClientList, o Options) error { + c.logger.Infof("waiting up to %s for nodes to finish warmup/stabilization", o.WarmupWait) + deadline := time.Now().Add(o.WarmupWait) + for { + if err := ctx.Err(); err != nil { + return err + } + ready := true + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil || s.IsWarmingUp { + ready = false + } + } + if ready { + c.logger.Info("all observed nodes finished warmup") + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("nodes still warming up after %s", o.WarmupWait) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + +// driveIncrease uploads BlobSize random blobs to randomly-chosen nodes until any observed +// node reaches TargetRadius. Each node uploads under its own mutable batch (this check's +// label). It raises the per-node high-water peak as it goes and returns the batches it +// created (in creation order) for the optional force-decrease step. +func (c *Check) driveIncrease(ctx context.Context, nodes orchestration.ClientList, rnd *rand.Rand, peak map[string]uint8, o Options) ([]batchRef, error) { + c.logger.Infof("driving increase: %d-byte blobs to random nodes until storageRadius>=%d (max %d uploads, timeout %s)", + o.BlobSize, o.TargetRadius, o.MaxUploads, o.IncreaseTimeout) + start := time.Now() + deadline := start.Add(o.IncreaseTimeout) + t := test.NewTest(c.logger) + data := make([]byte, o.BlobSize) + batchByNode := make(map[string]string, len(nodes)) + var batches []batchRef + + for i := 1; i <= o.MaxUploads; i++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("storageRadius did not reach %d within %s (%d uploads) — is the bee reserve patch active and is there enough data?", o.TargetRadius, o.IncreaseTimeout, i-1) + } + + uploader := nodes[rnd.Intn(len(nodes))] + batchID, ok := batchByNode[uploader.Name()] + if !ok { + id, err := uploader.GetOrCreateMutableBatch(ctx, o.PostageTTL, o.PostageDepth, o.PostageLabel) + if err != nil { + // Transient per-node error (e.g. a 503 from chainstate); another random + // node is picked next iteration, so skip rather than fail the whole run. + c.logger.Warningf("create batch on %s failed, skipping upload #%d: %v", uploader.Name(), i, err) + continue + } + batchID = id + batchByNode[uploader.Name()] = id + batches = append(batches, batchRef{node: uploader, batchID: id}) + c.logger.WithField("batch_id", id).Infof("node %s: using batch", uploader.Name()) + } + + if _, err := crand.Read(data); err != nil { + return nil, fmt.Errorf("generate random data: %w", err) + } + if _, _, err := t.Upload(ctx, uploader, data, batchID, nil); err != nil { + c.logger.Errorf("upload #%d to %s failed: %v", i, uploader.Name(), err) + continue + } + + _, mx := c.updatePeak(ctx, nodes, peak) + c.logger.Infof("increase: upload #%d to %s, max storageRadius=%d (target %d)", i, uploader.Name(), mx, o.TargetRadius) + if mx >= o.TargetRadius { + c.metrics.TimeToIncrease.Set(time.Since(start).Seconds()) + c.logger.Infof("reached storageRadius %d after %d uploads (~%.1f MiB) across %d batch(es) in %s", + mx, i, float64(int64(i)*o.BlobSize)/(1<<20), len(batches), time.Since(start).Round(time.Second)) + return batches, nil + } + } + return nil, fmt.Errorf("storageRadius did not reach %d after %d uploads", o.TargetRadius, o.MaxUploads) +} + +// diluteToMinValidity dilutes a batch as far down as the contract allows — each +DiluteStep +// (default 1) roughly halves its remaining TTL — stopping when a dilution is rejected because +// it would drop the batch's validity below the chain's minimumValidityBlocks floor. That floor +// makes expiry itself unreachable by dilution, so this only *hastens* the natural expiry: it +// leaves the batch at its minimum TTL (~minimumValidityBlocks) so it expires within the check's +// budget rather than its full original TTL. On expiry the batch's chunks are evicted and reserve +// commitment drops. Best-effort — a read/dilute error just stops early (the batch keeps whatever +// TTL it reached), so this never fails the check on its own. +func (c *Check) diluteToMinValidity(ctx context.Context, b batchRef, o Options) { + c.logger.Infof("diluting batch %s on %s toward minimum validity to hasten expiry", b.batchID, b.node.Name()) + for k := 0; k < o.MaxDilutions; k++ { + if ctx.Err() != nil { + return + } + stamp, err := b.node.PostageStamp(ctx, b.batchID) + if err != nil { + c.logger.Warningf("read batch %s on %s: %v; stopping dilution", b.batchID, b.node.Name(), err) + return + } + if stamp.BatchTTL < 0 { + // batchTTL -1 means "never expires" — the chain postage price is 0, so there is no + // expiry to hasten. Skip (the local idle tick still drives the decrease). + c.logger.Warningf("batch %s on %s has infinite TTL (chain price is 0) — nothing to hasten; skipping dilution", b.batchID, b.node.Name()) + return + } + if !stamp.Exists || stamp.BatchTTL == 0 { + c.logger.Infof("batch %s on %s already expired (ttl=%ds)", b.batchID, b.node.Name(), stamp.BatchTTL) + return + } + newDepth := uint64(stamp.Depth) + o.DiluteStep + if err := b.node.DilutePostageBatch(ctx, b.batchID, newDepth, o.GasPrice); err != nil { + // Rejected: the resulting validity would fall below minimumValidityBlocks — the + // batch is at its minimum TTL, the expected stop condition. + c.logger.Infof("batch %s on %s at minimum validity: ttl ~%ds (~%.1fh); dilute to depth %d rejected (%v)", + b.batchID, b.node.Name(), stamp.BatchTTL, float64(stamp.BatchTTL)/3600, newDepth, err) + return + } + c.metrics.Dilutions.Inc() + c.logger.Infof("diluted batch %s on %s to depth %d (ttl was %ds ~%.1fh)", b.batchID, b.node.Name(), newDepth, stamp.BatchTTL, float64(stamp.BatchTTL)/3600) + } + c.logger.Warningf("batch %s on %s: reached max %d dilutions without hitting the validity floor", b.batchID, b.node.Name(), o.MaxDilutions) +} + +// waitPullSyncIdle blocks until every node reports pullsyncRate==0 (pull-sync idle) or +// SyncSettleWait elapses. Idle is the reserve worker's decrease precondition, so this is a +// best-effort gate: on timeout it logs and proceeds (observeDecrease will fail if the radius +// never drops). It keeps raising peak, since a decrease can begin as soon as sync idles. +func (c *Check) waitPullSyncIdle(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8, o Options) { + c.logger.Infof("uploads stopped; waiting up to %s for pull-sync to go idle (pullsyncRate<=%.2f on all %d node(s))", o.SyncSettleWait, pullSyncIdleRate, len(nodes)) + deadline := time.Now().Add(o.SyncSettleWait) + for { + idle, maxRate := true, 0.0 + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + idle = false + continue + } + c.emit(n.Name(), s) + if s.StorageRadius > peak[n.Name()] { + peak[n.Name()] = s.StorageRadius + } + if s.PullsyncRate > pullSyncIdleRate { + idle = false + } + if s.PullsyncRate > maxRate { + maxRate = s.PullsyncRate + } + } + if idle { + c.logger.Infof("pull-sync idle on all nodes (pullsyncRate<=%.2f) — decrease gate open", pullSyncIdleRate) + return + } + if time.Now().After(deadline) { + c.logger.Warningf("pull-sync did not go idle within %s (max pullsyncRate=%.4f); proceeding to observe anyway", o.SyncSettleWait, maxRate) + return + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return // context cancelled (e.g. check timeout) — stop waiting + } + } +} + +// observeDecrease watches for any node's storageRadius to fall below its peak — the reserve +// worker's decrease once pull-sync is idle. No decrease within DecreaseTimeout is the failure +// signal (a stuck puller never lets SyncRate reach 0, so the radius never decreases). +func (c *Check) observeDecrease(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8, o Options) error { + c.logger.Infof("watching for a radius decrease below the peak %v (timeout %s)", peak, o.DecreaseTimeout) + start := time.Now() + deadline := start.Add(o.DecreaseTimeout) + for { + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + continue + } + c.emit(n.Name(), s) + if s.StorageRadius > peak[n.Name()] { + peak[n.Name()] = s.StorageRadius + } + if s.StorageRadius < peak[n.Name()] { + c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) + c.logger.Infof("radius decrease observed on %s (%d -> %d) after %s — done", + n.Name(), peak[n.Name()], s.StorageRadius, time.Since(start).Round(time.Second)) + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("no radius decrease below peak %v within %s after pull-sync idled — the reserve worker never ticked the radius down (a stuck puller keeps SyncRate>0, cf. PR #581)", peak, o.DecreaseTimeout) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + +// updatePeak polls each node, emits metrics, raises the per-node high-water peak, and +// returns the min and max current storageRadius across the nodes that responded. +func (c *Check) updatePeak(ctx context.Context, nodes orchestration.ClientList, peak map[string]uint8) (minR, maxR uint8) { + seen := false + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + c.logger.Debugf("%s: status error: %v", n.Name(), err) + continue + } + c.emit(n.Name(), s) + if s.StorageRadius > peak[n.Name()] { + peak[n.Name()] = s.StorageRadius + } + if s.StorageRadius > maxR { + maxR = s.StorageRadius + } + if !seen || s.StorageRadius < minR { + minR = s.StorageRadius + seen = true + } + } + return minR, maxR +} + +// snapshot polls every node, emits metrics, and logs a per-node line for the given phase. +func (c *Check) snapshot(ctx context.Context, nodes orchestration.ClientList, phase string) { + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + c.logger.Debugf("%s: status error: %v", n.Name(), err) + continue + } + c.emit(n.Name(), s) + c.logger.Infof("[%s] %s: storageRadius=%d reserveSize=%d withinR=%d pullsyncRate=%.4f warmingUp=%t", + phase, n.Name(), s.StorageRadius, s.ReserveSize, s.ReserveSizeWithinRadius, s.PullsyncRate, s.IsWarmingUp) + } +} + +func (c *Check) emit(node string, s *api.StatusResponse) { + c.metrics.StorageRadius.WithLabelValues(node).Set(float64(s.StorageRadius)) + c.metrics.ReserveSize.WithLabelValues(node).Set(float64(s.ReserveSize)) + c.metrics.ReserveWithinRadius.WithLabelValues(node).Set(float64(s.ReserveSizeWithinRadius)) + c.metrics.PullsyncRate.WithLabelValues(node).Set(s.PullsyncRate) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} diff --git a/pkg/check/reserveradiusv2/README.md b/pkg/check/reserveradiusv2/README.md new file mode 100644 index 00000000..3a8660ae --- /dev/null +++ b/pkg/check/reserveradiusv2/README.md @@ -0,0 +1,68 @@ +# reserve-radius-v2 check + +A **simplified, reproducible** reserve-radius scenario built to capture pull-sync issues +around storage-radius changes and postage-batch dilution. + +Differences from v1 (`reserve-radius`): + +- **Granular fill control** — instead of one large depth-22 batch per node, it creates + **many small batches** (default depth 18) and uploads a **fixed amount of data per batch** + (default 4 MiB ≈ 1024 chunks), so the reserve fill is known and reproducible. +- **Capacity-threshold targeting** — it fills until any node's `reserveSizeWithinRadius` + is **just over 50%** of the configured reserve capacity (default target 55% of 4000 + chunks), the trigger point for the radius decrease. +- **Dilutions as distinct events** — after pull-sync settles, batches are diluted **one at + a time** (default one per minute), each incrementing a plain `dilutions_total` counter so + a Grafana time series shows exactly when dilutions occur relative to radius changes. + +## Flow + +1. Wait for all observed nodes to finish warmup. +2. **Fill**: create batches round-robin across nodes (label `-`), upload + `data-per-batch` bytes under each, stop when max fill ≥ `target-fill-percent`. +3. Wait for pull-sync to go idle (`pullsyncRate <= 0.05`, best-effort). +4. **Decrease**: dilute one batch every `dilute-interval` while polling; pass on the first + `storageRadius` drop below its peak, fail after `decrease-timeout` (stuck pull-sync). + +## Options + +Defaults in `NewDefaultOptions()`; YAML keys wired in `pkg/config/check.go` under +`reserve-radius-v2`. Key ones: + +| yaml | default | purpose | +| --- | --- | --- | +| `postage-amount` | `1000` | amount per batch (each batch is created fresh via `CreatePostageBatch`, never reused) | +| `batch-depth` | `18` | depth of each small batch | +| `blob-size` | `1048576` (1 MiB) | bytes per upload | +| `data-per-batch` | `4194304` (4 MiB) | bytes uploaded under each batch | +| `max-batches` | `50` | cap on batches in the fill phase | +| `reserve-capacity` | `4000` | reserve capacity in chunks (must match the bee patch) | +| `target-fill-percent` | `55` | stop filling when any node crosses this % of capacity | +| `dilute` | `true` | dilute batches to drive the decrease | +| `dilute-interval` | `1m` | spacing between dilutions (distinct chart events) | +| `max-dilution-rounds` | `5` | each round dilutes every batch once | +| `decrease-timeout` | `30m` | max wait for a radius decrease (raise to ~6h on testnet) | + +## Metrics + +Namespace `beekeeper`, subsystem `check_reserve_radius_v2`: + +- `…_storage_radius{node}`, `…_reserve_size{node}`, `…_reserve_within_radius{node}`, + `…_reserve_fill_percent{node}`, `…_pullsync_rate{node}` — gauges polled every `poll-interval` +- `…_radius_events_total{node,direction}` — counter of observed radius increases/decreases +- `…_batches_created_total`, `…_uploaded_bytes_total`, `…_dilutions_total` — counters +- `…_time_to_decrease_seconds` — gauge, decrease-phase start → first observed decrease + +## Requirements + +Same as v1: a **patched bee image** (`radius_reserve.patch`: capacity 4000, fast wakeup; +`radius_threshold.patch`) — on stock capacity the radius never moves. `reserve-capacity` +must match the patched value. A non-zero chain postage price is required for dilution to +have an effect (at price 0 batches have infinite TTL and dilution is skipped). + +## Running it + +```sh +./dist/beekeeper check --cluster-name=local-dns --checks=ci-reserve-radius-v2 \ + --timeout=90m --metrics-enabled=true --metrics-pusher-address=localhost:9091 --log-verbosity=info +``` diff --git a/pkg/check/reserveradiusv2/metrics.go b/pkg/check/reserveradiusv2/metrics.go new file mode 100644 index 00000000..d6772f4a --- /dev/null +++ b/pkg/check/reserveradiusv2/metrics.go @@ -0,0 +1,61 @@ +package reserveradiusv2 + +import ( + m "github.com/ethersphere/beekeeper/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type metrics struct { + StorageRadius *prometheus.GaugeVec + ReserveSize *prometheus.GaugeVec + ReserveWithinRadius *prometheus.GaugeVec + ReserveFillPercent *prometheus.GaugeVec + PullsyncRate *prometheus.GaugeVec + RadiusEvents *prometheus.CounterVec + BatchesCreated prometheus.Counter + UploadedBytes prometheus.Counter + Dilutions prometheus.Counter + TimeToDecrease prometheus.Gauge +} + +func newMetrics(subsystem string) metrics { + return metrics{ + StorageRadius: nodeGauge(subsystem, "storage_radius", "Storage radius reported by /status, per node."), + ReserveSize: nodeGauge(subsystem, "reserve_size", "Reserve size (chunks) reported by /status, per node."), + ReserveWithinRadius: nodeGauge(subsystem, "reserve_within_radius", "Reserve chunks within the storage radius, per node."), + ReserveFillPercent: nodeGauge(subsystem, "reserve_fill_percent", "Reserve fill as percent of configured capacity, per node."), + PullsyncRate: nodeGauge(subsystem, "pullsync_rate", "Pull-sync rate reported by /status, per node."), + RadiusEvents: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "radius_events_total", Help: "Storage radius changes observed, per node and direction (increase/decrease).", + }, []string{"node", "direction"}), + BatchesCreated: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "batches_created_total", Help: "Postage batches created during the fill phase.", + }), + UploadedBytes: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "uploaded_bytes_total", Help: "Bytes uploaded during the fill phase.", + }), + Dilutions: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "dilutions_total", Help: "Postage-batch dilutions applied during the decrease phase.", + }), + TimeToDecrease: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, Subsystem: subsystem, + Name: "time_to_decrease_seconds", Help: "Seconds from the start of the decrease phase to the first observed radius decrease.", + }), + } +} + +func nodeGauge(subsystem, name, help string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec( + prometheus.GaugeOpts{Namespace: m.Namespace, Subsystem: subsystem, Name: name, Help: help}, + []string{"node"}, + ) +} + +// Report implements the metrics.Reporter interface. +func (c *Check) Report() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(c.metrics) +} diff --git a/pkg/check/reserveradiusv2/reserveradiusv2.go b/pkg/check/reserveradiusv2/reserveradiusv2.go new file mode 100644 index 00000000..7f6c4ff4 --- /dev/null +++ b/pkg/check/reserveradiusv2/reserveradiusv2.go @@ -0,0 +1,380 @@ +// Package reserveradiusv2 is a simplified, reproducible reserve-radius scenario built to +// capture pull-sync issues around radius changes and batch dilution. Instead of blindly +// uploading until a target radius (v1), it fills the reserve with a known amount of data +// per small postage batch until just over the 50% capacity threshold, then dilutes the +// batches one at a time and verifies the storage radius decreases while pull-sync resyncs. +package reserveradiusv2 + +import ( + "context" + crand "crypto/rand" + "errors" + "fmt" + "time" + + "github.com/ethersphere/beekeeper/pkg/bee" + "github.com/ethersphere/beekeeper/pkg/bee/api" + "github.com/ethersphere/beekeeper/pkg/beekeeper" + "github.com/ethersphere/beekeeper/pkg/logging" + "github.com/ethersphere/beekeeper/pkg/orchestration" + "github.com/ethersphere/beekeeper/pkg/random" + "github.com/ethersphere/beekeeper/pkg/test" +) + +// Options represents reserve-radius-v2 check options. +type Options struct { + RndSeed int64 + PostageAmount int64 // amount per batch (each batch is created fresh, never reused) + BatchDepth uint64 // depth of each small batch + PostageLabel string // label prefix; each batch gets label- + UploadGroups []string // node groups to upload to / observe (empty = all full nodes) + BlobSize int64 // bytes per upload + DataPerBatch int64 // bytes uploaded under each batch + MaxBatches int // cap on batches during the fill phase + ReserveCapacity uint64 // reserve capacity in chunks (patched bee: 4000) + TargetFillPercent float64 // stop filling when any node's reserve reaches this % of capacity + Dilute bool // dilute batches after the fill to trigger the decrease + DiluteStep uint64 // depth increase per dilution + DiluteInterval time.Duration // spacing between dilutions so they show as distinct events + MaxDilutionRounds int // cap on dilution rounds (each round dilutes every batch once) + GasPrice string + WarmupWait time.Duration + FillTimeout time.Duration + SyncSettleWait time.Duration // after filling, max wait for pull-sync to go idle + DecreaseTimeout time.Duration // max time to observe a radius decrease + PollInterval time.Duration +} + +// NewDefaultOptions returns new default options. +func NewDefaultOptions() Options { + return Options{ + RndSeed: time.Now().UnixNano(), + PostageAmount: 1000, + BatchDepth: 18, + PostageLabel: "reserve-radius-v2", + UploadGroups: []string{"bee"}, + BlobSize: 1 << 20, // 1 MiB + DataPerBatch: 4 << 20, // 4 MiB = ~1024 chunks per batch + MaxBatches: 50, + ReserveCapacity: 4000, // patched bee reserve; 50% = 2000 chunks + TargetFillPercent: 0.6, // just over the 50% decrease threshold + Dilute: true, + DiluteStep: 1, + DiluteInterval: time.Minute, + MaxDilutionRounds: 5, + GasPrice: "", + WarmupWait: 15 * time.Minute, + FillTimeout: 30 * time.Minute, + SyncSettleWait: 10 * time.Minute, + DecreaseTimeout: 30 * time.Minute, + PollInterval: 10 * time.Second, + } +} + +// pullSyncIdleRate is the pullsyncRate at or below which pull-sync is treated as idle +// (the rate decays to a small residual, never exactly 0). +const pullSyncIdleRate = 0.05 + +var _ beekeeper.Action = (*Check)(nil) + +// Check instance. +type Check struct { + metrics metrics + logger logging.Logger +} + +// NewCheck returns a new reserve-radius-v2 check. +func NewCheck(log logging.Logger) beekeeper.Action { + return &Check{ + metrics: newMetrics("check_reserve_radius_v2"), + logger: log, + } +} + +type batchRef struct { + node *bee.Client + batchID string + done bool // reached the validity floor or errored; skip in later rounds +} + +// state tracks per-node radius history across polls. +type state struct { + peak map[string]uint8 + last map[string]uint8 +} + +// Run fills the reserve just over the 50% capacity threshold using small batches with a +// fixed amount of data each, waits for pull-sync to settle, then dilutes the batches one +// at a time and verifies the storage radius decreases. +func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts any) error { + o, ok := opts.(Options) + if !ok { + return errors.New("invalid options type") + } + if o.ReserveCapacity == 0 { + return errors.New("reserve-capacity must be > 0") + } + if o.TargetFillPercent <= 50 { + c.logger.Warningf("target-fill-percent %.1f is not over 50%% — the decrease threshold may already be crossed before dilution", o.TargetFillPercent) + } + + c.logger.Infof("random seed: %d", o.RndSeed) + rnd := random.PseudoGenerator(o.RndSeed) + + fullNodeClients, err := cluster.ShuffledFullNodeClients(ctx, rnd) + if err != nil { + return fmt.Errorf("get shuffled full node clients: %w", err) + } + nodes := fullNodeClients + if len(o.UploadGroups) > 0 { + nodes = fullNodeClients.FilterByNodeGroups(o.UploadGroups) + } + if len(nodes) < 1 { + return fmt.Errorf("reserve-radius-v2 check requires at least 1 full node, got %d", len(nodes)) + } + c.logger.Infof("observing %d node(s); reserve capacity %d chunks, fill target %.1f%%", len(nodes), o.ReserveCapacity, o.TargetFillPercent) + + st := &state{peak: make(map[string]uint8), last: make(map[string]uint8)} + c.poll(ctx, nodes, st, o) + + + // for { + // c.poll(ctx, nodes, st, o) + // time.Sleep(5 * time.Second) + // } + _, err = c.fill(ctx, nodes, st, o) + if err != nil { + return err + } + + c.logger.Infof("radius check fill done. waiting for pull sync to go idle") + c.poll(ctx, nodes, st, o) + + c.waitPullSyncIdle(ctx, nodes, st, o) + + c.logger.Infof("radius check fill done. pull sync idle, driving decrease") + c.poll(ctx, nodes, st, o) + + // return c.driveDecrease(ctx, nodes, batches, st, o) + return nil +} + +// fill creates small batches round-robin across nodes and uploads exactly DataPerBatch +// bytes under each, stopping once any node's reserve crosses TargetFillPercent. +func (c *Check) fill(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) ([]*batchRef, error) { + clusterSize := len(nodes) + stamps := 5 + // each stamp alone must push a single node over the target, since poll() reports the + // max fill across nodes and every stamp's data lands on just one node. + sizePerStamp := int(o.TargetFillPercent * float64(o.ReserveCapacity)) + + c.logger.Infof("fill: clusterSize %d, reserveCapacity %d, stamps %d, sizePerStamp(chunks) %d", clusterSize, o.ReserveCapacity, stamps, sizePerStamp) + + start := time.Now() + deadline := start.Add(o.FillTimeout) + t := test.NewTest(c.logger) + data := make([]byte, sizePerStamp*4096) + var batches []*batchRef + + // caveat: if there's an error, this won't work very well because + // the errors have a continue statement + for b := 0; b < stamps; b++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("reserve did not reach %.1f%% within %s (%d batches) — is the bee reserve patch active?", o.TargetFillPercent, o.FillTimeout, b-1) + } + + node := nodes[b] + label := fmt.Sprintf("%s-%d", o.PostageLabel, b) + batchID, err := node.CreatePostageBatch(ctx, o.PostageAmount, o.BatchDepth, label, false) + if err != nil { + c.logger.Warningf("create batch %s on %s failed, skipping: %v", label, node.Name(), err) + continue + } + batches = append(batches, &batchRef{node: node, batchID: batchID}) + c.metrics.BatchesCreated.Inc() + c.logger.WithField("batch_id", batchID).Infof("batch #%d created on %s (label %s)", b, node.Name(), label) + + if _, err := crand.Read(data); err != nil { + return nil, fmt.Errorf("generate random data: %w", err) + } + if _, _, err := t.Upload(ctx, node, data, batchID, nil); err != nil { + c.logger.Errorf("upload to %s failed: %v", node.Name(), err) + continue + } + c.metrics.UploadedBytes.Add(float64(o.BlobSize)) + + // brief buffer for the upload to land in the local reserve before scanning it; + // poll() reads /status/neighborhoods, which is computed fresh on every call, so + // this is just upload-commit latency, not a wait on any cached/ticked stat. + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return nil, err + } + maxFill, _ := c.poll(ctx, nodes, st, o) + c.logger.Infof("fill: batch #%d done (%d bytes), max reserve fill %.1f%% (target %.1f%%)", b, o.DataPerBatch, maxFill, o.TargetFillPercent) + if maxFill >= (o.TargetFillPercent * 100) { + c.logger.Infof("reached %.1f%% fill with %d batches (~%.1f MiB) in %s", + maxFill, len(batches), float64(int64(len(batches))*o.DataPerBatch)/(1<<20), time.Since(start).Round(time.Second)) + return batches, nil + } + } + return nil, fmt.Errorf("reserve did not reach %.1f%% after %d batches", o.TargetFillPercent, stamps) +} + +// waitPullSyncIdle blocks until every node reports pull-sync idle or SyncSettleWait +// elapses (best-effort: on timeout it logs and proceeds). +func (c *Check) waitPullSyncIdle(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) { + c.logger.Infof("uploads stopped; waiting up to %s for pull-sync to go idle (pullsyncRate<=%.2f on all nodes)", o.SyncSettleWait, pullSyncIdleRate) + deadline := time.Now().Add(o.SyncSettleWait) + for { + _, maxRate := c.poll(ctx, nodes, st, o) + if maxRate <= pullSyncIdleRate { + c.logger.Info("pull-sync idle on all nodes") + return + } + if time.Now().After(deadline) { + c.logger.Warningf("pull-sync did not go idle within %s (max pullsyncRate=%.4f); proceeding anyway", o.SyncSettleWait, maxRate) + return + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return + } + } +} + +// driveDecrease dilutes the batches one at a time (one dilution per DiluteInterval) while +// watching for any node's storageRadius to fall below its peak. Interleaving dilutions +// with polling makes each dilution a distinct event next to the radius/pull-sync series. +func (c *Check) driveDecrease(ctx context.Context, nodes orchestration.ClientList, batches []*batchRef, radiusState *state, o Options) error { + c.logger.Infof("watching for a radius decrease below peak %v (timeout %s, dilute=%t)", radiusState.peak, o.DecreaseTimeout, o.Dilute) + start := time.Now() + deadline := start.Add(o.DecreaseTimeout) + nextDilute := time.Now() + dilutions, maxDilutions := 0, len(batches)*o.MaxDilutionRounds + + for { + if err := ctx.Err(); err != nil { + return err + } + c.poll(ctx, nodes, radiusState, o) + for _, n := range nodes { + if radiusState.last[n.Name()] < radiusState.peak[n.Name()] { + c.metrics.TimeToDecrease.Set(time.Since(start).Seconds()) + c.logger.Infof("radius decrease observed on %s (%d -> %d) after %s and %d dilution(s) — done", + n.Name(), radiusState.peak[n.Name()], radiusState.last[n.Name()], time.Since(start).Round(time.Second), dilutions) + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("no radius decrease below peak %v within %s (%d dilutions applied) — possible stuck pull-sync", radiusState.peak, o.DecreaseTimeout, dilutions) + } + if o.Dilute && dilutions < maxDilutions && time.Now().After(nextDilute) { + if c.diluteNext(ctx, batches, dilutions, o) { + dilutions++ + } + nextDilute = time.Now().Add(o.DiluteInterval) + } + if err := sleepCtx(ctx, o.PollInterval); err != nil { + return err + } + } +} + +// diluteNext dilutes the next not-done batch (round-robin) by DiluteStep and reports +// whether a dilution was applied. A rejected dilution (validity floor) marks the batch done. +func (c *Check) diluteNext(ctx context.Context, batches []*batchRef, round int, o Options) bool { + for i := 0; i < len(batches); i++ { + b := batches[(round+i)%len(batches)] + if b.done { + continue + } + stamp, err := b.node.PostageStamp(ctx, b.batchID) + if err != nil { + c.logger.Warningf("read batch %s on %s: %v; skipping", b.batchID, b.node.Name(), err) + b.done = true + continue + } + if stamp.BatchTTL < 0 { + c.logger.Warningf("batch %s has infinite TTL (chain price 0) — dilution cannot hasten expiry; skipping", b.batchID) + b.done = true + continue + } + newDepth := uint64(stamp.Depth) + o.DiluteStep + if err := b.node.DilutePostageBatch(ctx, b.batchID, newDepth, o.GasPrice); err != nil { + c.logger.Infof("batch %s at validity floor (ttl ~%ds); dilute to depth %d rejected: %v", b.batchID, stamp.BatchTTL, newDepth, err) + b.done = true + continue + } + c.metrics.Dilutions.Inc() + c.logger.Infof("diluted batch %s on %s to depth %d (ttl was %ds)", b.batchID, b.node.Name(), newDepth, stamp.BatchTTL) + return true + } + return false +} + +// poll reads /status and /status/neighborhoods on every node, emits metrics, updates +// peak/last radius, and returns the max reserve fill percent and max pullsyncRate across +// nodes. Fill is computed from /status/neighborhoods (summed across neighborhoods) rather +// than /status's ReserveSizeWithinRadius, which bee only refreshes on a fixed interval +// (15 minutes in production) and can lag a just-written chunk by that long. +func (c *Check) poll(ctx context.Context, nodes orchestration.ClientList, st *state, o Options) (maxFill, maxRate float64) { + for _, n := range nodes { + s, err := n.Status(ctx) + if err != nil { + c.logger.Debugf("%s: status error: %v", n.Name(), err) + continue + } + name := n.Name() + + nh, err := n.Neighborhoods(ctx) + if err != nil { + c.logger.Debugf("%s: neighborhoods error: %v", n.Name(), err) + continue + } + var reserveSize uint64 + for _, stat := range nh.Neighborhoods { + reserveSize += uint64(stat.ReserveSizeWithinRadius) + } + fill := float64(reserveSize) / float64(o.ReserveCapacity) * 100 + c.emit(name, s, fill) + + if prev, seen := st.last[name]; seen && s.StorageRadius != prev { + dir := "increase" + if s.StorageRadius < prev { + dir = "decrease" + } + c.metrics.RadiusEvents.WithLabelValues(name, dir).Inc() + c.logger.Infof("%s: storageRadius %d -> %d (%s), fill %.1f%%, pullsyncRate %.4f", name, prev, s.StorageRadius, dir, fill, s.PullsyncRate) + } + st.last[name] = s.StorageRadius + if s.StorageRadius > st.peak[name] { + st.peak[name] = s.StorageRadius + } + if fill > maxFill { + maxFill = fill + } + if s.PullsyncRate > maxRate { + maxRate = s.PullsyncRate + } + } + return maxFill, maxRate +} + +func (c *Check) emit(node string, s *api.StatusResponse, fill float64) { + c.metrics.StorageRadius.WithLabelValues(node).Set(float64(s.StorageRadius)) + c.metrics.ReserveSize.WithLabelValues(node).Set(float64(s.ReserveSize)) + c.metrics.ReserveWithinRadius.WithLabelValues(node).Set(float64(s.ReserveSizeWithinRadius)) + c.metrics.ReserveFillPercent.WithLabelValues(node).Set(fill) + c.metrics.PullsyncRate.WithLabelValues(node).Set(s.PullsyncRate) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} diff --git a/pkg/check/runner.go b/pkg/check/runner.go index 71c89e2b..33a92865 100644 --- a/pkg/check/runner.go +++ b/pkg/check/runner.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "strings" + "sync" "time" "github.com/ethersphere/beekeeper/pkg/beekeeper" @@ -46,7 +47,7 @@ func NewCheckRunner( } } -func (c *CheckRunner) Run(ctx context.Context, checks []string) error { +func (c *CheckRunner) Run(ctx context.Context, checks []string, parallel bool) error { if len(checks) == 0 { return nil } @@ -91,35 +92,30 @@ func (c *CheckRunner) Run(ctx context.Context, checks []string) error { }) } - checkResults := make([]checkResult, 0, len(validatedChecks)) - hasFailures := false - - // run checks - for _, check := range validatedChecks { - c.logger.WithFields(map[string]any{ - "type": check.typeName, - "options": fmt.Sprintf("%+v", check.options), - }).Infof("running check: %s", check.name) + // run checks — each writes its own index, so the slice needs no locking + checkResults := make([]checkResult, len(validatedChecks)) + if parallel { + c.logger.WithField("count", len(validatedChecks)).Info("running checks concurrently") + var wg sync.WaitGroup + for i, check := range validatedChecks { + wg.Go(func() { + checkResults[i] = c.runOne(ctx, check) + }) + } + wg.Wait() + } else { + for i, check := range validatedChecks { + checkResults[i] = c.runOne(ctx, check) + } + } - err := check.Run(ctx, c.cluster) - if err != nil { + hasFailures := false + for _, r := range checkResults { + if r.err != nil { hasFailures = true - c.logger.WithFields(map[string]any{ - "type": check.typeName, - "error": err, - }).Errorf("'%s' check failed", check.name) - } else { - c.logger.WithField("type", check.typeName).Infof("'%s' check completed successfully", check.name) + break } - - // append check result - checkResults = append(checkResults, checkResult{ - check: check.name, - err: err, - timestamp: time.Now(), - }) } - if hasFailures { return formatErrorReport(checkResults) } @@ -128,6 +124,32 @@ func (c *CheckRunner) Run(ctx context.Context, checks []string) error { return nil } +// runOne runs a single prepared check, logs the outcome, and returns its result. +// Each check carries its own timeout (checkRun.Run), and an error does not cancel +// the shared parent context, so concurrent checks fail independently. +func (c *CheckRunner) runOne(ctx context.Context, check checkRun) checkResult { + c.logger.WithFields(map[string]any{ + "type": check.typeName, + "options": fmt.Sprintf("%+v", check.options), + }).Infof("running check: %s", check.name) + + err := check.Run(ctx, c.cluster) + if err != nil { + c.logger.WithFields(map[string]any{ + "type": check.typeName, + "error": err, + }).Errorf("'%s' check failed", check.name) + } else { + c.logger.WithField("type", check.typeName).Infof("'%s' check completed successfully", check.name) + } + + return checkResult{ + check: check.name, + err: err, + timestamp: time.Now(), + } +} + type checkRun struct { name string typeName string diff --git a/pkg/config/check.go b/pkg/config/check.go index 06332fb1..fca43df9 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -30,6 +30,8 @@ import ( "github.com/ethersphere/beekeeper/pkg/check/pullsync" "github.com/ethersphere/beekeeper/pkg/check/pushsync" "github.com/ethersphere/beekeeper/pkg/check/redundancy" + "github.com/ethersphere/beekeeper/pkg/check/reserveradius" + "github.com/ethersphere/beekeeper/pkg/check/reserveradiusv2" "github.com/ethersphere/beekeeper/pkg/check/retrieval" "github.com/ethersphere/beekeeper/pkg/check/settlements" "github.com/ethersphere/beekeeper/pkg/check/smoke" @@ -481,6 +483,7 @@ var Checks = map[string]CheckType{ DownloadGroups *[]string `yaml:"download-groups"` MaxCommittedDepth *uint8 `yaml:"max-committed-depth"` CommittedDepthCheckWait *time.Duration `yaml:"committed-depth-check-wait"` + DecreaseHold *time.Duration `yaml:"decrease-hold"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) @@ -495,6 +498,73 @@ var Checks = map[string]CheckType{ return opts, nil }, }, + "reserve-radius": { + NewAction: reserveradius.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + RndSeed *int64 `yaml:"rnd-seed"` + PostageTTL *time.Duration `yaml:"postage-ttl"` + PostageDepth *uint64 `yaml:"postage-depth"` + PostageLabel *string `yaml:"postage-label"` + UploadGroups *[]string `yaml:"upload-groups"` + BlobSize *int64 `yaml:"blob-size"` + MaxUploads *int `yaml:"max-uploads"` + TargetRadius *uint8 `yaml:"target-radius"` + ForceDecrease *bool `yaml:"force-decrease"` + DiluteStep *uint64 `yaml:"dilute-step"` + MaxDilutions *int `yaml:"max-dilutions"` + GasPrice *string `yaml:"gas-price"` + WarmupWait *time.Duration `yaml:"warmup-wait"` + IncreaseTimeout *time.Duration `yaml:"increase-timeout"` + SyncSettleWait *time.Duration `yaml:"sync-settle-wait"` + DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + PollInterval *time.Duration `yaml:"poll-interval"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := reserveradius.NewDefaultOptions() + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, + "reserve-radius-v2": { + NewAction: reserveradiusv2.NewCheck, + NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) { + checkOpts := new(struct { + RndSeed *int64 `yaml:"rnd-seed"` + PostageAmount *int64 `yaml:"postage-amount"` + BatchDepth *uint64 `yaml:"batch-depth"` + PostageLabel *string `yaml:"postage-label"` + UploadGroups *[]string `yaml:"upload-groups"` + BlobSize *int64 `yaml:"blob-size"` + DataPerBatch *int64 `yaml:"data-per-batch"` + MaxBatches *int `yaml:"max-batches"` + ReserveCapacity *uint64 `yaml:"reserve-capacity"` + TargetFillPercent *float64 `yaml:"target-fill-percent"` + Dilute *bool `yaml:"dilute"` + DiluteStep *uint64 `yaml:"dilute-step"` + DiluteInterval *time.Duration `yaml:"dilute-interval"` + MaxDilutionRounds *int `yaml:"max-dilution-rounds"` + GasPrice *string `yaml:"gas-price"` + WarmupWait *time.Duration `yaml:"warmup-wait"` + FillTimeout *time.Duration `yaml:"fill-timeout"` + SyncSettleWait *time.Duration `yaml:"sync-settle-wait"` + DecreaseTimeout *time.Duration `yaml:"decrease-timeout"` + PollInterval *time.Duration `yaml:"poll-interval"` + }) + if err := check.Options.Decode(checkOpts); err != nil { + return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) + } + opts := reserveradiusv2.NewDefaultOptions() + if err := applyCheckConfig(checkGlobalConfig, checkOpts, &opts); err != nil { + return nil, fmt.Errorf("applying options: %w", err) + } + return opts, nil + }, + }, "soc": { NewAction: soc.NewCheck, NewOptions: func(checkGlobalConfig CheckGlobalConfig, check Check) (any, error) {