From 6f17b04e56e5a65edcd6484f1a3ecd6c536dc1ee Mon Sep 17 00:00:00 2001 From: x3c41a Date: Tue, 7 Jul 2026 12:43:03 +0200 Subject: [PATCH] Add monitoring stack: Prometheus + Grafana dashboards docker-compose bringing up Prometheus + Grafana that scrape the parachain collator's Substrate metrics, the relay validator, and the domain-state exporter (examples/papi/chain-exporter.ts). Three provisioned dashboards: chain health (block height/finality/txpool/peers + per-class block weight), economics (provider stake/balances/slashes/checkpoint accruals), and protocol activity (agreements, challenge lifecycle, checkpoints). Dashboard JSON is generated by grafana/gen-dashboards.mjs. Pin the collator and relay Prometheus ports in zombienet.toml so the stack can scrape them at fixed addresses. Part of #267. --- monitoring/README.md | 64 ++++ monitoring/docker-compose.yml | 47 +++ .../grafana/dashboards/w3s-chain-health.json | 319 ++++++++++++++++++ .../grafana/dashboards/w3s-economics.json | 267 +++++++++++++++ .../dashboards/w3s-protocol-activity.json | 312 +++++++++++++++++ monitoring/grafana/gen-dashboards.mjs | 203 +++++++++++ .../provisioning/dashboards/dashboards.yml | 12 + .../provisioning/datasources/prometheus.yml | 10 + monitoring/prometheus/prometheus.yml | 25 ++ zombienet.toml | 5 + 10 files changed, 1264 insertions(+) create mode 100644 monitoring/README.md create mode 100644 monitoring/docker-compose.yml create mode 100644 monitoring/grafana/dashboards/w3s-chain-health.json create mode 100644 monitoring/grafana/dashboards/w3s-economics.json create mode 100644 monitoring/grafana/dashboards/w3s-protocol-activity.json create mode 100644 monitoring/grafana/gen-dashboards.mjs create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 00000000..20233f25 --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,64 @@ +# Monitoring stack + +Ephemeral Prometheus + Grafana for local stress and soak runs +(paritytech/web3-storage#267). Brings up dashboards for chain health, +economics, and protocol activity against a running chain and the domain-state +exporter. + +## Scrape targets + +| Target | Source | Port | +| --- | --- | --- | +| `web3-storage-chain` | `examples/papi/chain-exporter.ts` (domain state) | 9700 | +| `parachain-node` | collator's built-in Substrate metrics | 9615 | +| `relay-node` | relay validator's Substrate metrics (optional) | 9616 | + +The collator and relay Prometheus ports are pinned in `zombienet.toml` +(zombienet otherwise assigns a random port each run). + +## Run it + +```bash +# 1. chain + provider (from repo root) +just start-chain +just start-provider inmemory 3333 ./provider-data "" true + +# 2. domain-state exporter +pnpm --filter web3-storage-papi-demo run chain-exporter + +# 3. monitoring stack +cd monitoring && docker compose up -d +``` + +- Grafana: http://localhost:3000 (anonymous viewer; admin/admin to edit) +- Prometheus: http://localhost:9090 + +Dashboards are provisioned automatically under the "Web3 Storage" folder. + +Generate some load to watch, e.g. the challenge sweep: + +```bash +pnpm --filter web3-storage-papi-demo run lower-timeouts -- --set ChallengeTimeout=30 +pnpm --filter web3-storage-papi-demo run challenge-saturation -- --count 800 +``` + +## Dashboards + +- **Chain Health** — block height (best vs finalized), finality lag, block + production rate, txpool, peers, and per-class block weight (the mandatory + class spikes at a challenge-deadline block, PoV budget 5 MiB). +- **Economics** — provider stake, free/reserved balance, cumulative slashes, + checkpoint rewards and pool. +- **Protocol Activity** — agreements active vs expired (the lazy-cleanup + residual), pending challenges, challenge lifecycle (created/defended/slashed) + and rate, checkpoints submitted/missed, provider/bucket counts. + +The dashboard JSON is generated by `grafana/gen-dashboards.mjs`; edit that and +re-run `node grafana/gen-dashboards.mjs grafana/dashboards` rather than hand- +editing the JSON. + +## Pointing at a different target (e.g. PreviewNet) + +The containers reach the host via `host.docker.internal`. To scrape a remote +chain or exporter instead, edit the target addresses in +`prometheus/prometheus.yml`. diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 00000000..c2ed96a6 --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,47 @@ +# Ephemeral monitoring stack for local stress/soak runs (paritytech/web3-storage#267). +# +# Prometheus + Grafana that scrape three targets on the host: +# - the parachain collator's Substrate Prometheus endpoint (pinned to :9615 +# in zombienet.toml) +# - the relay validator's endpoint (:9616, optional) +# - the domain-state exporter (examples/papi/chain-exporter.ts, :9700) +# +# The nodes and the exporter run on the host (zombienet native), so the +# containers reach them via host.docker.internal; the extra_hosts mapping makes +# that resolve on Linux too. To point at a different target (e.g. PreviewNet), +# edit prometheus/prometheus.yml. +services: + prometheus: + image: prom/prometheus:v3.1.0 + container_name: w3s-prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=30d + ports: + - "9090:9090" + volumes: + - ./prometheus:/etc/prometheus:ro + - prometheus-data:/prometheus + extra_hosts: + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:11.4.0 + container_name: w3s-grafana + depends_on: + - prometheus + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + +volumes: + prometheus-data: + grafana-data: diff --git a/monitoring/grafana/dashboards/w3s-chain-health.json b/monitoring/grafana/dashboards/w3s-chain-health.json new file mode 100644 index 00000000..08b4052e --- /dev/null +++ b/monitoring/grafana/dashboards/w3s-chain-health.json @@ -0,0 +1,319 @@ +{ + "uid": "w3s-chain-health", + "title": "Web3 Storage — Chain Health", + "tags": [ + "web3-storage", + "stress" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 1, + "title": "Block height (best vs finalized)", + "description": "Parachain best and finalized height.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "substrate_block_height{job=\"parachain-node\",status=\"best\"}", + "legendFormat": "best" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "substrate_block_height{job=\"parachain-node\",status=\"finalized\"}", + "legendFormat": "finalized" + } + ] + }, + { + "id": 2, + "title": "Finality lag (blocks)", + "description": "best - finalized. A growing lag means finalization is stalling.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "substrate_block_height{job=\"parachain-node\",status=\"best\"} - substrate_block_height{job=\"parachain-node\",status=\"finalized\"}", + "legendFormat": "lag" + } + ] + }, + { + "id": 3, + "title": "Block production rate (blocks/s)", + "description": "~0.167/s at a healthy 6s block time; a dip means slow or stalled production.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(substrate_block_height{job=\"parachain-node\",status=\"best\"}[1m])", + "legendFormat": "best/s" + } + ] + }, + { + "id": 4, + "title": "Txpool ready transactions", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "substrate_ready_transactions_number{job=\"parachain-node\"}", + "legendFormat": "ready" + } + ] + }, + { + "id": 5, + "title": "Block PoV (proof_size) by class", + "description": "On-chain block weight from the exporter. The mandatory class spikes at a challenge-deadline block (the on_finalize slash sweep). Max line = 5 MiB PoV budget.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + }, + "max": 5242880 + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_block_weight_proof_size", + "legendFormat": "{{class}}" + } + ] + }, + { + "id": 6, + "title": "Block ref_time by class (ms)", + "description": "2000 ms is the block ref_time budget.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + }, + "max": 2000 + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_block_weight_ref_time / 1e9", + "legendFormat": "{{class}}" + } + ] + }, + { + "id": 7, + "title": "Peers", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "substrate_sub_libp2p_peers_count{job=\"parachain-node\"}", + "legendFormat": "peers" + } + ] + } + ] +} diff --git a/monitoring/grafana/dashboards/w3s-economics.json b/monitoring/grafana/dashboards/w3s-economics.json new file mode 100644 index 00000000..0814cd1b --- /dev/null +++ b/monitoring/grafana/dashboards/w3s-economics.json @@ -0,0 +1,267 @@ +{ + "uid": "w3s-economics", + "title": "Web3 Storage — Economics", + "tags": [ + "web3-storage", + "stress" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 8, + "title": "Provider stake", + "description": "Bonded stake per provider. Drops to 0 when fully slashed.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_provider_stake", + "legendFormat": "{{provider}}" + } + ] + }, + { + "id": 9, + "title": "Provider free balance", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_provider_free_balance", + "legendFormat": "{{provider}}" + } + ] + }, + { + "id": 10, + "title": "Provider reserved balance", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_provider_reserved_balance", + "legendFormat": "{{provider}}" + } + ] + }, + { + "id": 11, + "title": "Challenge slashes (cumulative)", + "description": "Total challenges resolved by slash (timeout or invalid response).", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_challenge_slashed_total", + "legendFormat": "slashed" + } + ] + }, + { + "id": 12, + "title": "Checkpoint rewards pending", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_checkpoint_rewards_pending_total", + "legendFormat": "rewards" + } + ] + }, + { + "id": 13, + "title": "Checkpoint pool balance", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_checkpoint_pool_total", + "legendFormat": "pool" + } + ] + } + ] +} diff --git a/monitoring/grafana/dashboards/w3s-protocol-activity.json b/monitoring/grafana/dashboards/w3s-protocol-activity.json new file mode 100644 index 00000000..6144c01a --- /dev/null +++ b/monitoring/grafana/dashboards/w3s-protocol-activity.json @@ -0,0 +1,312 @@ +{ + "uid": "w3s-protocol-activity", + "title": "Web3 Storage — Protocol Activity", + "tags": [ + "web3-storage", + "stress" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [] + }, + "panels": [ + { + "id": 14, + "title": "Agreements (active vs expired)", + "description": "Expired-but-unswept is the lazy-cleanup residual (#177).", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 30, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_agreements_active", + "legendFormat": "active" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_agreements_expired", + "legendFormat": "expired (unswept)" + } + ] + }, + { + "id": 15, + "title": "Pending challenges", + "description": "Unresolved challenges across all deadlines. Spikes during saturation.", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_challenges_pending_total", + "legendFormat": "pending" + } + ] + }, + { + "id": 16, + "title": "Challenge lifecycle (cumulative)", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_challenge_created_total", + "legendFormat": "created" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_challenge_defended_total", + "legendFormat": "defended" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_challenge_slashed_total", + "legendFormat": "slashed" + } + ] + }, + { + "id": 17, + "title": "Challenge creation rate (/s)", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "rate(web3storage_challenge_created_total[1m])", + "legendFormat": "created/s" + } + ] + }, + { + "id": 18, + "title": "Checkpoints (cumulative)", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_provider_checkpoint_submitted_total", + "legendFormat": "submitted" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_checkpoint_miss_penalized_total", + "legendFormat": "miss-penalized" + } + ] + }, + { + "id": 19, + "title": "Providers / buckets", + "description": "", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_providers_total", + "legendFormat": "providers" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "web3storage_buckets_total", + "legendFormat": "buckets" + } + ] + } + ] +} diff --git a/monitoring/grafana/gen-dashboards.mjs b/monitoring/grafana/gen-dashboards.mjs new file mode 100644 index 00000000..c8355cdb --- /dev/null +++ b/monitoring/grafana/gen-dashboards.mjs @@ -0,0 +1,203 @@ +import { writeFileSync, mkdirSync } from "node:fs"; + +const OUT = process.argv[2]; +mkdirSync(OUT, { recursive: true }); + +const DS = { type: "prometheus", uid: "prometheus" }; +let idSeq = 1; + +// target: {expr, legend?} +function panel(title, targets, opts = {}) { + const { + type = "timeseries", + unit = "short", + w = 12, + h = 8, + x = 0, + y = 0, + desc = "", + max = null, + stack = false, + } = opts; + const fieldConfig = { + defaults: { + unit, + custom: + type === "timeseries" + ? { drawStyle: "line", fillOpacity: stack ? 30 : 10, stacking: { mode: stack ? "normal" : "none" } } + : {}, + ...(max != null ? { max } : {}), + }, + overrides: [], + }; + return { + id: idSeq++, + title, + description: desc, + type, + datasource: DS, + gridPos: { h, w, x, y }, + fieldConfig, + options: type === "stat" ? { reduceOptions: { calcs: ["lastNotNull"] }, textMode: "value_and_name" } : {}, + targets: targets.map((t, i) => ({ + refId: String.fromCharCode(65 + i), + datasource: DS, + expr: t.expr, + legendFormat: t.legend ?? "", + })), + }; +} + +function dashboard(uid, title, panels) { + return { + uid, + title, + tags: ["web3-storage", "stress"], + timezone: "browser", + schemaVersion: 39, + version: 1, + refresh: "10s", + time: { from: "now-30m", to: "now" }, + templating: { list: [] }, + panels, + }; +} + +// ── Chain health ───────────────────────────────────────────────────────────── +const chainHealth = dashboard("w3s-chain-health", "Web3 Storage — Chain Health", [ + panel( + "Block height (best vs finalized)", + [ + { expr: 'substrate_block_height{job="parachain-node",status="best"}', legend: "best" }, + { expr: 'substrate_block_height{job="parachain-node",status="finalized"}', legend: "finalized" }, + ], + { x: 0, y: 0, desc: "Parachain best and finalized height." }, + ), + panel( + "Finality lag (blocks)", + [ + { + expr: 'substrate_block_height{job="parachain-node",status="best"} - substrate_block_height{job="parachain-node",status="finalized"}', + legend: "lag", + }, + ], + { x: 12, y: 0, desc: "best - finalized. A growing lag means finalization is stalling." }, + ), + panel( + "Block production rate (blocks/s)", + [{ expr: 'rate(substrate_block_height{job="parachain-node",status="best"}[1m])', legend: "best/s" }], + { x: 0, y: 8, desc: "~0.167/s at a healthy 6s block time; a dip means slow or stalled production." }, + ), + panel( + "Txpool ready transactions", + [{ expr: 'substrate_ready_transactions_number{job="parachain-node"}', legend: "ready" }], + { x: 12, y: 8 }, + ), + panel( + "Block PoV (proof_size) by class", + [{ expr: 'web3storage_block_weight_proof_size', legend: "{{class}}" }], + { + x: 0, + y: 16, + unit: "bytes", + max: 5 * 1024 * 1024, + desc: "On-chain block weight from the exporter. The mandatory class spikes at a challenge-deadline block (the on_finalize slash sweep). Max line = 5 MiB PoV budget.", + }, + ), + panel( + "Block ref_time by class (ms)", + [{ expr: 'web3storage_block_weight_ref_time / 1e9', legend: "{{class}}" }], + { x: 12, y: 16, unit: "ms", max: 2000, desc: "2000 ms is the block ref_time budget." }, + ), + panel( + "Peers", + [{ expr: 'substrate_sub_libp2p_peers_count{job="parachain-node"}', legend: "peers" }], + { x: 0, y: 24, h: 6 }, + ), +]); + +// ── Economics ──────────────────────────────────────────────────────────────── +const economics = dashboard("w3s-economics", "Web3 Storage — Economics", [ + panel("Provider stake", [{ expr: "web3storage_provider_stake", legend: "{{provider}}" }], { + x: 0, + y: 0, + desc: "Bonded stake per provider. Drops to 0 when fully slashed.", + }), + panel( + "Provider free balance", + [{ expr: "web3storage_provider_free_balance", legend: "{{provider}}" }], + { x: 12, y: 0 }, + ), + panel( + "Provider reserved balance", + [{ expr: "web3storage_provider_reserved_balance", legend: "{{provider}}" }], + { x: 0, y: 8 }, + ), + panel( + "Challenge slashes (cumulative)", + [{ expr: "web3storage_challenge_slashed_total", legend: "slashed" }], + { x: 12, y: 8, desc: "Total challenges resolved by slash (timeout or invalid response)." }, + ), + panel( + "Checkpoint rewards pending", + [{ expr: "web3storage_checkpoint_rewards_pending_total", legend: "rewards" }], + { x: 0, y: 16 }, + ), + panel( + "Checkpoint pool balance", + [{ expr: "web3storage_checkpoint_pool_total", legend: "pool" }], + { x: 12, y: 16 }, + ), +]); + +// ── Protocol activity ──────────────────────────────────────────────────────── +const activity = dashboard("w3s-protocol-activity", "Web3 Storage — Protocol Activity", [ + panel( + "Agreements (active vs expired)", + [ + { expr: "web3storage_agreements_active", legend: "active" }, + { expr: "web3storage_agreements_expired", legend: "expired (unswept)" }, + ], + { x: 0, y: 0, stack: true, desc: "Expired-but-unswept is the lazy-cleanup residual (#177)." }, + ), + panel( + "Pending challenges", + [{ expr: "web3storage_challenges_pending_total", legend: "pending" }], + { x: 12, y: 0, desc: "Unresolved challenges across all deadlines. Spikes during saturation." }, + ), + panel( + "Challenge lifecycle (cumulative)", + [ + { expr: "web3storage_challenge_created_total", legend: "created" }, + { expr: "web3storage_challenge_defended_total", legend: "defended" }, + { expr: "web3storage_challenge_slashed_total", legend: "slashed" }, + ], + { x: 0, y: 8 }, + ), + panel( + "Challenge creation rate (/s)", + [{ expr: "rate(web3storage_challenge_created_total[1m])", legend: "created/s" }], + { x: 12, y: 8 }, + ), + panel( + "Checkpoints (cumulative)", + [ + { expr: "web3storage_provider_checkpoint_submitted_total", legend: "submitted" }, + { expr: "web3storage_checkpoint_miss_penalized_total", legend: "miss-penalized" }, + ], + { x: 0, y: 16 }, + ), + panel( + "Providers / buckets", + [ + { expr: "web3storage_providers_total", legend: "providers" }, + { expr: "web3storage_buckets_total", legend: "buckets" }, + ], + { x: 12, y: 16 }, + ), +]); + +for (const d of [chainHealth, economics, activity]) { + writeFileSync(`${OUT}/${d.uid}.json`, JSON.stringify(d, null, 2) + "\n"); + console.log(`wrote ${d.uid}.json (${d.panels.length} panels)`); +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 00000000..03099e00 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: web3-storage + orgId: 1 + folder: Web3 Storage + type: file + disableDeletion: false + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..00f99157 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..e3f4a959 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,25 @@ +# Scrape config for the local stress/soak stack. Targets are on the host +# (zombienet native + the PAPI exporter), reached via host.docker.internal. +# For a remote target (e.g. PreviewNet), replace the target addresses below. +global: + scrape_interval: 6s # one parachain block + evaluation_interval: 15s + +scrape_configs: + # Domain-state exporter (examples/papi/chain-exporter.ts): provider balances, + # agreements, challenges, checkpoints, per-class block weight. + - job_name: web3-storage-chain + static_configs: + - targets: ["host.docker.internal:9700"] + + # Parachain collator's built-in Substrate metrics: block height, finality, + # txpool, peers. Port pinned in zombienet.toml. + - job_name: parachain-node + static_configs: + - targets: ["host.docker.internal:9615"] + + # Relay validator's Substrate metrics (optional). Pin its prometheus_port in + # zombienet.toml to enable; harmless to leave down otherwise. + - job_name: relay-node + static_configs: + - targets: ["host.docker.internal:9616"] diff --git a/zombienet.toml b/zombienet.toml index a224861e..e0784cca 100644 --- a/zombienet.toml +++ b/zombienet.toml @@ -12,6 +12,8 @@ default_args = ["-lruntime=info"] name = "alice" validator = true rpc_port = 9900 +# Pinned so the monitoring/ stack can scrape relay metrics at a fixed address. +prometheus_port = 9616 [[relaychain.nodes]] name = "bob" @@ -28,4 +30,7 @@ name = "alice" validator = true command = "{{PROJECT_ROOT}}/.bin/polkadot-omni-node" rpc_port = 2222 +# Pin the collator's Prometheus port so the monitoring/ stack can scrape it at a +# fixed address (zombienet otherwise assigns a random port each run). +prometheus_port = 9615 args = ["--collator", "-lruntime=info"]