Add storage-cli tooling crate with stress-test upload

.github/workflows/integration-tests.yml

@@ -585,6 +585,135 @@ jobs:
             /tmp/omni-node.log
             /tmp/provider.log
 
+  # ── Utils CLI tooling ─────────────────────────────────────────────
+  utils-integration-tests:
+    name: Utils Integration Tests
+    runs-on: parity-large
+    timeout-minutes: 30
+    needs: [set-image, build]
+    container:
+      image: ${{ needs.set-image.outputs.CI_IMAGE }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          persist-credentials: false
+
+      - name: Load common environment variables via env file
+        run: cat .github/env >> $GITHUB_ENV
+
+      - name: Install just
+        run: cargo install just --locked || true
+
+      - name: Cache Polkadot SDK binaries
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        id: sdk-cache
+        with:
+          path: |
+            .bin/polkadot
+            .bin/polkadot-execute-worker
+            .bin/polkadot-prepare-worker
+            .bin/polkadot-omni-node
+            .bin/chain-spec-builder
+          key: polkadot-sdk-${{ env.POLKADOT_SDK_VERSION }}
+
+      - name: Download Polkadot SDK binaries
+        if: steps.sdk-cache.outputs.cache-hit != 'true'
+        run: just polkadot_version="$POLKADOT_SDK_VERSION" download-polkadot-sdk-binaries
+
+      - name: Resolve chain-spec script
+        run: |
+          RUNTIME=web3-storage-paseo
+          SCRIPT=$(jq -r --arg r "$RUNTIME" '.[] | select(.name==$r) | .chain_spec_script // empty' scripts/runtimes-matrix.json)
+          if [ -z "$SCRIPT" ]; then
+            echo "ERROR: no chain_spec_script for runtime '$RUNTIME' in scripts/runtimes-matrix.json"
+            exit 1
+          fi
+          echo "Using chain-spec script: $SCRIPT"
+          echo "CHAIN_SPEC_SCRIPT=$SCRIPT" >> $GITHUB_ENV
+
+      # Before the download so the cached target/ can't clobber it. save-if
+      # false — only the build job writes this cache.
+      - name: Rust cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          shared-key: "integration-tests-build"
+          save-if: false
+
+      # Extract into target/release so the artifact's contents land back at
+      # their original paths (chmod, chain_spec_command scripts).
+      - name: Download build artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v6
+        with:
+          name: build
+          path: target/release
+
+      - name: Restore executable bits
+        run: chmod +x target/release/storage-provider-node
+
+      - name: Start chain (dev mode, 2s blocks)
+        uses: ./.github/actions/start-e2e-chain
+        with:
+          chain-spec-script: ${{ env.CHAIN_SPEC_SCRIPT }}
+
+      - name: Wait for parachain blocks
+        uses: ./.github/actions/wait-for-parachain
+        with:
+          log-file: /tmp/omni-node.log
+
+      - name: Register provider on-chain
+        run: |
+          echo "//Alice" > /tmp/alice-key && chmod 600 /tmp/alice-key
+          cargo run --release -p storage-client --example register_provider \
+            ws://127.0.0.1:2222 http://127.0.0.1:3333 /ip4/127.0.0.1/tcp/3333 /tmp/alice-key
+
+      - name: Start provider (inmemory) and wait for health
+        run: |
+          nohup ./target/release/storage-provider-node \
+            --keyfile /tmp/alice-key --storage-mode inmemory \
+            --bind-addr 0.0.0.0:3333 --chain-rpc ws://127.0.0.1:2222 \
+            --enable-checkpoint-coordinator \
+            --disable-auth-i-know-what-i-am-doing > /tmp/provider.log 2>&1 &
+
+      - name: Wait for provider health
+        uses: ./.github/actions/wait-for-provider-health
+
+      # Seed: //Bob negotiates terms and opens a bucket + primary agreement
+      # against the //Alice provider, giving the CLI something to upload to.
+      - name: Seed bucket + agreement (//Bob → //Alice provider)
+        run: |
+          cargo run --release -p storage-client --example complete_workflow \
+            ws://127.0.0.1:2222 http://127.0.0.1:3333 //Bob
+
+      # Run the CLI's stress-test upload command against the //Alice provider using 
+      # the //Bob key.
+      - name: storage-cli stress-test upload
+        run: |
+          cargo run --release -p storage-cli -- \
+            --suri //Bob \
+            stress-test upload \
+            --provider 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY \
+            --users 2 \
+            --uploads-per-user 5 \
+            --max-payload-size 1048576 \
+            --parallel-uploads
+
+      - name: Stop services
+        if: always()
+        run: |
+          pkill -f "polkadot-omni-node" 2>/dev/null || true
+          pkill -f "polkadot" 2>/dev/null || true
+          pkill -f "storage-provider-node" 2>/dev/null || true
+
+      - name: Upload logs (on failure)
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: utils-integration-test-logs
+          path: |
+            /tmp/omni-node.log
+            /tmp/provider.log
+
   # ── UI Playwright e2e ─────────────────────────────────────────────────────
   # Drive / console / provider UIs (Vite dev servers) exercised against a
   # standalone paseo dev chain (omni-node, 2s blocks, no relay chain) + a
@@ -776,7 +905,7 @@ jobs:
 
   integration-tests-complete:
     name: Integration Tests
-    needs: [changes, build, integration-tests, e2e-integration-tests, sc-integration-tests, ui-integration-tests, coverage-gate]
+    needs: [changes, build, integration-tests, e2e-integration-tests, sc-integration-tests, ui-integration-tests, utils-integration-tests, coverage-gate]
     if: always()
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -23,6 +23,9 @@ members = [
 	"storage-interfaces/s3/client",
 	"storage-interfaces/s3/pallet-s3-registry",
 	"storage-interfaces/s3/primitives",
+
+	# Developer tooling
+	"utils/storage-cli",
 ]
 
 [workspace.package]
@@ -131,6 +134,7 @@ scale-info = { version = "2.11.6", default-features = false, features = [
 ] }
 
 # External dependencies
+anyhow = { version = "1.0" }
 async-trait = "0.1"
 base64 = "0.22"
 bincode = "1.3"
@@ -172,6 +176,7 @@ sp-keystore = { version = "0.46.0", default-features = false }
 tempfile = "3"
 tokio-test = "0.4"
 
+
 [profile.release]
 opt-level = 3
 panic = "unwind"

utils/storage-cli/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "storage-cli"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Operator CLI for scalable Web3 storage (stress-test and ops tooling)"
+
+[dependencies]
+storage-client = { workspace = true }
+clap = { workspace = true, features = ["derive", "env"] }
+sp-core = { workspace = true, features = ["std"] }
+sp-runtime = { workspace = true, features = ["std"] }
+subxt-signer = { workspace = true }
+tokio = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+anyhow = { workspace = true }
+hex = { workspace = true }
+rand = { workspace = true }
+serde = { workspace = true, features = ["derive", "std"] }
+serde_json = { workspace = true, features = ["std"] }
+
+[dev-dependencies]
+tempfile = { workspace = true }

utils/storage-cli/README.md

@@ -0,0 +1,72 @@
+# storage-cli
+
+Storage CLI for [scalable Web3 storage](../../README.md). It consolidates the
+on-chain and off-chain storage operations that were previously scattered across
+`client/examples/*` and ad-hoc scripts into a single ergonomic binary, built on
+top of the [`storage-client`](../../client) SDK.
+
+## Usage
+
+```bash
+cargo run -p storage-cli -- --help
+cargo run -p storage-cli -- stress-test upload --help
+```
+
+### Global flags
+
+| Flag                 | Default                  | Env            | Description                                  |
+| -------------------- | ------------------------ | -------------- | -------------------------------------------- |
+| `--chain-rpc <URL>`  | `ws://127.0.0.1:2222`    | `CHAIN_RPC`    | Parachain RPC WebSocket endpoint.            |
+| `--provider-url <URL>` | `http://127.0.0.1:3333` | `PROVIDER_URL` | Provider node HTTP endpoint.                 |
+| `--suri <SURI>`      | —                        |                | Secret URI for the account, e.g. `//Alice`.  |
+| `--keyfile <FILE>`   | —                        |                | File whose contents are the SURI/seed.       |
+
+`--suri` and `--keyfile` are mutually exclusive; exactly one is required.
+
+## `stress-test upload`
+
+Uploads generated data to every bucket the account **already** has a storage
+agreement with the given provider for.
+
+```bash
+cargo run -p storage-cli -- \
+  --suri //Bob \
+  stress-test upload --provider <PROVIDER_SS58> --size 4096
+```
+
+| Param                        | Default     | Description                                          |
+| ---------------------------- | ----------- | ---------------------------------------------------- |
+| `--provider <ACCOUNT>`       | required    | Provider account (SS58 or `0x`-hex) to target.       |
+| `--max-buckets-to-write <N>` | all buckets | Cap the number of buckets written to.                |
+| `--size <BYTES>`             | `1048576`   | Bytes of generated data to upload per bucket.        |
+
+**Behavior**
+
+1. Derives the account from `--suri`/`--keyfile` and reads its buckets from chain
+   (`MemberBuckets[account]`).
+2. Keeps only buckets that have a `StorageAgreements[bucket][provider]` entry for
+   the given `--provider`.
+3. If none match, it exits with an error — it does **not** create any bucket or
+   agreement.
+4. Uploads generated data to each selected bucket over the provider's HTTP API.
+
+### Required on-chain setup
+
+The command only writes to buckets that already have an agreement, so the agreement
+must exist first. With a chain and provider running (`just start-chain`,
+`just start-provider`), open one — for example via the SDK example:
+
+```bash
+cargo run -p storage-client --example complete_workflow -- \
+  ws://127.0.0.1:2222 http://127.0.0.1:3333 //Bob
+```
+
+That negotiates terms and establishes an agreement, creating a bucket owned by
+`//Bob` with a primary agreement to the provider. Run `stress-test upload` as the
+same account (`--suri //Bob`) targeting that provider.
+
+## Limitations
+
+- **Agreement expiry is not checked.** A bucket is selected based on the presence
+  of a `StorageAgreements[bucket][provider]` entry; expired-but-not-yet-cleared
+  agreements are treated as matches.

utils/storage-cli/src/actions/mod.rs

@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//! Reusable storage actions that scenarios compose.
+//!
+//! Each action owns its `Operation` marker (from [`crate::metrics`]) and a
+//! primitive that performs the operation once and returns a measured
+//! [`crate::metrics::OpOutcome`]. Scenarios such as `stress-test` drive these
+//! primitives under load; a future `read`/`delete` action is a sibling module
+//! here, usable by any scenario without touching the metrics layer.
+
+pub mod upload;

utils/storage-cli/src/actions/upload.rs

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//! Upload action: perform a single off-chain HTTP upload and measure it.
+
+use std::time::Instant;
+
+use storage_client::{ChunkingStrategy, StorageUserClient};
+
+use crate::common::BucketId;
+use crate::metrics::{OpLabels, OpOutcome, Operation};
+
+/// Off-chain HTTP upload to a provider.
+pub struct Upload;
+
+impl Operation for Upload {
+    fn labels(&self) -> OpLabels {
+        OpLabels {
+            verb: "upload",
+            noun_plural: "uploads",
+            past_tense: "uploaded",
+        }
+    }
+}
+
+/// Perform one upload of `payload` to `bucket`, returning the measured outcome.
+pub async fn upload_once(
+    client: &StorageUserClient,
+    bucket: BucketId,
+    payload: &[u8],
+) -> OpOutcome {
+    let started = Instant::now();
+    match client
+        .upload(bucket, payload, ChunkingStrategy::default())
+        .await
+    {
+        Ok(_root) => OpOutcome::success(payload.len(), started.elapsed()),
+        Err(e) => OpOutcome::failure(payload.len(), started.elapsed(), e.to_string()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn upload_labels_match() {
+        let l = Upload.labels();
+        assert_eq!(l.verb, "upload");
+        assert_eq!(l.noun_plural, "uploads");
+        assert_eq!(l.past_tense, "uploaded");
+    }
+}