diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0c94173d..1487dc73 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -188,8 +188,7 @@ jobs: nohup ./target/release/storage-provider-node \ --keyfile /tmp/alice-key --storage-mode inmemory \ --bind-addr 0.0.0.0:3333 --chain-rpc ws://127.0.0.1:2222 \ - --enable-checkpoint-coordinator \ - --disable-auth-i-know-what-i-am-doing > /tmp/provider-inmemory.log 2>&1 & + --enable-checkpoint-coordinator > /tmp/provider-inmemory.log 2>&1 & - name: Wait for provider health uses: ./.github/actions/wait-for-provider-health @@ -320,8 +319,7 @@ jobs: nohup ./target/release/storage-provider-node \ --keyfile /tmp/alice-key --storage-mode inmemory \ --bind-addr 0.0.0.0:3333 --chain-rpc ws://127.0.0.1:2222 \ - --enable-checkpoint-coordinator \ - --disable-auth-i-know-what-i-am-doing > /tmp/provider.log 2>&1 & + --enable-checkpoint-coordinator > /tmp/provider.log 2>&1 & - name: Wait for provider health uses: ./.github/actions/wait-for-provider-health @@ -552,8 +550,7 @@ jobs: nohup ./target/release/storage-provider-node \ --keyfile /tmp/alice-key --storage-mode inmemory \ --bind-addr 0.0.0.0:3333 --chain-rpc ws://127.0.0.1:2222 \ - --enable-checkpoint-coordinator \ - --disable-auth-i-know-what-i-am-doing > /tmp/provider.log 2>&1 & + --enable-checkpoint-coordinator > /tmp/provider.log 2>&1 & - name: Wait for provider health uses: ./.github/actions/wait-for-provider-health @@ -684,8 +681,7 @@ jobs: nohup ./target/release/storage-provider-node \ --keyfile /tmp/alice-key --storage-mode inmemory \ --bind-addr 0.0.0.0:3333 --chain-rpc ws://127.0.0.1:2222 \ - --enable-checkpoint-coordinator \ - --disable-auth-i-know-what-i-am-doing > /tmp/provider.log 2>&1 & + --enable-checkpoint-coordinator > /tmp/provider.log 2>&1 & - name: Wait for provider health uses: ./.github/actions/wait-for-provider-health diff --git a/Cargo.lock b/Cargo.lock index f1240285..4e47383f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7894,7 +7894,6 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" name = "s3-client" version = "0.1.0" dependencies = [ - "bip39", "hex", "reqwest", "s3-primitives", diff --git a/README.md b/README.md index 20afc0a8..d71fd2e8 100644 --- a/README.md +++ b/README.md @@ -248,11 +248,10 @@ The provider node uses environment variables for configuration: ## Example: Basic Upload Flow ```rust -use storage_client::StorageUserClient; +use storage_client::{Signer, StorageUserClient}; // Connect to provider -let mut client = StorageUserClient::new(config); -client.connect_chain().await?; +let client = StorageUserClient::new(config, Signer::dev("alice")?)?; // Upload data (off-chain) let data = b"Hello, decentralized storage!"; diff --git a/client/INTEGRATION.md b/client/INTEGRATION.md index 3745fb79..cd44c209 100644 --- a/client/INTEGRATION.md +++ b/client/INTEGRATION.md @@ -34,7 +34,7 @@ Each specialized client (ProviderClient, AdminClient, etc.) uses the base client ### Basic Setup ```rust -use storage_client::{AdminClient, ClientConfig}; +use storage_client::{AdminClient, ClientConfig, Signer}; #[tokio::main] async fn main() -> Result<(), Box> { @@ -46,12 +46,9 @@ async fn main() -> Result<(), Box> { enable_retries: true, }; - // Create client and connect to chain - let mut client = AdminClient::new(config, "5GrwvaEF...".to_string())?; - client.base.connect_chain().await?; - - // Set signer (for testing) - client.base = client.base.with_dev_signer("alice")?; + // Create client (dev signer for testing) and connect to chain + let mut client = AdminClient::new(config, Signer::dev("alice")?)?; + client.connect().await?; // Now you can make on-chain calls let bucket_id = client.create_bucket(2).await?; @@ -66,16 +63,15 @@ async fn main() -> Result<(), Box> { For production, use actual keypairs instead of dev accounts: ```rust +use storage_client::{AdminClient, Signer}; use subxt_signer::sr25519::Keypair; // Load from seed phrase or keystore let keypair = Keypair::from_uri("//Alice")?; -// Set signer -if let Some(chain_client) = client.base.chain_client.as_mut() { - let client = Arc::make_mut(chain_client); - *client = client.clone().with_signer(keypair); -} +// Any subxt sr25519 keypair converts into a Signer +let mut client = AdminClient::new(config, Signer::from(keypair))?; +client.connect().await?; ``` ### Dynamic Extrinsics @@ -222,8 +218,8 @@ mod tests { }; let mut client = ProviderClient::new(config, "5FHne...".to_string())?; - client.base.connect_chain().await?; - client.base = client.base.with_dev_signer("bob")?; + client.connect().await?; + client.set_signer(Signer::dev("bob")?)?; let result = client.register( "/ip4/127.0.0.1/tcp/3333".to_string(), @@ -254,18 +250,20 @@ cargo test --features integration-tests ### "Not connected to chain" Error -Ensure you call `connect_chain()` before making on-chain calls: +Ensure you call `connect()` before making on-chain calls: ```rust -client.base.connect_chain().await?; +client.connect().await?; ``` ### "No signer configured" Error -Set a signer before submitting extrinsics: +Set a signer before submitting extrinsics. Only `ProviderClient` and +`ChallengerClient` need this step — `AdminClient` and `StorageUserClient` +take their signer at construction: ```rust -client.base = client.base.with_dev_signer("alice")?; +client.set_signer(Signer::dev("alice")?)?; ``` ### Transaction Fails diff --git a/client/README.md b/client/README.md index 71e3e1d7..d44da15e 100644 --- a/client/README.md +++ b/client/README.md @@ -32,19 +32,19 @@ tokio = { version = "1", features = ["full"] } ### Setup -All clients that need on-chain access must connect to the chain and set a signer: +All clients must connect to the chain before on-chain operations. `AdminClient` +and `StorageUserClient` take their signer at construction; `ProviderClient` and +`ChallengerClient` set it after connecting via `set_signer`: ```rust -use storage_client::{AdminClient, ClientConfig}; +use storage_client::{AdminClient, ClientConfig, Signer}; let config = ClientConfig::default(); // ws://localhost:2222 -let mut client = AdminClient::new(config, "5GrwvaEF...".to_string())?; +// Dev account for testing — use a real keypair in production! +let mut client = AdminClient::new(config, Signer::dev("alice")?)?; // Connect to chain -client.base.connect_chain().await?; - -// Set signer (for testing - use proper keypairs in production!) -client.base = client.base.with_dev_signer("alice")?; +client.connect().await?; // Now ready for on-chain operations ``` @@ -56,16 +56,12 @@ See [INTEGRATION.md](INTEGRATION.md) for detailed substrate integration guide. Upload, download, and verify data: ```rust -use storage_client::{StorageUserClient, ChunkingStrategy}; +use storage_client::{ChunkingStrategy, ClientConfig, Signer, StorageUserClient}; #[tokio::main] async fn main() -> Result<(), Box> { - // Create client - let mut client = StorageUserClient::with_defaults()?; - - // Connect to chain for commit operations - client.base.connect_chain().await?; - client.base = client.base.with_dev_signer("alice")?; + // Create client (default config, dev Alice signer) + let client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; // Upload data let data = b"My important data"; @@ -123,11 +119,11 @@ async fn main() -> Result<(), Box> { Create and manage buckets: ```rust -use storage_client::AdminClient; +use storage_client::{AdminClient, Signer}; #[tokio::main] async fn main() -> Result<(), Box> { - let client = AdminClient::with_defaults("5GrwvaEF...".to_string())?; + let client = AdminClient::with_defaults(Signer::dev("alice")?)?; // Create bucket let bucket_id = client.create_bucket(2).await?; // min 2 providers @@ -253,7 +249,7 @@ async fn main() -> Result<(), Box> { All clients can be configured with custom settings: ```rust -use storage_client::ClientConfig; +use storage_client::{ClientConfig, Signer}; let config = ClientConfig { chain_ws_url: "ws://localhost:2222".to_string(), @@ -262,7 +258,7 @@ let config = ClientConfig { enable_retries: true, }; -let client = StorageUserClient::new(config)?; +let client = StorageUserClient::new(config, Signer::from_seed("//Alice")?)?; ``` ### Error Handling @@ -359,7 +355,7 @@ cargo run --example complete_workflow ### Automated Spot-Checking ```rust -let mut client = StorageUserClient::with_defaults()?; +let mut client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; // Perform 10 random spot-checks let (passed, failed) = client.spot_check_batch( diff --git a/client/examples/complete_workflow.rs b/client/examples/complete_workflow.rs index 96eb2002..aca3af9d 100644 --- a/client/examples/complete_workflow.rs +++ b/client/examples/complete_workflow.rs @@ -84,9 +84,8 @@ async fn main() -> Result<(), Box> { // 2. Redeem the signed terms on-chain to open a bucket + primary agreement. println!("Establishing storage agreement on-chain..."); - let mut admin = AdminClient::new(chain_config.clone(), user_ss58.clone())?; + let mut admin = AdminClient::new(chain_config.clone(), user_keypair.clone().into())?; admin.connect().await?; - admin.set_signer(user_keypair)?; let bucket_id = admin .establish_storage_agreement(provider_ss58, signed) .await?; @@ -94,12 +93,12 @@ async fn main() -> Result<(), Box> { // 3. Upload + download against the provider node, verifying integrity. println!("Uploading and downloading data..."); - let user_config = ClientConfig { + let user_client_config = ClientConfig { chain_ws_url: chain_ws.to_string(), provider_urls: vec![provider_url.to_string()], ..Default::default() }; - let user = StorageUserClient::new(user_config)?; + let user = StorageUserClient::new(user_client_config, user_keypair.into())?; let data = b"hello e2e".to_vec(); let data_root = user .upload(bucket_id, &data, ChunkingStrategy::default()) diff --git a/client/examples/register_provider.rs b/client/examples/register_provider.rs index 8fec8dc3..b1857abd 100644 --- a/client/examples/register_provider.rs +++ b/client/examples/register_provider.rs @@ -69,7 +69,7 @@ async fn main() -> Result<(), Box> { let mut provider_client = ProviderClient::new(config, ss58_address.clone())?; provider_client.connect().await?; - provider_client.set_signer(keypair.clone())?; + provider_client.set_signer(keypair.clone().into())?; // Step 1: Register (idempotent — skip if already registered). const STAKE: u128 = 1_000_000_000_000_000; // 1000 tokens (MinProviderStake, 12 decimals) diff --git a/client/src/admin.rs b/client/src/admin.rs index 42241356..8682f6c9 100644 --- a/client/src/admin.rs +++ b/client/src/admin.rs @@ -13,45 +13,43 @@ use crate::agreement::SignedTerms; use crate::base::{BaseClient, ClientConfig, ClientError, ClientResult}; use crate::event_subscription::{EventParser, StorageEvent, StorageProviderEventParser}; use crate::substrate::{extrinsics, storage, SubstrateClient}; +use crate::Signer; use sp_core::H256; +use sp_runtime::AccountId32; use storage_primitives::{BucketId, Commitment, EndAction, Role}; use subxt::ext::scale_value::{Composite, ValueDef, Variant}; /// Client for bucket administrators. pub struct AdminClient { base: BaseClient, - admin_account: String, // Substrate account ID + signer: Signer, } impl AdminClient { - /// Create a new admin client. - pub fn new(config: ClientConfig, admin_account: String) -> ClientResult { + /// Create a new admin client. `signer` submits every extrinsic and + /// identifies the admin account. + pub fn new(config: ClientConfig, signer: Signer) -> ClientResult { Ok(Self { base: BaseClient::new(config)?, - admin_account, + signer, }) } - /// Create with default configuration. - pub fn with_defaults(admin_account: String) -> ClientResult { - Self::new(ClientConfig::default(), admin_account) - } - - /// Connect to the blockchain. Must be called before any on-chain operations. - pub async fn connect(&mut self) -> ClientResult<()> { - self.base.connect_chain().await + /// The admin account: the signer's public key. + fn admin_account(&self) -> AccountId32 { + AccountId32::new(self.signer.keypair().public_key().0) } - /// Set a development signer (alice, bob, charlie, dave, eve, ferdie). - /// Must be called after connect(). - pub fn set_dev_signer(&mut self, name: &str) -> ClientResult<()> { - self.base.set_dev_signer(name) + /// Create with default configuration. + pub fn with_defaults(signer: Signer) -> ClientResult { + Self::new(ClientConfig::default(), signer) } - /// Set a custom keypair signer loaded from a keyfile or seed. - /// Must be called after connect(). - pub fn set_signer(&mut self, signer: subxt_signer::sr25519::Keypair) -> ClientResult<()> { - self.base.set_signer(signer) + /// Connect to the blockchain and install the signer. Must be called before + /// any on-chain operations. + pub async fn connect(&mut self) -> ClientResult<()> { + self.base.connect_chain().await?; + self.base.set_signer(self.signer.clone()) } // ═════════════════════════════════════════════════════════════════════════ @@ -67,9 +65,9 @@ impl AdminClient { /// /// # Example /// ```no_run - /// # use storage_client::{AdminClient, NegotiateRequest, ProviderClient}; + /// # use storage_client::{AdminClient, NegotiateRequest, ProviderClient, Signer}; /// # async fn example() -> Result<(), Box> { - /// let client = AdminClient::with_defaults("5GrwvaEF...".to_string())?; + /// let client = AdminClient::with_defaults(Signer::dev("alice")?)?; /// let signed = ProviderClient::negotiate_terms( /// "http://provider.example:3333", /// &NegotiateRequest { @@ -101,7 +99,7 @@ impl AdminClient { tracing::info!( "Establishing storage agreement with provider {} for owner {} (max_bytes={}, duration={}, nonce={})", provider, - self.admin_account, + self.admin_account(), terms.max_bytes, terms.duration, terms.nonce, @@ -701,14 +699,13 @@ impl AdminClient { /// Get the buckets this admin account is a member of. pub async fn list_my_buckets(&self) -> ClientResult> { let chain = self.base.chain()?; - let admin_account = SubstrateClient::parse_account(&self.admin_account)?; let at = chain .api() .at_current_block() .await .map_err(|e| ClientError::Chain(format!("Failed to get storage: {e}")))?; - let (addr, keys) = storage::member_buckets(&admin_account); + let (addr, keys) = storage::member_buckets(&self.admin_account()); let thunk = at .storage() .try_fetch(addr, keys) diff --git a/client/src/base.rs b/client/src/base.rs index e82265e1..6d765ac2 100644 --- a/client/src/base.rs +++ b/client/src/base.rs @@ -12,7 +12,6 @@ use crate::substrate::SubstrateClient; use reqwest::Client as HttpClient; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use subxt_signer::sr25519::Keypair; use thiserror::Error; /// Client errors. @@ -118,29 +117,10 @@ impl BaseClient { }) } - /// Set a signer for submitting extrinsics (builder pattern). - pub fn with_dev_signer(mut self, name: &str) -> Result { - self.set_dev_signer(name)?; - Ok(self) - } - - /// Set a dev signer for submitting extrinsics (mutable reference). - pub fn set_dev_signer(&mut self, name: &str) -> Result<(), ClientError> { - if let Some(client) = self.chain_client.as_mut() { - let client = Arc::make_mut(client); - *client = client.clone().with_dev_signer(name)?; - Ok(()) - } else { - Err(ClientError::Config( - "Must connect to chain before setting signer".to_string(), - )) - } - } - - /// Set a custom keypair signer for submitting extrinsics. + /// Set the signer for submitting extrinsics. /// /// Must be called after `connect_chain()`. - pub fn set_signer(&mut self, signer: Keypair) -> Result<(), ClientError> { + pub fn set_signer(&mut self, signer: crate::Signer) -> Result<(), ClientError> { if let Some(client) = self.chain_client.as_mut() { let client = Arc::make_mut(client); *client = client.clone().with_signer(signer); diff --git a/client/src/challenger.rs b/client/src/challenger.rs index 97b6a7f6..1035ad3c 100644 --- a/client/src/challenger.rs +++ b/client/src/challenger.rs @@ -42,10 +42,9 @@ impl ChallengerClient { self.base.connect_chain().await } - /// Set a development signer (alice, bob, charlie, dave, eve, ferdie). - /// Must be called after connect(). - pub fn set_dev_signer(&mut self, name: &str) -> ClientResult<()> { - self.base.set_dev_signer(name) + /// Set the signer for submitting extrinsics. Must be called after connect(). + pub fn set_signer(&mut self, signer: crate::Signer) -> ClientResult<()> { + self.base.set_signer(signer) } // ═════════════════════════════════════════════════════════════════════════ diff --git a/client/src/checkpoint.rs b/client/src/checkpoint.rs index a41deefa..0b7c7874 100644 --- a/client/src/checkpoint.rs +++ b/client/src/checkpoint.rs @@ -772,17 +772,11 @@ impl CheckpointManager { } /// Set the signer for submitting transactions. - pub fn with_signer(mut self, signer: subxt_signer::sr25519::Keypair) -> Self { + pub fn with_signer(mut self, signer: crate::Signer) -> Self { self.chain_client = self.chain_client.with_signer(signer); self } - /// Set a development signer (for testing). - pub fn with_dev_signer(mut self, name: &str) -> Result { - self.chain_client = self.chain_client.with_dev_signer(name)?; - Ok(self) - } - // ======================================================================== // Provider Discovery // ======================================================================== @@ -1999,7 +1993,7 @@ impl CheckpointManager { /// // Setup challenger client /// let mut challenger = ChallengerClient::with_defaults("5GrwvaEF...".to_string())?; /// challenger.connect().await?; - /// challenger.set_dev_signer("alice")?; + /// challenger.set_signer(Signer::dev("alice")?)?; /// /// // Execute auto-challenges /// let result = manager.execute_auto_challenges(bucket_id, &challenger, 0.7).await?; @@ -2491,7 +2485,7 @@ pub async fn submit_checkpoint_simple( chain_endpoint: &str, bucket_id: BucketId, provider_endpoints: Vec, - signer_name: &str, + signer: crate::Signer, ) -> CheckpointResult { let manager = match CheckpointManager::new(chain_endpoint, CheckpointConfig::default()).await { Ok(m) => m, @@ -2502,16 +2496,9 @@ pub async fn submit_checkpoint_simple( } }; - let manager = manager.with_providers(provider_endpoints); - - let manager = match manager.with_dev_signer(signer_name) { - Ok(m) => m, - Err(e) => { - return CheckpointResult::TransactionFailed { - error: e.to_string(), - } - } - }; + let manager = manager + .with_providers(provider_endpoints) + .with_signer(signer); manager.submit_checkpoint(bucket_id).await } diff --git a/client/src/lib.rs b/client/src/lib.rs index 8aea7737..791730a1 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -11,10 +11,10 @@ //! ### For Storage Users //! [`StorageUserClient`](storage_user::StorageUserClient) - Upload, download, and verify data //! ```no_run -//! use storage_client::{StorageUserClient, ClientConfig, ChunkingStrategy}; +//! use storage_client::{StorageUserClient, ClientConfig, ChunkingStrategy, Signer}; //! //! # async fn example() -> Result<(), Box> { -//! let client = StorageUserClient::with_defaults()?; +//! let client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; //! //! // Upload data //! let data = b"Hello, decentralized world!"; @@ -51,10 +51,10 @@ //! ### For Bucket Administrators //! [`AdminClient`](admin::AdminClient) - Manage buckets and agreements //! ```no_run -//! use storage_client::{AdminClient, NegotiateRequest, ProviderClient}; +//! use storage_client::{AdminClient, NegotiateRequest, ProviderClient, Signer}; //! //! # async fn example() -> Result<(), Box> { -//! let client = AdminClient::with_defaults("5GrwvaEF...".to_string())?; +//! let client = AdminClient::with_defaults(Signer::dev("alice")?)?; //! //! // 1. Ask the provider node to sign agreement terms over HTTP. The //! // provider allocates the nonce + validity window and signs. @@ -110,6 +110,7 @@ pub mod encryption; pub mod event_subscription; pub mod provider; pub mod scale_decode; +pub mod signer; pub mod storage_user; pub mod substrate; pub mod verification; @@ -144,6 +145,7 @@ pub use event_subscription::{ StorageProviderEventParser, SubscriptionHandle, }; pub use provider::{ProviderClient, ProviderSettings}; +pub use signer::Signer; pub use storage_user::{ CheckpointSignatureResponse, CommitResponse, CommitmentResponse, ExistsResponse, HealthResponse, StorageUserClient, diff --git a/client/src/provider.rs b/client/src/provider.rs index ac9bed36..3049ee10 100644 --- a/client/src/provider.rs +++ b/client/src/provider.rs @@ -42,15 +42,8 @@ impl ProviderClient { self.base.connect_chain().await } - /// Set a development signer (alice, bob, charlie, dave, eve, ferdie). - /// Must be called after connect(). - pub fn set_dev_signer(&mut self, name: &str) -> ClientResult<()> { - self.base.set_dev_signer(name) - } - - /// Set a custom keypair signer loaded from a keyfile or seed. - /// Must be called after connect(). - pub fn set_signer(&mut self, signer: subxt_signer::sr25519::Keypair) -> ClientResult<()> { + /// Set the signer for submitting extrinsics. Must be called after connect(). + pub fn set_signer(&mut self, signer: crate::Signer) -> ClientResult<()> { self.base.set_signer(signer) } diff --git a/client/src/signer.rs b/client/src/signer.rs new file mode 100644 index 00000000..3784ad5b --- /dev/null +++ b/client/src/signer.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Unified signer for provider-auth headers and on-chain extrinsics. + +use crate::{ClientError, ClientResult}; +use std::str::FromStr; +use std::sync::Arc; +use subxt_signer::{ + sr25519::{dev, Keypair}, + SecretUri, +}; + +/// A signer used for both provider-request authentication headers and on-chain +/// extrinsic signing. +/// +/// Build it from a raw [`Keypair`], a secret URI / mnemonic ([`Signer::from_seed`]), +/// or a well-known dev account name ([`Signer::dev`]). Internally it is a +/// reference-counted sr25519 keypair (so cloning is cheap), and it implements +/// [`subxt::tx::Signer`] so it can be handed straight to subxt for extrinsic +/// submission. +#[derive(Clone)] +pub struct Signer(Arc); + +impl Signer { + /// Wrap an existing sr25519 keypair. + pub fn from_keypair(keypair: Keypair) -> Self { + Self(Arc::new(keypair)) + } + + /// Derive from a secret URI or mnemonic, e.g. `"//Alice"`, `""`, + /// or `"//hard/soft"`. + pub fn from_seed(seed: &str) -> ClientResult { + let uri = SecretUri::from_str(seed) + .map_err(|e| ClientError::Config(format!("invalid signer seed: {e}")))?; + Keypair::from_uri(&uri) + .map(|keypair| Self(Arc::new(keypair))) + .map_err(|e| ClientError::Config(format!("invalid signer seed: {e}"))) + } + + /// A well-known dev account by name: `"alice"`..`"ferdie"` (case-insensitive). + pub fn dev(name: &str) -> ClientResult { + let keypair = match name.to_ascii_lowercase().as_str() { + "alice" => dev::alice(), + "bob" => dev::bob(), + "charlie" => dev::charlie(), + "dave" => dev::dave(), + "eve" => dev::eve(), + "ferdie" => dev::ferdie(), + other => return Err(ClientError::Config(format!("unknown dev account: {other}"))), + }; + Ok(Self(Arc::new(keypair))) + } + + /// The underlying keypair, e.g. to build provider-auth headers. + pub fn keypair(&self) -> &Keypair { + &self.0 + } +} + +impl From for Signer { + fn from(keypair: Keypair) -> Self { + Self(Arc::new(keypair)) + } +} + +impl core::fmt::Debug for Signer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // Public key only — never print secret material. + write!(f, "Signer({:?})", self.0.public_key().0) + } +} + +/// Delegates to the inner keypair so a `Signer` can be passed straight to subxt. +impl subxt::tx::Signer for Signer +where + T: subxt::Config, + Keypair: subxt::tx::Signer, +{ + fn account_id(&self) -> T::AccountId { + >::account_id(self.keypair()) + } + + fn sign(&self, signer_payload: &[u8]) -> T::Signature { + >::sign(self.keypair(), signer_payload) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn seed_and_dev_agree_for_alice() { + let seed = Signer::from_seed("//Alice").unwrap(); + let dev = Signer::dev("Alice").unwrap(); + assert_eq!(seed.keypair().public_key().0, dev.keypair().public_key().0); + } + + #[test] + fn unknown_dev_account_errors() { + assert!(Signer::dev("nobody").is_err()); + } +} diff --git a/client/src/storage_user.rs b/client/src/storage_user.rs index 2ad5cf95..056671cf 100644 --- a/client/src/storage_user.rs +++ b/client/src/storage_user.rs @@ -12,7 +12,9 @@ use crate::base::{BaseClient, ChunkingStrategy, ClientConfig, ClientError, ClientResult}; use crate::encryption::{Cipher, EncryptionKey, XChaCha20Poly1305Cipher}; use crate::verification::ClientVerifier; +use crate::Signer; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use provider_negotiation::build_auth_header; use sp_core::H256; use storage_primitives::{blake2_256, BucketId}; @@ -21,21 +23,43 @@ pub struct StorageUserClient { base: BaseClient, verifier: ClientVerifier, cipher: Option>, + auth_signer: Signer, } impl StorageUserClient { /// Create a new storage user client. - pub fn new(config: ClientConfig) -> ClientResult { + /// + /// `auth_signer` authenticates every bucket-scoped provider request; the + /// provider always enforces auth, so it is mandatory. + pub fn new(config: ClientConfig, auth_signer: Signer) -> ClientResult { Ok(Self { base: BaseClient::new(config)?, verifier: ClientVerifier::new(), cipher: None, + auth_signer, }) } - /// Create with default configuration. - pub fn with_defaults() -> ClientResult { - Self::new(ClientConfig::default()) + /// Attach the signed `Authorization` header (`method` = upper-case HTTP verb). + fn sign( + &self, + req: reqwest::RequestBuilder, + method: &str, + bucket_id: BucketId, + ) -> reqwest::RequestBuilder { + let signer = self.auth_signer.keypair(); + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or_default(); + let header = build_auth_header( + &signer.public_key().0, + method, + bucket_id, + timestamp, + |msg| signer.sign(msg).0, + ); + req.header("Authorization", header) } /// Enable client-side encryption with a custom cipher (builder pattern). @@ -68,9 +92,9 @@ impl StorageUserClient { /// /// # Example /// ```no_run - /// # use storage_client::StorageUserClient; + /// # use storage_client::{ClientConfig, Signer, StorageUserClient}; /// # async fn example() -> Result<(), Box> { - /// let client = StorageUserClient::with_defaults()?; + /// let client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; /// let data = b"Hello, decentralized world!"; /// let data_root = client.upload(1, data, Default::default()).await?; /// println!("Uploaded data with root: 0x{}", hex::encode(data_root.as_bytes())); @@ -154,10 +178,10 @@ impl StorageUserClient { /// /// # Example /// ```no_run - /// # use storage_client::StorageUserClient; + /// # use storage_client::{ClientConfig, Signer, StorageUserClient}; /// # use sp_core::H256; /// # async fn example(data_root: H256) -> Result<(), Box> { - /// let client = StorageUserClient::with_defaults()?; + /// let client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; /// let data = client.download(&data_root, 0, 1024).await?; /// println!("Downloaded {} bytes", data.len()); /// # Ok(()) @@ -237,10 +261,10 @@ impl StorageUserClient { /// /// # Example /// ```no_run - /// # use storage_client::StorageUserClient; + /// # use storage_client::{ClientConfig, Signer, StorageUserClient}; /// # use sp_core::H256; /// # async fn example(hash: H256) -> Result<(), Box> { - /// let client = StorageUserClient::with_defaults()?; + /// let client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; /// let (data, children) = client.read_node(&hash).await?; /// println!("Read {} bytes", data.len()); /// # Ok(()) @@ -316,10 +340,10 @@ impl StorageUserClient { /// /// # Example /// ```no_run - /// # use storage_client::StorageUserClient; + /// # use storage_client::{ClientConfig, Signer, StorageUserClient}; /// # use sp_core::H256; /// # async fn example(data_root: H256) -> Result<(), Box> { - /// let client = StorageUserClient::with_defaults()?; + /// let client = StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?; /// let commitment = client.commit(1, vec![data_root], 0u64).await?; /// println!("Committed with MMR root: {}", commitment.mmr_root); /// # Ok(()) @@ -342,13 +366,12 @@ impl StorageUserClient { nonce, }; - let response = self + let req = self .base .http .post(format!("{provider_url}/commit")) - .json(&request) - .send() - .await?; + .json(&request); + let response = self.sign(req, "POST", bucket_id).send().await?; if !response.status().is_success() { return Err(ClientError::Api(format!( @@ -501,13 +524,12 @@ impl StorageUserClient { }), }; - let response = self + let req = self .base .http .put(format!("{provider_url}/node")) - .json(&request) - .send() - .await?; + .json(&request); + let response = self.sign(req, "PUT", bucket_id).send().await?; if !response.status().is_success() { return Err(ClientError::Api(format!( diff --git a/client/src/substrate.rs b/client/src/substrate.rs index 53e2c370..17f778a8 100644 --- a/client/src/substrate.rs +++ b/client/src/substrate.rs @@ -6,15 +6,14 @@ //! the storage parachain. use crate::base::ClientError; +use crate::Signer; use codec::Encode; use futures::StreamExt; use sp_core::H256; use sp_runtime::AccountId32; use std::str::FromStr; -use std::sync::Arc; use storage_primitives::{ChunkLocation, Commitment}; use subxt::{OnlineClient, PolkadotConfig}; -use subxt_signer::sr25519::{dev, Keypair}; pub const PALLET_NAME: &str = "StorageProvider"; @@ -22,7 +21,7 @@ pub const PALLET_NAME: &str = "StorageProvider"; #[derive(Clone)] pub struct SubstrateClient { api: OnlineClient, - signer: Option>, + signer: Option, } impl SubstrateClient { @@ -36,36 +35,20 @@ impl SubstrateClient { } /// Set the signer for this client (for submitting extrinsics). - pub fn with_signer(mut self, signer: Keypair) -> Self { - self.signer = Some(Arc::new(signer)); + pub fn with_signer(mut self, signer: Signer) -> Self { + self.signer = Some(signer); self } - /// Create a client with a development keypair (for testing). - pub fn with_dev_signer(mut self, name: &str) -> Result { - let keypair = match name { - "alice" => dev::alice(), - "bob" => dev::bob(), - "charlie" => dev::charlie(), - "dave" => dev::dave(), - "eve" => dev::eve(), - "ferdie" => dev::ferdie(), - _ => return Err(ClientError::Config(format!("Unknown dev account: {name}"))), - }; - self.signer = Some(Arc::new(keypair)); - Ok(self) - } - /// Get the API client. pub fn api(&self) -> &OnlineClient { &self.api } /// Get the signer if available. - pub fn signer(&self) -> Result<&Keypair, ClientError> { + pub fn signer(&self) -> Result<&Signer, ClientError> { self.signer .as_ref() - .map(|s| s.as_ref()) .ok_or_else(|| ClientError::Config("No signer configured".to_string())) } diff --git a/client/tests/common/mod.rs b/client/tests/common/mod.rs index 773ad2ca..1640883f 100644 --- a/client/tests/common/mod.rs +++ b/client/tests/common/mod.rs @@ -6,12 +6,14 @@ use sp_core::crypto::Ss58Codec; use sp_core::Pair; use sp_runtime::AccountId32; use std::sync::{Arc, OnceLock}; +use std::time::Duration; use storage_client::{ sign_terms, AdminClient, AgreementTermsOf, ChallengerClient, ClientConfig, DiscoveryClient, - ProviderClient, ProviderSettings, StorageUserClient, + ProviderClient, ProviderSettings, Signer, StorageUserClient, }; -use storage_primitives::AgreementTerms; -use storage_provider_node::{create_router, ProviderState, Storage}; +use storage_primitives::{AgreementTerms, Role}; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{create_router, NullNonceStore, ProviderDeps, ProviderState, Storage}; use tokio::net::TcpListener; use tokio::sync::{Mutex, MutexGuard}; @@ -46,8 +48,8 @@ const MIN_STAKE: u128 = 1_000 * 1_000_000_000_000u128; /// Derive the SS58 address for a dev account by name ("alice", "bob", …). /// -/// Avoids hardcoding addresses in tests. Uses the same derivation path that -/// `set_dev_signer` uses internally, so the account and signer always match. +/// Avoids hardcoding addresses in tests. Delegates to [`Signer::dev`], so the +/// account and signer always match. #[allow(dead_code)] pub fn dev_ss58(name: &str) -> String { dev_account(name).to_ss58check() @@ -59,18 +61,8 @@ pub fn dev_ss58(name: &str) -> String { /// storage queries / parser APIs that take `&AccountId32` directly. #[allow(dead_code)] pub fn dev_account(name: &str) -> AccountId32 { - use subxt_signer::sr25519::dev; - - let keypair = match name { - "alice" => dev::alice(), - "bob" => dev::bob(), - "charlie" => dev::charlie(), - "dave" => dev::dave(), - "eve" => dev::eve(), - "ferdie" => dev::ferdie(), - other => panic!("unknown dev account: {other}"), - }; - AccountId32::from(keypair.public_key().0) + let signer = Signer::dev(name).expect("known dev account"); + AccountId32::from(signer.keypair().public_key().0) } // ─── Chain client config ────────────────────────────────────────────────────── @@ -117,7 +109,7 @@ pub async fn chain_setup() -> Option { if provider.connect().await.is_err() { return None; } - provider.set_dev_signer("alice").ok()?; + provider.set_signer(Signer::dev("alice").ok()?).ok()?; let already_registered = matches!( provider.get_provider_info(&dev_account("alice")).await, @@ -154,11 +146,10 @@ pub async fn chain_setup() -> Option { // Alice is both the provider (signs terms) and the bucket owner (redeems). // Nonces must be unique per provider, so we pick one based on the current // block-ish counter — using time-since-epoch nanos so reruns don't collide. - let mut admin = AdminClient::new(chain_config(), alice_ss58.clone()).ok()?; + let mut admin = AdminClient::new(chain_config(), Signer::dev("alice").ok()?).ok()?; if admin.connect().await.is_err() { return None; } - admin.set_dev_signer("alice").ok()?; let nonce = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -193,11 +184,10 @@ pub async fn chain_setup() -> Option { /// Build an `AdminClient` signed by Alice. Returns `None` if the chain is down. #[allow(dead_code)] pub async fn alice_admin() -> Option { - let mut client = AdminClient::new(chain_config(), dev_ss58("alice")).ok()?; + let mut client = AdminClient::new(chain_config(), Signer::dev("alice").ok()?).ok()?; if client.connect().await.is_err() { return None; } - client.set_dev_signer("alice").ok()?; Some(client) } @@ -208,7 +198,7 @@ pub async fn alice_challenger() -> Option { if client.connect().await.is_err() { return None; } - client.set_dev_signer("alice").ok()?; + client.set_signer(Signer::dev("alice").ok()?).ok()?; Some(client) } @@ -219,7 +209,7 @@ pub async fn alice_provider() -> Option { if client.connect().await.is_err() { return None; } - client.set_dev_signer("alice").ok()?; + client.set_signer(Signer::dev("alice").ok()?).ok()?; Some(client) } @@ -239,14 +229,22 @@ pub async fn dev_discovery() -> Option { /// /// Uses `//Alice` as the signing key so endpoints that sign commitments /// (`/commit`, `/commitment`, `/checkpoint/sign`, `/delete`) work end-to-end. +/// `//Alice` is granted `Admin` on every bucket. pub async fn start_test_provider() -> String { - let storage = Arc::new(Storage::new()); - let state = Arc::new( - ProviderState::with_seed(storage, "//Alice") - .expect("//Alice is a valid SURI") - .with_auth_disabled(), - ); - let app = create_router(state); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![( + dev_account("alice"), + Role::Admin, + )])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, "//Alice").expect("//Alice is a valid SURI"); + let app = create_router(Arc::new(state)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -271,13 +269,19 @@ pub async fn start_providers(n: usize) -> Vec { /// Build a `StorageUserClient` pointed at the given provider URL. /// The chain WS URL is a placeholder — these integration tests never touch the chain. +/// +/// Signs provider requests as `//Alice`, matching the `Admin` member granted by +/// [`start_test_provider`], so uploads/commits are authorized. #[allow(dead_code)] pub fn make_client(provider_url: String) -> StorageUserClient { - StorageUserClient::new(ClientConfig { - chain_ws_url: CHAIN_WS.to_string(), - provider_urls: vec![provider_url], - timeout_secs: 10, - enable_retries: false, - }) + StorageUserClient::new( + ClientConfig { + chain_ws_url: CHAIN_WS.to_string(), + provider_urls: vec![provider_url], + timeout_secs: 10, + enable_retries: false, + }, + subxt_signer::sr25519::dev::alice().into(), + ) .expect("ClientConfig should be valid") } diff --git a/docs/design/CHECKPOINT_PROTOCOL.md b/docs/design/CHECKPOINT_PROTOCOL.md index e9488f61..3e6f3cf3 100644 --- a/docs/design/CHECKPOINT_PROTOCOL.md +++ b/docs/design/CHECKPOINT_PROTOCOL.md @@ -645,8 +645,8 @@ With this protocol, the user API becomes simple: let mut fs_client = FileSystemClient::new( "ws://localhost:2222", "http://localhost:3333", -).await? - .with_dev_signer("alice").await?; + Signer::dev("alice")?, +).await?; // Create drive - checkpoints handled automatically based on strategy let drive_id = fs_client.create_drive( @@ -778,13 +778,13 @@ use storage_client::{ ### CheckpointManager Usage ```rust -use storage_client::{CheckpointManager, CheckpointConfig, CheckpointResult}; +use storage_client::{CheckpointManager, CheckpointConfig, CheckpointResult, Signer}; // Create manager let manager = CheckpointManager::new("ws://localhost:2222", CheckpointConfig::default()) .await? .with_providers(vec!["http://localhost:3333".to_string()]) - .with_dev_signer("alice")?; + .with_signer(Signer::dev("alice")?); // Submit checkpoint let result = manager.submit_checkpoint(bucket_id).await; @@ -811,7 +811,7 @@ match result { ```rust use storage_client::{ - CheckpointManager, BatchedCheckpointConfig, BatchedInterval, CheckpointCallback, + CheckpointManager, BatchedCheckpointConfig, BatchedInterval, CheckpointCallback, Signer, }; use std::sync::Arc; @@ -819,7 +819,7 @@ use std::sync::Arc; let manager = Arc::new(CheckpointManager::new("ws://localhost:2222", CheckpointConfig::default()) .await? .with_providers(vec!["http://localhost:3333".to_string()]) - .with_dev_signer("alice")?); + .with_signer(Signer::dev("alice")?)); // Configure batched checkpoints let config = BatchedCheckpointConfig { @@ -930,11 +930,11 @@ if let Some(conflict) = conflict { ### FileSystemClient Integration ```rust -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; -let mut fs_client = FileSystemClient::new("ws://localhost:2222", "http://localhost:3333") - .await? - .with_dev_signer("alice").await?; +let mut fs_client = + FileSystemClient::new("ws://localhost:2222", "http://localhost:3333", Signer::dev("alice")?) + .await?; // Enable automatic checkpoints for a drive fs_client.enable_auto_checkpoints( diff --git a/docs/design/CLIENT_SIDE_ENCRYPTION.md b/docs/design/CLIENT_SIDE_ENCRYPTION.md index 9764a56a..53a53e4e 100644 --- a/docs/design/CLIENT_SIDE_ENCRYPTION.md +++ b/docs/design/CLIENT_SIDE_ENCRYPTION.md @@ -90,7 +90,8 @@ The user provides a 256-bit (32-byte) symmetric key. This is the simplest possib ```rust // Rust let key = EncryptionKey::generate(); -let client = StorageUserClient::with_defaults()?.with_encryption_key(&key); +let client = + StorageUserClient::new(ClientConfig::default(), Signer::dev("alice")?)?.with_encryption_key(&key); ``` ```typescript diff --git a/docs/design/scalable-web3-storage-implementation.md b/docs/design/scalable-web3-storage-implementation.md index 15607c24..a6da96b3 100644 --- a/docs/design/scalable-web3-storage-implementation.md +++ b/docs/design/scalable-web3-storage-implementation.md @@ -1755,10 +1755,6 @@ Rules: `Writer` for uploads/commits, `Admin` for delete and other destructive ops. - The membership cache uses stale-while-revalidate: if the chain is briefly unreachable, cached membership keeps working. -- Authentication is enforced by default. The only way to turn it off is the - deliberately verbose `--disable-auth-i-know-what-i-am-doing` flag, which makes - every endpoint publicly readable and writable. It exists for throwaway local - experiments only and must never be used for a real provider. ### Content-Addressed Storage diff --git a/docs/filesystems/API_REFERENCE.md b/docs/filesystems/API_REFERENCE.md index c1cc4dbc..5d4061ad 100644 --- a/docs/filesystems/API_REFERENCE.md +++ b/docs/filesystems/API_REFERENCE.md @@ -326,12 +326,17 @@ High-level client for file system operations with blockchain integration using ` pub async fn new( chain_endpoint: &str, provider_endpoint: &str, + signer: Signer, ) -> Result ``` **Parameters:** - `chain_endpoint`: Parachain WebSocket endpoint (e.g., `"ws://127.0.0.1:2222"`) - `provider_endpoint`: Storage provider HTTP endpoint (e.g., `"http://127.0.0.1:3333"`) +- `signer`: Signs on-chain extrinsics and provider HTTP requests (the provider + always enforces auth). Build with `Signer::dev("alice")` for testing, or + `Signer::from_seed(...)` / `Signer::from_keypair(...)` for real keys — + never use dev accounts in production. **Returns:** - `Ok(FileSystemClient)`: Client connected to blockchain and provider @@ -339,68 +344,15 @@ pub async fn new( **Example:** ```rust -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; let mut fs_client = FileSystemClient::new( "ws://127.0.0.1:2222", "http://127.0.0.1:3333", + Signer::dev("alice")?, ).await?; ``` -**Note:** After creating the client, you must set a signer using `with_dev_signer()` or `with_signer()`. - ---- - -#### `with_dev_signer` - -Set up a development signer for testing. - -```rust -pub async fn with_dev_signer(self, name: &str) -> Result -``` - -**Parameters:** -- `name`: Dev account name (`"alice"`, `"bob"`, `"charlie"`, `"dave"`, `"eve"`, `"ferdie"`) - -**Returns:** -- `Ok(FileSystemClient)`: Client with dev signer configured -- `Err(FsClientError)`: Invalid account name - -**Example:** -```rust -let fs_client = fs_client - .with_dev_signer("alice") - .await?; -``` - -**Use Case:** Testing and development only. Never use dev accounts in production! - ---- - -#### `with_signer` - -Set up a production signer. - -```rust -pub fn with_signer(self, signer: Keypair) -> Self -``` - -**Parameters:** -- `signer`: SR25519 keypair for signing transactions - -**Returns:** -- `FileSystemClient`: Client with production signer configured - -**Example:** -```rust -use subxt_signer::sr25519::Keypair; - -let keypair = Keypair::from_seed("your secure seed phrase")?; -let fs_client = fs_client.with_signer(keypair); -``` - -**Use Case:** Production deployments with secure key management. - --- ### Drive Operations @@ -1332,7 +1284,7 @@ let node = DirectoryNode::decode(&bytes[..])?; ## Complete Example ```rust -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; use file_system_primitives::CommitStrategy; #[tokio::main] @@ -1341,7 +1293,7 @@ async fn main() -> Result<(), Box> { let mut fs_client = FileSystemClient::new( "ws://127.0.0.1:2222", "http://127.0.0.1:3333", - keypair, + Signer::dev("alice")?, ).await?; // 2. Create drive diff --git a/docs/filesystems/USER_GUIDE.md b/docs/filesystems/USER_GUIDE.md index 7a43355a..524939ff 100644 --- a/docs/filesystems/USER_GUIDE.md +++ b/docs/filesystems/USER_GUIDE.md @@ -60,23 +60,18 @@ file-system-primitives = { path = "path/to/storage-interfaces/file-system/primit ### Initialize Client ```rust -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; // Initialize client with blockchain connection let mut fs_client = FileSystemClient::new( "ws://127.0.0.1:2222", // Parachain WebSocket endpoint "http://127.0.0.1:3333", // Storage provider HTTP endpoint + Signer::dev("alice")?, // Use Alice's dev account ).await?; -// Set up signing (for testing with dev accounts) -fs_client = fs_client - .with_dev_signer("alice") // Use Alice's dev account - .await?; - -// Or use a real keypair for production: -// use subxt_signer::sr25519::Keypair; -// let keypair = Keypair::from_seed("your seed phrase")?; -// fs_client = fs_client.with_signer(keypair).await?; +// Or use a real key for production: +// let signer = Signer::from_seed("your seed phrase")?; +// or: Signer::from_keypair(keypair) ``` **Blockchain Integration:** diff --git a/docs/getting-started/LAYER1_QUICKSTART.md b/docs/getting-started/LAYER1_QUICKSTART.md index c3669d4d..74b6e69a 100644 --- a/docs/getting-started/LAYER1_QUICKSTART.md +++ b/docs/getting-started/LAYER1_QUICKSTART.md @@ -112,7 +112,7 @@ cargo run -p file-system-client --example basic_usage ### Rust SDK Usage ```rust -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; use file_system_primitives::CommitStrategy; #[tokio::main] @@ -121,9 +121,8 @@ async fn main() -> Result<(), Box> { let mut client = FileSystemClient::new( "ws://127.0.0.1:2222", // Parachain WebSocket "http://127.0.0.1:3333", // Provider HTTP - ).await? - .with_dev_signer("alice") // Use Alice for testing - .await?; + Signer::dev("alice")?, // Use Alice for testing + ).await?; // Create a drive (like a mounted filesystem) let drive_id = client.create_drive( @@ -195,15 +194,15 @@ cargo run -p s3-client --example basic_usage ### Rust SDK Usage ```rust -use s3_client::{S3Client, PutObjectOptions}; +use s3_client::{PutObjectOptions, S3Client, Signer}; #[tokio::main] async fn main() -> Result<(), Box> { // Create S3 client let client = S3Client::new( - "ws://127.0.0.1:2222", // Parachain WebSocket - "http://127.0.0.1:3333", // Provider HTTP - "//Alice", // Seed phrase + "ws://127.0.0.1:2222", // Parachain WebSocket + "http://127.0.0.1:3333", // Provider HTTP + Signer::from_seed("//Alice")?, // Signer ).await?; // Create a bucket diff --git a/examples/contracts/StorageMarketplace.sol b/examples/contracts/StorageMarketplace.sol index dc93d5b1..e13819ff 100644 --- a/examples/contracts/StorageMarketplace.sol +++ b/examples/contracts/StorageMarketplace.sol @@ -20,7 +20,10 @@ import "./IWeb3Storage.sol"; /// agreement, returning the new bucket id. The contract records the /// (user, bucket) pair so users can wind their agreement down later /// without touching the chain directly. -/// 3. To wind down, the user calls `endMyAgreement(bucketId, provider)`. +/// 3. Because the contract is the bucket admin, the buyer calls +/// `grantWriter(bucketId, member)` to authorize a real key to upload — +/// the provider enforces auth against the bucket's on-chain membership. +/// 4. To wind down, the user calls `endMyAgreement(bucketId, provider)`. /// Only the original buyer can end their bucket's agreement. /// /// Off-chain ops (uploads, challenges, checkpoints) are unchanged and bypass @@ -36,6 +39,7 @@ contract StorageMarketplace { mapping(uint64 => address) public bucketOwner; event BucketBoughtFor(address indexed user, uint64 indexed bucketId); + event WriterGranted(address indexed user, uint64 indexed bucketId, bytes32 member); event AgreementEnded(address indexed user, uint64 indexed bucketId); /// Buy storage on behalf of `msg.sender` by redeeming provider-signed @@ -59,6 +63,15 @@ contract StorageMarketplace { emit BucketBoughtFor(msg.sender, bucketId); } + /// Grant `member` (a substrate `AccountId32`) `Writer` access on a bucket the + /// caller bought, so that key can sign uploads to the provider. The contract + /// is the bucket admin; only the original buyer may grant. + function grantWriter(uint64 bucketId, bytes32 member) external { + require(bucketOwner[bucketId] == msg.sender, "Not your bucket"); + WEB3_STORAGE.setMember(bucketId, member, 1); // role 1 = Writer + emit WriterGranted(msg.sender, bucketId, member); + } + /// End an agreement on a bucket the caller bought. Pays the provider in /// full (no burn). Provider must be passed as substrate `AccountId32` /// (`bytes32`); see README for how to obtain it. diff --git a/examples/papi/e2e/03-s3-bucket-and-objects.ts b/examples/papi/e2e/03-s3-bucket-and-objects.ts index c5b575e4..d9b2da48 100644 --- a/examples/papi/e2e/03-s3-bucket-and-objects.ts +++ b/examples/papi/e2e/03-s3-bucket-and-objects.ts @@ -48,7 +48,9 @@ async function createS3BucketWithStorage( maxBytes, duration, }); - return createS3Bucket(api, client, name, provider, signed); + // Finalize: putChunk to this bucket reads membership from the provider's + // finalized view, so an in-block create would race it. + return createS3Bucket(api, client, name, provider, signed, { mode: "finalized" }); } async function main() { @@ -89,7 +91,7 @@ async function main() { tests.push({ name: "3.2 Put object metadata", fn: async () => { - const obj = await putChunk(PROVIDER_URL, layer0BucketId, "hello from e2e test"); + const obj = await putChunk(PROVIDER_URL, layer0BucketId, "hello from e2e test", client); await putObjectMetadata(api, client, s3BucketId, "test.txt", obj, "text/plain"); const stored = (await api.query.S3Registry.Objects.getValue( s3BucketId, @@ -104,7 +106,7 @@ async function main() { tests.push({ name: "3.3 Put with user metadata", fn: async () => { - const obj = await putChunk(PROVIDER_URL, layer0BucketId, "data with metadata"); + const obj = await putChunk(PROVIDER_URL, layer0BucketId, "data with metadata", client); await putObjectMetadata(api, client, s3BucketId, "meta.txt", obj, "text/plain", [ ["author", "e2e-test"], ["version", "1"], @@ -186,7 +188,7 @@ async function main() { name2, { maxBytes: maxCapacity, duration } ); - const obj = await putChunk(PROVIDER_URL, l0, "not empty"); + const obj = await putChunk(PROVIDER_URL, l0, "not empty", client); await putObjectMetadata(api, client, bid, "file.txt", obj, "text/plain"); const tx = api.tx.S3Registry.delete_s3_bucket({ s3_bucket_id: bid }); await submitTxExpectFailure(tx, client.signer, "BucketNotEmpty", "3.7"); @@ -260,7 +262,7 @@ async function main() { edgeName, { maxBytes: maxCapacity, duration } ); - const obj = await putChunk(PROVIDER_URL, l0, "deep nested"); + const obj = await putChunk(PROVIDER_URL, l0, "deep nested", client); await putObjectMetadata(api, client, bid, "a/b/c/d.txt", obj, "text/plain"); const stored = await api.query.S3Registry.Objects.getValue( bid, @@ -288,9 +290,9 @@ async function main() { upsertName, { maxBytes: maxCapacity, duration } ); - const obj1 = await putChunk(PROVIDER_URL, l0, "version 1"); + const obj1 = await putChunk(PROVIDER_URL, l0, "version 1", client); await putObjectMetadata(api, client, bid, "file.txt", obj1, "text/plain"); - const obj2 = await putChunk(PROVIDER_URL, l0, "version 2 updated"); + const obj2 = await putChunk(PROVIDER_URL, l0, "version 2 updated", client); await putObjectMetadata(api, client, bid, "file.txt", obj2, "text/plain"); const stored = (await api.query.S3Registry.Objects.getValue( bid, diff --git a/examples/papi/e2e/04-data-upload-and-retrieval.ts b/examples/papi/e2e/04-data-upload-and-retrieval.ts index 7708474b..f36d8e8b 100644 --- a/examples/papi/e2e/04-data-upload-and-retrieval.ts +++ b/examples/papi/e2e/04-data-upload-and-retrieval.ts @@ -16,6 +16,7 @@ import { downloadChunk, ensureProviderRegistered, makeSigner, + signProviderRequest, toHex, uploadChunk, } from "@web3-storage/sdk"; @@ -42,13 +43,24 @@ async function main() { // Create a bucket for upload tests by redeeming provider-signed terms. const maxCapacity = 10_485_760n; // 10 MiB const duration = 100; - const { bucketId } = await negotiateAndEstablish(api, PROVIDER_URL, client, provider, { - maxBytes: maxCapacity, - duration, - }); + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + client, + provider, + { maxBytes: maxCapacity, duration }, + true, // finalize: an immediate provider upload reads finalized membership + ); const tests: Array<{ name: string; fn: () => Promise }> = []; + // The provider always enforces auth on the S3 routes (Reader to GET/HEAD/list, + // Writer to PUT/DELETE). Bob owns this bucket (Admin), so signing each request + // with his keypair satisfies every role. The signed message includes the HTTP + // verb, so it must match the request method exactly. + const s3Auth = (method: string): Record => + signProviderRequest(client.keypair!, method, bucketId); + // ── Different sizes ─────────────────────────────────────────────────────── tests.push({ @@ -56,7 +68,7 @@ async function main() { fn: async () => { const data = "x".repeat(100); const nonce = Number(await api.query.System.Number.getValue()); - const { hash, commit } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const { hash, commit } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); assert.ok(commit.mmr_root, "Should return mmr_root"); const downloaded = await downloadChunk(PROVIDER_URL, hash); assert.deepStrictEqual(downloaded, new TextEncoder().encode(data), "Downloaded data should match"); @@ -68,7 +80,7 @@ async function main() { fn: async () => { const data = randomBytes(64 * 1024); const nonce = Number(await api.query.System.Number.getValue()); - const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); const downloaded = await downloadChunk(PROVIDER_URL, hash); assert.deepStrictEqual(new Uint8Array(downloaded), data, "64KB roundtrip integrity"); }, @@ -79,7 +91,7 @@ async function main() { fn: async () => { const data = randomBytes(256 * 1024); const nonce = Number(await api.query.System.Number.getValue()); - const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); const downloaded = await downloadChunk(PROVIDER_URL, hash); assert.deepStrictEqual(new Uint8Array(downloaded), data, "256KB roundtrip integrity"); }, @@ -92,7 +104,7 @@ async function main() { const nonce = Number(await api.query.System.Number.getValue()); for (let i = 0; i < 5; i++) { const data = `sequential upload #${i} @ ${Date.now()}`; - const result = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const result = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); uploads.push(result); } // Verify leaf indices are incrementing. @@ -122,15 +134,11 @@ async function main() { url.searchParams.set("key", "e2e-test.txt"); const putResp = await fetch(url, { method: "PUT", - headers: { "Content-Type": "text/plain" }, + headers: { "Content-Type": "text/plain", ...s3Auth("PUT") }, body, }); - if (!putResp.ok) { - // S3 endpoints may not be available; skip gracefully. - console.log(" S3 PUT returned %d — skipping (S3 endpoints may not be enabled)", putResp.status); - return; - } - const getResp = await fetch(url); + assert.ok(putResp.ok, `S3 PUT should succeed, got ${putResp.status}`); + const getResp = await fetch(url, { headers: s3Auth("GET") }); assert.ok(getResp.ok, `GET should succeed, got ${getResp.status}`); const downloaded = await getResp.text(); assert.strictEqual(downloaded, body, "S3 GET content should match PUT"); @@ -142,12 +150,7 @@ async function main() { fn: async () => { const url = new URL(`/s3/${bucketId}/object`, PROVIDER_URL); url.searchParams.set("key", "e2e-test.txt"); - const resp = await fetch(url, { method: "HEAD" }); - if (!resp.ok && resp.status === 404) { - console.log(" S3 HEAD returned 404 — S3 endpoints may not be enabled, skipping"); - return; - } - // If it worked, just verify we got headers back. + const resp = await fetch(url, { method: "HEAD", headers: s3Auth("HEAD") }); assert.ok(resp.ok || resp.status === 405, `HEAD should return 200 or 405, got ${resp.status}`); }, }); @@ -156,13 +159,15 @@ async function main() { name: "4.7 S3 list objects", fn: async () => { const url = new URL(`/s3/${bucketId}/objects`, PROVIDER_URL); - const resp = await fetch(url); - if (!resp.ok && resp.status !== 200) { - console.log(" S3 list returned %d — skipping", resp.status); - return; - } + const resp = await fetch(url, { headers: s3Auth("GET") }); + assert.ok(resp.ok, `S3 list should succeed, got ${resp.status}`); const data = await resp.json(); - assert.ok(Array.isArray(data.contents || data), "List should return an array"); + assert.ok(Array.isArray(data.contents), "List should return a contents array"); + // 4.5 PUT this key into the bucket and 4.8 hasn't deleted it yet. + assert.ok( + data.contents.some((o: { key: string }) => o.key === "e2e-test.txt"), + "List should include the object uploaded in 4.5" + ); }, }); @@ -171,17 +176,11 @@ async function main() { fn: async () => { const url = new URL(`/s3/${bucketId}/object`, PROVIDER_URL); url.searchParams.set("key", "e2e-test.txt"); - const resp = await fetch(url, { method: "DELETE" }); - if (!resp.ok && resp.status === 404) { - console.log(" S3 DELETE not available, skipping"); - return; - } + const resp = await fetch(url, { method: "DELETE", headers: s3Auth("DELETE") }); + assert.ok(resp.ok, `S3 DELETE should succeed, got ${resp.status}`); // Verify the object is gone. - const getResp = await fetch(url); - assert.ok( - getResp.status === 404 || !getResp.ok, - `GET after DELETE should 404, got ${getResp.status}` - ); + const getResp = await fetch(url, { headers: s3Auth("GET") }); + assert.strictEqual(getResp.status, 404, `GET after DELETE should 404, got ${getResp.status}`); }, }); @@ -208,7 +207,7 @@ async function main() { fn: async () => { const binary = randomBytes(512); const nonce = Number(await api.query.System.Number.getValue()); - const { hash } = await uploadChunk(PROVIDER_URL, bucketId, binary, nonce); + const { hash } = await uploadChunk(PROVIDER_URL, bucketId, binary, nonce, client); const downloaded = await downloadChunk(PROVIDER_URL, hash); assert.deepStrictEqual(new Uint8Array(downloaded), binary, "Binary roundtrip should match"); }, @@ -219,8 +218,8 @@ async function main() { fn: async () => { const data = "duplicate content for e2e"; const nonce = Number(await api.query.System.Number.getValue()); - const first = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); - const second = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const first = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); + const second = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); assert.strictEqual(first.hash, second.hash, "Same data should produce same hash"); assert.notStrictEqual( first.commit.leaf_indices[0], @@ -237,7 +236,7 @@ async function main() { const bytes = new TextEncoder().encode(data); const expectedHash = toHex(blake2b256(bytes)); const nonce = Number(await api.query.System.Number.getValue()); - const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, client); assert.strictEqual(hash, expectedHash, "Provider hash should match local blake2-256"); }, }); diff --git a/examples/papi/e2e/05-checkpoint-and-challenges.ts b/examples/papi/e2e/05-checkpoint-and-challenges.ts index ada5fc3e..d66b6cbe 100644 --- a/examples/papi/e2e/05-checkpoint-and-challenges.ts +++ b/examples/papi/e2e/05-checkpoint-and-challenges.ts @@ -51,14 +51,18 @@ async function main() { // Create a bucket + agreement + upload data for checkpoint tests. const maxBytes = 1_048_576n; const duration = 200; - const { bucketId } = await negotiateAndEstablish(api, PROVIDER_URL, client, provider, { - maxBytes, - duration, - }); + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + client, + provider, + { maxBytes, duration }, + true, // finalize: an immediate provider upload reads finalized membership + ); const payload = `checkpoint-test @ ${Date.now()}`; const uploadNonce = Number(await api.query.System.Number.getValue()); - const upload = await uploadChunk(PROVIDER_URL, bucketId, payload, uploadNonce); + const upload = await uploadChunk(PROVIDER_URL, bucketId, payload, uploadNonce, client); const uploadInfo = { leafIndex: upload.commit.leaf_indices[0], mmrRoot: upload.commit.mmr_root, diff --git a/examples/papi/e2e/10-edge-cases-and-adversarial.ts b/examples/papi/e2e/10-edge-cases-and-adversarial.ts index 45cb51c6..4686bdc1 100644 --- a/examples/papi/e2e/10-edge-cases-and-adversarial.ts +++ b/examples/papi/e2e/10-edge-cases-and-adversarial.ts @@ -5,7 +5,8 @@ * * Tests: balance accounting, capacity tracking, frozen buckets, * concurrent operations, data integrity, and access-control rejections - * (non-admin writes, freeze without a checkpoint). + * (non-admin writes, freeze without a checkpoint, unsigned/non-member + * provider uploads). * * Usage: node e2e/10-edge-cases-and-adversarial.js [chain_ws] [provider_url] */ @@ -21,6 +22,7 @@ import { makeSigner, READ_OPTS, setMember, + signProviderRequest, submitClientCheckpoint, toHex, uploadChunk, @@ -126,13 +128,17 @@ async function main() { tests.push({ name: "10.5 Freeze is irreversible", fn: async () => { - const { bucketId } = await negotiateAndEstablish(api, PROVIDER_URL, bob, provider, { - maxBytes, - duration: 100, - }); + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + bob, + provider, + { maxBytes, duration: 100 }, + true, // finalize: immediate upload reads finalized membership + ); // freeze_bucket requires a snapshot (checkpoint) to exist. const nonce = Number(await api.query.System.Number.getValue()); - await uploadChunk(PROVIDER_URL, bucketId, "data for snapshot", nonce); + await uploadChunk(PROVIDER_URL, bucketId, "data for snapshot", nonce, bob); const ck = await fetchCheckpointSignature(PROVIDER_URL, bucketId, nonce); await submitClientCheckpoint(api, bob, provider, bucketId, ck); await freezeBucket(api, bob, bucketId); @@ -148,13 +154,17 @@ async function main() { tests.push({ name: "10.6 Checkpoint after freeze", fn: async () => { - const { bucketId } = await negotiateAndEstablish(api, PROVIDER_URL, bob, provider, { - maxBytes, - duration: 100, - }); + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + bob, + provider, + { maxBytes, duration: 100 }, + true, // finalize: immediate upload reads finalized membership + ); // Upload some data. const nonce1 = Number(await api.query.System.Number.getValue()); - await uploadChunk(PROVIDER_URL, bucketId, "pre-freeze data", nonce1); + await uploadChunk(PROVIDER_URL, bucketId, "pre-freeze data", nonce1, bob); // Checkpoint before freeze. const ck1 = await fetchCheckpointSignature(PROVIDER_URL, bucketId, nonce1); await submitClientCheckpoint(api, bob, provider, bucketId, ck1); @@ -162,7 +172,7 @@ async function main() { await freezeBucket(api, bob, bucketId); // Upload more data. const nonce2 = Number(await api.query.System.Number.getValue()); - await uploadChunk(PROVIDER_URL, bucketId, "post-freeze data", nonce2); + await uploadChunk(PROVIDER_URL, bucketId, "post-freeze data", nonce2, bob); // Checkpoint after freeze — should still work (captures frozen_start_seq). const ck2 = await fetchCheckpointSignature(PROVIDER_URL, bucketId, nonce2); const result = await submitClientCheckpoint(api, bob, provider, bucketId, ck2); @@ -206,15 +216,19 @@ async function main() { tests.push({ name: "10.8 Upload verify blake2-256", fn: async () => { - const { bucketId } = await negotiateAndEstablish(api, PROVIDER_URL, bob, provider, { - maxBytes, - duration: 100, - }); + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + bob, + provider, + { maxBytes, duration: 100 }, + true, // finalize: immediate upload reads finalized membership + ); const data = "integrity check data for blake2-256"; const bytes = new TextEncoder().encode(data); const expectedHash = toHex(blake2b256(bytes)); const nonce = Number(await api.query.System.Number.getValue()); - const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const { hash } = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, bob); assert.strictEqual(hash, expectedHash, "Provider hash should match local blake2-256"); }, }); @@ -222,14 +236,18 @@ async function main() { tests.push({ name: "10.9 Identical content → same hash, different MMR leaves", fn: async () => { - const { bucketId } = await negotiateAndEstablish(api, PROVIDER_URL, bob, provider, { - maxBytes, - duration: 100, - }); + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + bob, + provider, + { maxBytes, duration: 100 }, + true, // finalize: immediate upload reads finalized membership + ); const data = "identical content for dedup test"; const nonce = Number(await api.query.System.Number.getValue()); - const r1 = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); - const r2 = await uploadChunk(PROVIDER_URL, bucketId, data, nonce); + const r1 = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, bob); + const r2 = await uploadChunk(PROVIDER_URL, bucketId, data, nonce, bob); assert.strictEqual(r1.hash, r2.hash, "Hashes should match for identical content"); assert.notStrictEqual( r1.commit.leaf_indices[0], @@ -285,6 +303,46 @@ async function main() { }, }); + tests.push({ + name: "10.13 Provider rejects unsigned and non-member uploads", + fn: async () => { + const { bucketId } = await negotiateAndEstablish( + api, + PROVIDER_URL, + bob, + provider, + { maxBytes, duration: 100 }, + true, // finalize: the provider resolves membership from finalized state + ); + const bytes = new TextEncoder().encode("must not land"); + const body = JSON.stringify({ + bucket_id: Number(bucketId), + hash: toHex(blake2b256(bytes)), + data: Buffer.from(bytes).toString("base64"), + children: null, + }); + + // No Authorization header at all → 401. + const unsigned = await fetch(`${PROVIDER_URL}/node`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body, + }); + assert.strictEqual(unsigned.status, 401, "unsigned upload must be rejected with 401"); + + // Valid signature from Eve, who holds no role on the bucket → 403. + const eveSigned = await fetch(`${PROVIDER_URL}/node`, { + method: "PUT", + headers: { + "Content-Type": "application/json", + ...signProviderRequest(eve.keypair, "PUT", bucketId), + }, + body, + }); + assert.strictEqual(eveSigned.status, 403, "non-member upload must be rejected with 403"); + }, + }); + await runSuite("10 — Edge Cases & Adversarial", tests, { api, papi }); try { diff --git a/examples/papi/e2e/helpers.ts b/examples/papi/e2e/helpers.ts index 803e0e93..d8fb5148 100644 --- a/examples/papi/e2e/helpers.ts +++ b/examples/papi/e2e/helpers.ts @@ -247,8 +247,18 @@ export async function negotiateAndEstablish( owner: ChainSigner, provider: ChainSigner, opts: NegotiateOpts, + // Finalize the establish when an immediate provider upload follows: the + // provider reads bucket membership from finalized state, so an in-block + // establish would race it (see tx.ts — "finalized" for cross-referenced effects). + finalized = false, ): Promise<{ bucketId: bigint; signed: SignedTerms }> { const signed = await negotiateSigned(api, providerUrl, owner, provider, opts); - const { bucketId } = await establishStorageAgreement(api, owner, provider, signed); + const { bucketId } = await establishStorageAgreement( + api, + owner, + provider, + signed, + finalized ? { mode: "finalized" } : {}, + ); return { bucketId, signed }; } diff --git a/examples/papi/full-flow.ts b/examples/papi/full-flow.ts index b9b86673..94ce096e 100644 --- a/examples/papi/full-flow.ts +++ b/examples/papi/full-flow.ts @@ -65,7 +65,10 @@ async function setupAgreement( provider: ChainSigner ): Promise { const maxBytes = 1_073_741_824n; // 1 GiB - const duration = 15; + // Must outlast Step 6's checkpoint challenge, which requires a live agreement: + // the finalized establish + off-chain challenge land it at ~block 16, so 15 + // expired too soon. 30 leaves ~14 blocks of margin for finality jitter. + const duration = 30; console.log( " Negotiating signed terms with provider (max_bytes=%s, duration=%d)...", maxBytes, @@ -85,15 +88,19 @@ async function setupAgreement( signed.terms.valid_until ); console.log(" Redeeming on-chain via establish_storage_agreement..."); - const { bucketId } = await establishStorageAgreement(api, client, provider, signed); + // Finalize: the immediate upload reads bucket membership from the provider's + // finalized view, so an in-block establish would race it. + const { bucketId } = await establishStorageAgreement(api, client, provider, signed, { + mode: "finalized", + }); console.log(" Bucket %s opened with primary agreement", bucketId); return bucketId; } -async function uploadAndVerify(api: ParachainApi, bucketId: bigint) { +async function uploadAndVerify(api: ParachainApi, bucketId: bigint, signer: ChainSigner) { const payload = `Hello, Web3 Storage! [${new Date().toISOString()}] provider=${PROVIDER_SEED}`; const nonce = Number(await api.query.System.Number.getValue()); - const { hash, data, commit } = await uploadChunk(PROVIDER_URL, bucketId, payload, nonce); + const { hash, data, commit } = await uploadChunk(PROVIDER_URL, bucketId, payload, nonce, signer); console.log(" Uploaded %d bytes, mmr_root=%s", data.length, commit.mmr_root); const downloaded = await downloadChunk(PROVIDER_URL, hash); @@ -179,7 +186,7 @@ async function main() { const bucketId = await setupAgreement(api, PROVIDER_URL, client, provider); console.log("\n=== Step 2: Upload data ==="); - const upload = await uploadAndVerify(api, bucketId); + const upload = await uploadAndVerify(api, bucketId, client); console.log("\n=== Step 3: Off-chain challenge ==="); const offchainId = await challengeOffchain( diff --git a/examples/papi/sc-coverage.ts b/examples/papi/sc-coverage.ts index 7ac9483c..635afd43 100644 --- a/examples/papi/sc-coverage.ts +++ b/examples/papi/sc-coverage.ts @@ -254,11 +254,17 @@ async function main() { // sometimes the rpc returns old data, and the tests run sequentially, so // bump the expected NextBucketId by hand for the post-call assertion. nextBucketBefore += 1n; - r = await callPrecompile(api, client, WEB3_STORAGE_ADDR, iWeb3, "establishStorageAgreement", [ - toHex(providerBytes32), - signedC.terms, - signedC.signature, - ]); + // Finalize: the upload below reads bucketC membership from the provider's + // finalized view, so an in-block establish would race it. + r = await callPrecompile( + api, + client, + WEB3_STORAGE_ADDR, + iWeb3, + "establishStorageAgreement", + [toHex(providerBytes32), signedC.terms, signedC.signature], + { finalized: true }, + ); const createdC = assertEvent(r.events, "StorageProvider", "BucketCreated", "establishStorageAgreement"); const bucketC = createdC.bucket_id; assert.strictEqual(bucketC, nextBucketBefore); @@ -266,7 +272,7 @@ async function main() { console.log(" preconditions: uploadChunk + submitClientCheckpoint"); const uploadNonce = Number(await api.query.System.Number.getValue()); - const upload = await uploadChunk(providerUrl, bucketC, "coverage-test", uploadNonce); + const upload = await uploadChunk(providerUrl, bucketC, "coverage-test", uploadNonce, client); const ckNonce = Number(await api.query.System.Number.getValue()); const ck = await fetchCheckpointSignature(providerUrl, bucketC, ckNonce); await submitClientCheckpoint(api, client, provider, bucketC, ck); diff --git a/examples/papi/sc-flow.ts b/examples/papi/sc-flow.ts index 0fd02e40..f85792e7 100644 --- a/examples/papi/sc-flow.ts +++ b/examples/papi/sc-flow.ts @@ -164,11 +164,19 @@ async function main() { assert(bought, "BucketBoughtFor not emitted by contract"); console.log(" BucketBoughtFor user:", (bought.args as any).user); - // 5) Off-chain ops are unchanged — they bypass the contract entirely. - console.log("\n[5/6] Off-chain upload + challenge round-trip…"); + // 5) The bucket admin is the contract's keyless account, so the contract + // grants the client's key Writer access before it can sign uploads. + console.log("\n[5/6] grantWriter(client) + off-chain upload + challenge round-trip…"); + const grantData = encodeCall(abi, "grantWriter", [ + bucketId, + toHex(client.publicKey), + ]); + // Finalize: the upload below reads the just-granted Writer membership from + // the provider's finalized view, so an in-block grant would race it. + await callContract(api, client, deployed.addressBytes, grantData, { finalized: true }); const payload = `Hello via SC! ${new Date().toISOString()}`; const uploadNonce = Number(await api.query.System.Number.getValue()); - const upload = await uploadChunk(PROVIDER_URL, bucketId, payload, uploadNonce); + const upload = await uploadChunk(PROVIDER_URL, bucketId, payload, uploadNonce, client); const downloaded = await downloadChunk(PROVIDER_URL, upload.hash); assert.deepStrictEqual( downloaded, diff --git a/justfile b/justfile index c1317f16..41f58954 100644 --- a/justfile +++ b/justfile @@ -188,7 +188,7 @@ start-e2e-chain RUNTIME="web3-storage-paseo": check # just start-provider # inmemory, //Alice key, port 3333, auth enforced # just start-provider MODE=disk PORT=3334 # disk storage on port 3334 # just start-provider KEYFILE=/path/to/seed MODE=disk # custom key from file -start-provider MODE="inmemory" PORT=PROVIDER_PORT STORAGE_PATH="./provider-data" KEYFILE="" DISABLE_AUTH="false": build-provider +start-provider MODE="inmemory" PORT=PROVIDER_PORT STORAGE_PATH="./provider-data" KEYFILE="": build-provider #!/usr/bin/env bash set -euo pipefail echo "" @@ -200,9 +200,6 @@ start-provider MODE="inmemory" PORT=PROVIDER_PORT STORAGE_PATH="./provider-data" if [ "{{MODE}}" = "disk" ]; then EXTRA_ARGS="--storage-path {{STORAGE_PATH}}" fi - if [ "{{DISABLE_AUTH}}" = "true" ]; then - EXTRA_ARGS="$EXTRA_ARGS --disable-auth-i-know-what-i-am-doing" - fi if [ -n "{{KEYFILE}}" ]; then KEY_ARGS="--keyfile {{KEYFILE}}" else diff --git a/packages/layer0/src/provider-http.ts b/packages/layer0/src/provider-http.ts index 699d0cdc..b227e426 100644 --- a/packages/layer0/src/provider-http.ts +++ b/packages/layer0/src/provider-http.ts @@ -7,9 +7,15 @@ */ import { blake2b256 } from "@polkadot-labs/hdkd-helpers"; -import { base64ToBytes, bytesToBase64 } from "@web3-storage/core"; +import { + base64ToBytes, + bytesToBase64, + signProviderRequest, + type SigningKeypair, +} from "@web3-storage/core"; import { asHex, toHex, type ParachainApi } from "./address.js"; +import type { ChainSigner } from "./signers.js"; import { READ_OPTS } from "./tx.js"; export interface ProviderFetchOpts { @@ -22,6 +28,12 @@ export interface ProviderFetchOpts { */ params?: Record; body?: unknown; + /** + * When set, attach the signed `Authorization` header the provider verifies + * (`provider-node/src/auth.rs`) for a bucket-scoped, role-gated request + * (`PUT /node`, `POST /commit`, …). Omit for public/read endpoints. + */ + sign?: { keypair: SigningKeypair; bucketId: bigint | number }; } export async function providerFetch( @@ -33,9 +45,16 @@ export async function providerFetch( if (opts.params) { for (const [k, v] of Object.entries(opts.params)) url.searchParams.set(k, String(v)); } + const method = opts.method || "GET"; + const headers: Record = {}; + if (opts.body) headers["Content-Type"] = "application/json"; + // auth.rs reconstructs the message from the upper-case HTTP verb; signing with + // anything else would fail verification. + if (opts.sign) + Object.assign(headers, signProviderRequest(opts.sign.keypair, method.toUpperCase(), opts.sign.bucketId)); const resp = await fetch(url, { - method: opts.method || "GET", - headers: opts.body ? { "Content-Type": "application/json" } : undefined, + method, + headers: Object.keys(headers).length ? headers : undefined, body: opts.body ? JSON.stringify(opts.body) : undefined, }); if (!resp.ok) throw new Error(`${path}: ${resp.status} ${await resp.text()}`); @@ -84,12 +103,17 @@ export interface PutChunkResult { * PUT a single chunk to the provider without requesting an MMR commitment. * Suitable for S3-style object uploads where the Layer 1 metadata records * the CID itself and no Layer 0 checkpoint follows immediately. + * + * `signer` authenticates the `PUT /node` request; it must hold a Writer/Admin + * role on `bucketId` (the provider always enforces this). */ export async function putChunk( providerUrl: string, bucketId: bigint | number, data: Uint8Array | string, + signer: ChainSigner, ): Promise { + const sign = { keypair: signer.keypair, bucketId }; const bytes = data instanceof Uint8Array ? data : new TextEncoder().encode(data); const cid = blake2b256(bytes); const hash = toHex(cid); @@ -101,6 +125,7 @@ export async function putChunk( data: bytesToBase64(bytes), children: null, }, + sign, }); return { hash, cid, size: BigInt(bytes.length), data: bytes }; } @@ -109,13 +134,19 @@ export async function putChunk( * PUT a chunk to the provider and request an MMR commitment. Returns the * chunk hash, original bytes, and the /commit response (mmr_root, * leaf_indices, start_seq, provider_signature). + * + * `signer` authenticates the `PUT /node` and `POST /commit` requests; it must + * hold a Writer/Admin role on `bucketId` (the provider always enforces this). + * `nonce` is threaded into `/commit` for the pallet's signature-recency check. */ export async function uploadChunk( providerUrl: string, bucketId: bigint | number, data: Uint8Array | string, nonce: bigint | number, + signer: ChainSigner, ): Promise<{ hash: string; data: Uint8Array; commit: any }> { + const sign = { keypair: signer.keypair, bucketId }; const bytes = data instanceof Uint8Array ? data : new TextEncoder().encode(data); const hash = toHex(blake2b256(bytes)); await providerFetch(providerUrl, "/node", { @@ -126,6 +157,7 @@ export async function uploadChunk( data: bytesToBase64(bytes), children: null, }, + sign, }); // `nonce` is the block at which the caller intends to submit the extrinsic // consuming the resulting `provider_signature`. The pallet rejects signatures @@ -133,6 +165,7 @@ export async function uploadChunk( const commit = await providerFetch(providerUrl, "/commit", { method: "POST", body: { bucket_id: Number(bucketId), data_roots: [hash], nonce: Number(nonce) }, + sign, }); return { hash, data: bytes, commit }; } diff --git a/packages/layer0/src/signers.ts b/packages/layer0/src/signers.ts index a9e28249..ce3e53c3 100644 --- a/packages/layer0/src/signers.ts +++ b/packages/layer0/src/signers.ts @@ -34,12 +34,10 @@ export interface ChainSigner { /** Dev-account name, when this is one of the well-known dev signers. */ name?: DevAccountName; /** - * Raw keypair, when locally derived. Needed for provider request signing + * Raw keypair backing the signer. Required for provider request signing * (raw sr25519 over the auth message — PolkadotSigner.signBytes may wrap). - * Wallet-extension signers cannot provide this; signed provider requests - * are then unavailable (unauthenticated providers still work). */ - keypair?: Keypair; + keypair: Keypair; } /** diff --git a/packages/layer1/src/base-client.ts b/packages/layer1/src/base-client.ts new file mode 100644 index 00000000..27b056de --- /dev/null +++ b/packages/layer1/src/base-client.ts @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared plumbing for the layer-1 clients (FileSystemClient, S3Client): + * signer state, provider-URL resolution, fetch injection, and the chain + * read/submit defaults. The subclasses add only their pallet + HTTP surface. + */ + +import { signProviderRequest, type HttpFetchOpts } from "@web3-storage/core"; +import type { + ChainSigner, + ParachainApi, + SubmitOpts, + TxStatusListener, +} from "@web3-storage/layer0"; + +import { ProviderUrlResolver } from "./provider-url.js"; + +export interface Layer1ClientOptions { + api: ParachainApi; + signer?: ChainSigner | null; + /** Explicit provider URL (dev/tests) — skips on-chain resolution. */ + providerUrl?: string; + /** Injection point for unit tests. */ + fetch?: typeof fetch; + /** Tx progress listener. Default null (silent) — apps drive their own UI. */ + onStatus?: TxStatusListener | null; + /** + * Read view for chain lookups. Defaults to "finalized" (UI-grade, + * reorg-safe). Tests/examples pass READ_OPTS ({at: "best"}) to match their + * in-block submission semantics. + */ + readOpts?: { at: "best" | "finalized" }; + /** + * Submission doneness. Defaults to "finalized" (UI-grade). Tests/examples + * pass "best" for speed. + */ + submitMode?: "best" | "finalized"; +} + +export abstract class Layer1Client { + protected readonly api: ParachainApi; + private signer: ChainSigner | null; + protected readonly providers: ProviderUrlResolver; + protected readonly fetchOpts: HttpFetchOpts; + protected readonly onStatus: TxStatusListener | null; + protected readonly readOpts: { at: "best" | "finalized" }; + protected readonly submitMode: "best" | "finalized"; + protected readonly creationUrlOverride?: string; + + constructor(opts: Layer1ClientOptions) { + this.api = opts.api; + this.signer = opts.signer ?? null; + this.readOpts = opts.readOpts ?? { at: "finalized" }; + this.submitMode = opts.submitMode ?? "finalized"; + this.providers = new ProviderUrlResolver(opts.api, opts.providerUrl, this.readOpts); + this.creationUrlOverride = opts.providerUrl; + this.fetchOpts = opts.fetch ? { fetchImpl: opts.fetch } : {}; + this.onStatus = opts.onStatus ?? null; + } + + setSigner(signer: ChainSigner | null): void { + this.signer = signer; + } + + protected requireSigner(): ChainSigner { + if (!this.signer) throw new Error("Signer not set"); + return this.signer; + } + + protected submitOpts(): SubmitOpts { + return { mode: this.submitMode, retryStale: 0, onStatus: this.onStatus }; + } + + protected authHeaders(method: string, bucketId: bigint): Record { + return signProviderRequest(this.requireSigner().keypair, method, bucketId); + } +} diff --git a/packages/layer1/src/fs/client.ts b/packages/layer1/src/fs/client.ts index 49f56026..40750dc0 100644 --- a/packages/layer1/src/fs/client.ts +++ b/packages/layer1/src/fs/client.ts @@ -5,8 +5,8 @@ * surface. Chain ops delegate to the layer-0 pallet wrappers (silent, no * auto-retry, finalized submission + finalized reads by default — UI-grade, * reorg-safe; tests/examples opt into in-block/best via readOpts/submitMode); - * HTTP ops go through core's retrying fetch and are signed when the signer - * has a raw keypair. + * HTTP ops go through core's retrying fetch and are signed with the signer's + * raw keypair, which the provider always requires. * * Verification: `downloadByCid` is verified (single chunk — its hash IS the * CID; the layer-0 downloadChunk throws CidMismatchError). Path-based @@ -15,11 +15,7 @@ * (Rust-client parity) — tracked separately. */ -import { - httpFetch, - signProviderRequest, - type HttpFetchOpts, -} from "@web3-storage/core"; +import { httpFetch } from "@web3-storage/core"; import { createDrive as createDriveTx, deleteDrive as deleteDriveTx, @@ -28,18 +24,11 @@ import { setMember as setMemberTx, shareDrive as shareDriveTx, unshareDrive as unshareDriveTx, - type ChainSigner, - type ParachainApi, - type SubmitOpts, - type TxStatusListener, type WaitOpts, } from "@web3-storage/layer0"; -import { - ProviderUrlResolver, - resolveBucketProviders, - resolveCreationTerms, -} from "../provider-url.js"; +import { Layer1Client, type Layer1ClientOptions } from "../base-client.js"; +import { resolveBucketProviders, resolveCreationTerms } from "../provider-url.js"; import type { BucketMember, CheckpointDuty, @@ -53,27 +42,7 @@ import type { UploadResult, } from "./types.js"; -export interface FileSystemClientOptions { - api: ParachainApi; - signer?: ChainSigner | null; - /** Explicit provider URL (dev/tests) — skips on-chain resolution. */ - providerUrl?: string; - /** Injection point for unit tests. */ - fetch?: typeof fetch; - /** Tx progress listener. Default null (silent) — apps drive their own UI. */ - onStatus?: TxStatusListener | null; - /** - * Read view for chain lookups. Defaults to "finalized" (UI-grade, - * reorg-safe). Tests/examples pass READ_OPTS ({at: "best"}) to match their - * in-block submission semantics. - */ - readOpts?: { at: "best" | "finalized" }; - /** - * Submission doneness. Defaults to "finalized" (UI-grade). Tests/examples - * pass "best" for speed. - */ - submitMode?: "best" | "finalized"; -} +export type FileSystemClientOptions = Layer1ClientOptions; function decodeName(name: Uint8Array | string | undefined | null): string | null { if (name == null) return null; @@ -85,45 +54,7 @@ function decodeName(name: Uint8Array | string | undefined | null): string | null } } -export class FileSystemClient { - private readonly api: ParachainApi; - private signer: ChainSigner | null; - private readonly providers: ProviderUrlResolver; - private readonly fetchOpts: HttpFetchOpts; - private readonly onStatus: TxStatusListener | null; - private readonly readOpts: { at: "best" | "finalized" }; - private readonly submitMode: "best" | "finalized"; - private readonly creationUrlOverride?: string; - - constructor(opts: FileSystemClientOptions) { - this.api = opts.api; - this.signer = opts.signer ?? null; - this.readOpts = opts.readOpts ?? { at: "finalized" }; - this.submitMode = opts.submitMode ?? "finalized"; - this.providers = new ProviderUrlResolver(opts.api, opts.providerUrl, this.readOpts); - this.creationUrlOverride = opts.providerUrl; - this.fetchOpts = opts.fetch ? { fetchImpl: opts.fetch } : {}; - this.onStatus = opts.onStatus ?? null; - } - - setSigner(signer: ChainSigner | null): void { - this.signer = signer; - } - - private requireSigner(): ChainSigner { - if (!this.signer) throw new Error("Signer not set"); - return this.signer; - } - - private submitOpts(): SubmitOpts { - return { mode: this.submitMode, retryStale: 0, onStatus: this.onStatus }; - } - - private authHeaders(method: string, bucketId: bigint): Record { - const kp = this.signer?.keypair; - return kp ? signProviderRequest(kp, method, bucketId) : {}; - } - +export class FileSystemClient extends Layer1Client { // ── Drive chain ops ───────────────────────────────────────────────────── /** diff --git a/packages/layer1/src/s3/client.ts b/packages/layer1/src/s3/client.ts index 9e3edd58..4a2a8fc3 100644 --- a/packages/layer1/src/s3/client.ts +++ b/packages/layer1/src/s3/client.ts @@ -6,7 +6,7 @@ * wrappers (silent, no auto-retry, finalized submission + finalized reads by * default — UI-grade; tests/examples opt into in-block/best via * readOpts/submitMode); HTTP ops go through core's retrying fetch and are - * signed when the signer has a raw keypair. + * signed with the signer's raw keypair, which the provider always requires. * * Bytes are opaque here: client-side encryption (when used) wraps/unwraps * app-side, so CID verification covers exactly what the provider stores. @@ -17,9 +17,7 @@ import { CidMismatchError, DEFAULT_CHUNK_SIZE, httpFetch, - signProviderRequest, toHex, - type HttpFetchOpts, } from "@web3-storage/core"; import { asHex, @@ -27,18 +25,11 @@ import { deleteObjectMetadata as deleteObjectMetadataTx, deleteS3Bucket as deleteS3BucketTx, putObjectMetadata as putObjectMetadataTx, - type ChainSigner, - type ParachainApi, - type SubmitOpts, - type TxStatusListener, type WaitOpts, } from "@web3-storage/layer0"; -import { - ProviderUrlResolver, - resolveBucketProviders, - resolveCreationTerms, -} from "../provider-url.js"; +import { Layer1Client, type Layer1ClientOptions } from "../base-client.js"; +import { resolveBucketProviders, resolveCreationTerms } from "../provider-url.js"; import type { BucketInfo, BucketRef, @@ -50,69 +41,11 @@ import type { PutObjectResult, } from "./types.js"; -export interface S3ClientOptions { - api: ParachainApi; - signer?: ChainSigner | null; - /** Explicit provider URL (dev/tests) — skips on-chain resolution. */ - providerUrl?: string; - /** Injection point for unit tests. */ - fetch?: typeof fetch; - /** Tx progress listener. Default null (silent) — apps drive their own UI. */ - onStatus?: TxStatusListener | null; - /** - * Read view for chain lookups. Defaults to "finalized" (UI-grade, - * reorg-safe). Tests/examples pass READ_OPTS ({at: "best"}) to match their - * in-block submission semantics. - */ - readOpts?: { at: "best" | "finalized" }; - /** - * Submission doneness. Defaults to "finalized" (UI-grade). Tests/examples - * pass "best" for speed. - */ - submitMode?: "best" | "finalized"; -} +export type S3ClientOptions = Layer1ClientOptions; const utf8 = (s: string) => new TextEncoder().encode(s); -export class S3Client { - private readonly api: ParachainApi; - private signer: ChainSigner | null; - private readonly providers: ProviderUrlResolver; - private readonly fetchOpts: HttpFetchOpts; - private readonly onStatus: TxStatusListener | null; - private readonly readOpts: { at: "best" | "finalized" }; - private readonly submitMode: "best" | "finalized"; - private readonly creationUrlOverride?: string; - - constructor(opts: S3ClientOptions) { - this.api = opts.api; - this.signer = opts.signer ?? null; - this.readOpts = opts.readOpts ?? { at: "finalized" }; - this.submitMode = opts.submitMode ?? "finalized"; - this.providers = new ProviderUrlResolver(opts.api, opts.providerUrl, this.readOpts); - this.creationUrlOverride = opts.providerUrl; - this.fetchOpts = opts.fetch ? { fetchImpl: opts.fetch } : {}; - this.onStatus = opts.onStatus ?? null; - } - - setSigner(signer: ChainSigner | null): void { - this.signer = signer; - } - - private requireSigner(): ChainSigner { - if (!this.signer) throw new Error("Signer not set"); - return this.signer; - } - - private submitOpts(): SubmitOpts { - return { mode: this.submitMode, retryStale: 0, onStatus: this.onStatus }; - } - - private authHeaders(method: string, layer0BucketId: bigint): Record { - const kp = this.signer?.keypair; - return kp ? signProviderRequest(kp, method, layer0BucketId) : {}; - } - +export class S3Client extends Layer1Client { // ── Validation (S3 conventions, lifted from the original SDK stub) ────── validateBucketName(name: string): void { diff --git a/provider-node/README.md b/provider-node/README.md index 4d26ef2e..b35d44c1 100644 --- a/provider-node/README.md +++ b/provider-node/README.md @@ -14,10 +14,13 @@ just health # check provider is up ## Authentication -Authentication is enforced by default. Mutating and bucket-scoped endpoints -require a signed `Authorization` header; the only way to turn enforcement off is -the `--disable-auth-i-know-what-i-am-doing` flag, which opens every endpoint to -the public and is meant strictly for throwaway local experiments. +Authentication is always enforced: every mutating Layer-0 endpoint (`PUT +/node`, `POST /commit`, `POST /delete`) and all fs/s3 endpoints require a +signed `Authorization` header, and there is no way to turn enforcement off. +Layer-0 content-addressed reads (`GET /node`, `GET /read`, `POST +/fetch_nodes`, the commitment/proof endpoints) are currently unauthenticated; +whether they should require the `Reader` role is tracked in +[#228](https://github.com/paritytech/web3-storage/issues/228). The client signs an sr25519 message binding the request to a bucket and a timestamp: diff --git a/provider-node/src/auth.rs b/provider-node/src/auth.rs index d6f661e8..4183934f 100644 --- a/provider-node/src/auth.rs +++ b/provider-node/src/auth.rs @@ -44,6 +44,17 @@ pub trait MembershipResolver: Send + Sync { async fn fetch_members(&self, bucket_id: u64) -> Result, String>; } +/// A [`MembershipResolver`] that returns a fixed member set for every bucket. +/// Used by integration tests across crates. +pub struct StaticMembershipResolver(pub Vec<(AccountId32, Role)>); + +#[async_trait::async_trait] +impl MembershipResolver for StaticMembershipResolver { + async fn fetch_members(&self, _bucket_id: u64) -> Result, String> { + Ok(self.0.clone()) + } +} + /// Membership cache backed by chain queries via subxt. pub struct MembershipCache { cache: DashMap, @@ -175,64 +186,108 @@ impl MembershipResolver for ChainMembershipResolver { None => return Ok(vec![]), }; + // `members` is a `BoundedVec`. Depending on how the runtime's + // scale-info nests it, scale_value may wrap the sequence (and each + // `AccountId32`) in extra single-field composites, so walk the value + // tree and pull out every `{ account, role }` struct rather than + // assuming a fixed shape. let mut members = Vec::new(); - if let subxt::ext::scale_value::ValueDef::Composite( - subxt::ext::scale_value::Composite::Unnamed(items), - ) = &members_val.value - { - for item in items { - let account_val = item.at("account"); - let role_val = item.at("role"); - - if let (Some(account_v), Some(role_v)) = (account_val, role_val) { - let account_bytes = extract_account_bytes(account_v); - if let Some(bytes) = account_bytes { - let account = AccountId32::from(bytes); - let role = extract_role(role_v); - members.push((account, role)); - } - } - } + collect_members(members_val, &mut members); + + if members.is_empty() { + tracing::warn!(bucket_id, value = ?members_val, "auth: decoded zero members"); + } else { + tracing::debug!(bucket_id, count = members.len(), "auth: resolved members"); } Ok(members) } } +/// Recursively pull `(account, role)` pairs out of a decoded `members` value, +/// tolerating any wrapper composites that `BoundedVec` / `AccountId32` type +/// info introduces. A `Member` is the composite that carries both an +/// `account` and a `role` field. +fn collect_members(val: &subxt::ext::scale_value::Value, out: &mut Vec<(AccountId32, Role)>) { + use subxt::dynamic::At; + use subxt::ext::scale_value::{Composite, ValueDef}; + + if let (Some(account_v), Some(role_v)) = (val.at("account"), val.at("role")) { + if let Some(bytes) = extract_account_bytes(account_v) { + out.push((AccountId32::from(bytes), extract_role(role_v))); + return; + } + } + + match &val.value { + ValueDef::Composite(Composite::Named(fields)) => { + for field in fields { + collect_members(&field.1, out); + } + } + ValueDef::Composite(Composite::Unnamed(items)) => { + for item in items { + collect_members(item, out); + } + } + _ => {} + } +} + +/// Extract a 32-byte account id, descending through any wrapper composites +/// (`AccountId32` -> `[u8; 32]` can be one or more composite layers) and +/// collecting the `u8` leaves. fn extract_account_bytes(val: &subxt::ext::scale_value::Value) -> Option<[u8; 32]> { + let mut bytes = Vec::with_capacity(32); + collect_u8_leaves(val, &mut bytes); + if bytes.len() == 32 { + let mut arr = [0u8; 32]; + arr.copy_from_slice(&bytes); + Some(arr) + } else { + None + } +} + +fn collect_u8_leaves(val: &subxt::ext::scale_value::Value, out: &mut Vec) { use subxt::ext::scale_value::{Composite, Primitive, ValueDef}; match &val.value { + ValueDef::Primitive(Primitive::U128(n)) => out.push(*n as u8), ValueDef::Composite(Composite::Unnamed(items)) => { - let bytes: Vec = items - .iter() - .filter_map(|item| match &item.value { - ValueDef::Primitive(Primitive::U128(n)) => Some(*n as u8), - _ => None, - }) - .collect(); - if bytes.len() == 32 { - let mut arr = [0u8; 32]; - arr.copy_from_slice(&bytes); - Some(arr) - } else { - None + for item in items { + collect_u8_leaves(item, out); } } - _ => None, + ValueDef::Composite(Composite::Named(fields)) => { + for field in fields { + collect_u8_leaves(&field.1, out); + } + } + _ => {} } } +/// Decode a `Role`, descending through wrapper composites to the enum variant. fn extract_role(val: &subxt::ext::scale_value::Value) -> Role { - use subxt::ext::scale_value::ValueDef; - if let ValueDef::Variant(variant) = &val.value { - match variant.name.as_str() { - "Admin" => Role::Admin, - "Writer" => Role::Writer, - "Reader" => Role::Reader, - _ => Role::Reader, // default to least privilege + find_role_variant(val).unwrap_or(Role::Reader) +} + +fn find_role_variant(val: &subxt::ext::scale_value::Value) -> Option { + use subxt::ext::scale_value::{Composite, ValueDef}; + match &val.value { + ValueDef::Variant(variant) => match variant.name.as_str() { + "Admin" => Some(Role::Admin), + "Writer" => Some(Role::Writer), + "Reader" => Some(Role::Reader), + _ => None, + }, + ValueDef::Composite(Composite::Unnamed(items)) => { + items.iter().find_map(|v| find_role_variant(v)) } - } else { - Role::Reader + ValueDef::Composite(Composite::Named(fields)) => { + fields.iter().find_map(|f| find_role_variant(&f.1)) + } + _ => None, } } @@ -318,7 +373,7 @@ pub fn verify_signature( ); // Verify signature - let message = format!("web3storage:{method}:{bucket_id}:{timestamp_str}"); + let message = provider_negotiation::auth_message(method, bucket_id, timestamp_str); if !sr25519::Pair::verify(&signature, message.as_bytes(), &pubkey) { return Err(Error::AuthRequired); } @@ -328,9 +383,9 @@ pub fn verify_signature( /// Check that the caller has sufficient permissions. /// -/// Auth is enforced by default. This is a no-op (returns `Ok`) only when the -/// operator started the node with `--disable-auth-i-know-what-i-am-doing`, which -/// strips the membership config from the state. +/// Auth is always enforced. The caller must present a valid signed +/// `Authorization` header whose account holds the [`RequiredRole`] for the +/// bucket; otherwise the request is rejected. pub async fn require_role( state: &ProviderState, auth_header: Option<&str>, @@ -339,19 +394,11 @@ pub async fn require_role( required: RequiredRole, max_skew: Duration, ) -> Result<(), Error> { - if !state.auth_enabled { - return Ok(()); - } - - let cache = state - .membership_cache - .as_ref() - .ok_or_else(|| Error::Internal("Auth enabled but no membership cache".to_string()))?; - let header = auth_header.ok_or(Error::AuthRequired)?; let account = verify_signature(header, method, bucket_id, max_skew)?; - let role = cache + let role = state + .membership_cache .get_role(bucket_id, &account) .await .map_err(|e| Error::Internal(format!("Membership lookup failed: {e}")))? @@ -382,13 +429,12 @@ mod tests { bucket_id: u64, timestamp: u64, ) -> String { - let message = format!("web3storage:{method}:{bucket_id}:{timestamp}"); - let signature = keypair.sign(message.as_bytes()); - format!( - "Web3Storage 0x{}:0x{}:{}", - hex::encode(keypair.public().0), - hex::encode(signature.0), - timestamp + provider_negotiation::build_auth_header( + &keypair.public().0, + method, + bucket_id, + timestamp, + |msg| keypair.sign(msg).0, ) } @@ -399,6 +445,36 @@ mod tests { .as_secs() } + /// Regression test for the dynamic decoding of `StorageProvider.Buckets`. + #[test] + fn collect_members_handles_chain_value_nesting() { + use subxt::ext::scale_value::Value; + + let acct = [9u8; 32]; + // AccountId32 -> [u8; 32]: two composite layers around the byte leaves. + let account = Value::unnamed_composite(vec![Value::unnamed_composite( + acct.iter().map(|b| Value::u128(*b as u128)), + )]); + let member = Value::named_composite(vec![ + ("account".to_string(), account), + ("role".to_string(), Value::unnamed_variant("Writer", vec![])), + ]); + let sequence = Value::unnamed_composite(vec![member]); + // BoundedVec wrapper around the member sequence. + let members_val = Value::unnamed_composite(vec![sequence]); + + let mut out = Vec::new(); + collect_members(&members_val, &mut out); + + assert_eq!( + out.len(), + 1, + "member must be recovered through both wrappers" + ); + assert_eq!(out[0].0, AccountId32::from(acct)); + assert!(matches!(out[0].1, Role::Writer)); + } + #[test] fn test_verify_valid_signature() { let keypair = sr25519::Pair::from_string("//Alice", None).unwrap(); diff --git a/provider-node/src/chain_state_coordinator.rs b/provider-node/src/chain_state_coordinator.rs index 1622783e..3be219f0 100644 --- a/provider-node/src/chain_state_coordinator.rs +++ b/provider-node/src/chain_state_coordinator.rs @@ -63,12 +63,19 @@ pub struct ChainState { impl Default for ChainState { fn default() -> Self { + Self::with_nonce_store(Arc::new(NullNonceStore)) + } +} + +impl ChainState { + /// Fresh chain state whose nonce counter persists through `store`. + pub fn with_nonce_store(store: Arc) -> Self { Self { current_block: AtomicU32::new(0), constants: RwLock::new(None), provider_info: RwLock::new(None), nonce_counter: RwLock::new(None), - nonce_store: Arc::new(NullNonceStore), + nonce_store: store, } } } diff --git a/provider-node/src/cli.rs b/provider-node/src/cli.rs index 79588167..2d53fc0f 100644 --- a/provider-node/src/cli.rs +++ b/provider-node/src/cli.rs @@ -179,18 +179,8 @@ pub struct CheckpointParams { } /// Parameters for authentication and authorization. -/// -/// Authentication and role-based access control are enforced by default. The -/// only way to turn them off is the deliberately verbose -/// `--disable-auth-i-know-what-i-am-doing` flag below. #[derive(Debug, clap::Args)] pub struct AuthParams { - /// Disable ALL authentication and role-based access control. Every endpoint - /// becomes publicly readable AND writable by anyone. Intended only for - /// throwaway local experiments — never run a real provider this way. - #[arg(long)] - pub disable_auth_i_know_what_i_am_doing: bool, - /// Cache TTL in seconds for membership lookups from the chain. #[arg( long, diff --git a/provider-node/src/command.rs b/provider-node/src/command.rs index b06ed715..3727329a 100644 --- a/provider-node/src/command.rs +++ b/provider-node/src/command.rs @@ -10,9 +10,9 @@ use crate::{ subxt_client::SubxtChainClient, ChainStateCoordinatorHandle, ChallengeResponder, ChallengeResponderConfig, ChallengeResponderHandle, CheckpointCoordinator, CheckpointCoordinatorConfig, - CheckpointCoordinatorHandle, DiskStorage, NonceStore, NullNonceStore, ProviderState, - ReplicaSyncCoordinator, ReplicaSyncCoordinatorConfig, ReplicaSyncCoordinatorHandle, Storage, - StorageBackend, + CheckpointCoordinatorHandle, DiskStorage, NonceStore, NullNonceStore, ProviderDeps, + ProviderState, ReplicaSyncCoordinator, ReplicaSyncCoordinatorConfig, + ReplicaSyncCoordinatorHandle, Storage, StorageBackend, }; use clap::Parser; use std::net::SocketAddr; @@ -52,13 +52,30 @@ pub async fn run() -> Result<(), Box> { } }; + // Membership-based auth over the chain's bucket member sets. + let resolver = ChainMembershipResolver::new(cli.rpc.chain_rpc.clone()); + let ttl = Duration::from_secs(cli.auth.auth_cache_ttl); + let membership = Arc::new(MembershipCache::new(Box::new(resolver), ttl)); + tracing::info!( + "Auth: membership cache_ttl={}s, max_skew={}s", + cli.auth.auth_cache_ttl, + cli.auth.auth_max_skew + ); + + let deps = ProviderDeps { + storage, + nonce_store, + membership, + auth_max_skew: Duration::from_secs(cli.auth.auth_max_skew), + }; + // Resolve provider identity let seed = cli.key.load_seed()?; - let mut state = match &seed { + let state = match &seed { Some(seed) => { - let state = ProviderState::with_seed(storage, seed)?; + let state = ProviderState::with_seed(deps, seed)?; tracing::info!("Signing enabled for account: {}", state.provider_id); - configure_state(state, &cli) + state } None => { let provider_id = cli @@ -71,14 +88,10 @@ pub async fn run() -> Result<(), Box> { provider_id ); - configure_state(ProviderState::with_provider_id(storage, provider_id), &cli) + ProviderState::with_provider_id(deps, provider_id) } - }; - - // Install the nonce store before sharing `state` across coordinators: while - // it is still solely owned here, `chain_state`'s Arc has a single owner, so - // the in-place install succeeds. - state.set_nonce_store(nonce_store); + } + .with_cors_origins(cli.rpc.cors_allowed_origins.clone()); let state = Arc::new(state); @@ -136,36 +149,6 @@ pub async fn run() -> Result<(), Box> { Ok(()) } -/// Apply CLI-derived CORS and auth settings to a freshly constructed state. -/// -/// Authentication is enforced by default: unless the operator explicitly passed -/// `--disable-auth-i-know-what-i-am-doing`, we wire in the on-chain membership -/// resolver so every bucket-scoped request is checked against the caller's role. -fn configure_state(state: ProviderState, cli: &Cli) -> ProviderState { - let mut state = state.with_cors_origins(cli.rpc.cors_allowed_origins.clone()); - - if cli.auth.disable_auth_i_know_what_i_am_doing { - state = state.with_auth_disabled(); - tracing::warn!( - "AUTHENTICATION DISABLED via --disable-auth-i-know-what-i-am-doing: \ - every endpoint is publicly readable and writable by anyone. \ - Never do this outside a throwaway local environment." - ); - } else { - let resolver = ChainMembershipResolver::new(cli.rpc.chain_rpc.clone()); - let ttl = Duration::from_secs(cli.auth.auth_cache_ttl); - let cache = MembershipCache::new(Box::new(resolver), ttl); - state.set_auth_config(Arc::new(cache), Duration::from_secs(cli.auth.auth_max_skew)); - tracing::info!( - "Auth enforced (cache_ttl={}s, max_skew={}s)", - cli.auth.auth_cache_ttl, - cli.auth.auth_max_skew - ); - } - - state -} - /// Start the chain-state coordinator, which keeps `chain_state.current_block` /// and `chain_state.provider_info` in sync with the chain. /// diff --git a/provider-node/src/lib.rs b/provider-node/src/lib.rs index 3aef0ab2..fec02d6e 100644 --- a/provider-node/src/lib.rs +++ b/provider-node/src/lib.rs @@ -65,6 +65,19 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; +/// Everything a servable [`ProviderState`] requires. +pub struct ProviderDeps { + /// Local storage backend. + pub storage: Arc, + /// Persistence backing for the nonce counter — [`NullNonceStore`] in + /// in-memory mode, the disk store in disk mode. + pub nonce_store: Arc, + /// Membership cache for role lookups. + pub membership: Arc, + /// Maximum allowed clock skew for request timestamps. + pub auth_max_skew: Duration, +} + /// Provider node state shared across handlers. pub struct ProviderState { /// Local storage backend @@ -79,13 +92,8 @@ pub struct ProviderState { pub fs_index: FsIndexManager, /// Channel to send commands to the checkpoint coordinator (if running). pub checkpoint_cmd_tx: std::sync::Mutex>>, - /// Whether membership auth is enforced. Enforced by default at startup; the - /// node clears this only when started with - /// `--disable-auth-i-know-what-i-am-doing`. - pub auth_enabled: bool, - /// Membership cache for role lookups. Set whenever auth is enforced (i.e. - /// always, unless the operator opted out via the escape-hatch flag). - pub membership_cache: Option>, + /// Membership cache for role lookups. + pub membership_cache: Arc, /// Maximum allowed clock skew for request timestamps. pub auth_max_skew: Duration, /// Browser origins allowed via CORS. `None` (the default) keeps the @@ -99,12 +107,14 @@ pub struct ProviderState { impl ProviderState { /// Shared constructor body for [`with_provider_id`](Self::with_provider_id) - /// and [`with_seed`](Self::with_seed). All other fields take their defaults; - fn from_parts( - storage: Arc, - provider_id: String, - keypair: Option, - ) -> Self { + /// and [`with_seed`](Self::with_seed). + fn from_parts(deps: ProviderDeps, provider_id: String, keypair: Option) -> Self { + let ProviderDeps { + storage, + nonce_store, + membership, + auth_max_skew, + } = deps; Self { storage, provider_id, @@ -112,29 +122,28 @@ impl ProviderState { s3_index: S3IndexManager::new(), fs_index: FsIndexManager::new(), checkpoint_cmd_tx: std::sync::Mutex::new(None), - auth_enabled: true, - membership_cache: None, - auth_max_skew: Duration::from_secs(300), + membership_cache: membership, + auth_max_skew, cors_allowed_origins: None, - chain_state: Arc::new(ChainState::default()), + chain_state: Arc::new(ChainState::with_nonce_store(nonce_store)), } } /// Create state for a provider that cannot sign: `provider_id` is used as-is /// for identity and on-chain reconciliation, and signing endpoints stay /// unavailable. For a signing provider use [`with_seed`](Self::with_seed). - pub fn with_provider_id(storage: Arc, provider_id: String) -> Self { - Self::from_parts(storage, provider_id, None) + pub fn with_provider_id(deps: ProviderDeps, provider_id: String) -> Self { + Self::from_parts(deps, provider_id, None) } /// Create with a seed phrase or derivation path (e.g., "//Alice", "//Bob"). - pub fn with_seed(storage: Arc, seed: &str) -> Result { + pub fn with_seed(deps: ProviderDeps, seed: &str) -> Result { let keypair = sr25519::Pair::from_string(seed, None) .map_err(|e| format!("Failed to create keypair: {e:?}"))?; let provider_id = keypair.public().to_ss58check(); - Ok(Self::from_parts(storage, provider_id, Some(keypair))) + Ok(Self::from_parts(deps, provider_id, Some(keypair))) } /// Restrict the browser origins allowed via CORS. `None` (the default) keeps @@ -144,44 +153,6 @@ impl ProviderState { self } - /// Enable membership-based auth, wiring in the role-lookup cache and the - /// maximum tolerated clock skew for request timestamps. - pub fn set_auth_config( - &mut self, - membership_cache: Arc, - max_skew: Duration, - ) { - self.auth_enabled = true; - self.membership_cache = Some(membership_cache); - self.auth_max_skew = max_skew; - } - - /// Turn off membership auth, leaving every endpoint publicly accessible. - /// Reserved for the `--disable-auth-i-know-what-i-am-doing` escape hatch and - /// for tests that exercise non-auth behavior. - pub fn with_auth_disabled(mut self) -> Self { - self.auth_enabled = false; - self.membership_cache = None; - self - } - - /// Install the nonce-counter persistence backend. - /// - /// Must be called while `self` is still solely owned — before it is wrapped - /// in an `Arc` and shared with the coordinators — because `chain_state` is - /// mutated in place via `Arc::get_mut`. If `chain_state` is already shared - /// the store is left as the default `NullNonceStore` (disk-mode persistence - /// disabled) and an error is logged rather than silently dropping it. - pub fn set_nonce_store(&mut self, store: Arc) { - match Arc::get_mut(&mut self.chain_state) { - Some(cs) => cs.nonce_store = store, - None => tracing::error!( - "nonce store install skipped: chain_state Arc has multiple owners; \ - disk-mode persistence is disabled for this run" - ), - } - } - /// Set the checkpoint coordinator command sender (called after coordinator starts). pub fn set_checkpoint_handle(&self, handle: &CheckpointCoordinatorHandle) { if let Ok(mut tx) = self.checkpoint_cmd_tx.lock() { @@ -206,17 +177,22 @@ impl ProviderState { mod tests { use super::*; - fn test_storage() -> Arc { - Arc::new(storage::Storage::new()) - } - #[test] fn sign_without_keypair_refuses_with_signing_unavailable() { // The pre-fix behaviour silently returned 64 zero bytes. The new // contract is that `sign()` MUST return `Err(SigningUnavailable)` // when no keypair is configured, so the HTTP layer can map it to a // 503 instead of emitting a cryptographically invalid placeholder. - let state = ProviderState::with_provider_id(test_storage(), "no-key-provider".to_string()); + let deps = ProviderDeps { + storage: Arc::new(storage::Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(auth::MembershipCache::new( + Box::new(auth::StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_provider_id(deps, "no-key-provider".to_string()); let err = state .sign(b"any message") .expect_err("must refuse to sign without a keypair"); @@ -229,7 +205,16 @@ mod tests { // Alice's public key. This catches any regression where sign() ever // returns the 0x00..00 placeholder again, and also catches the more // subtle case where the bytes look random but aren't valid sr25519. - let state = ProviderState::with_seed(test_storage(), "//Alice").unwrap(); + let deps = ProviderDeps { + storage: Arc::new(storage::Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(auth::MembershipCache::new( + Box::new(auth::StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, "//Alice").unwrap(); let message = b"commitment-payload-bytes"; let sig_hex = state.sign(message).expect("signing succeeds with keypair"); @@ -271,7 +256,16 @@ mod tests { // message produce different signatures, but both must verify. This // test guards against accidentally swapping to a backend that // returns a constant value (e.g. zero bytes). - let state = ProviderState::with_seed(test_storage(), "//Alice").unwrap(); + let deps = ProviderDeps { + storage: Arc::new(storage::Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(auth::MembershipCache::new( + Box::new(auth::StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, "//Alice").unwrap(); let alice_pub = keypair_for("//Alice").public(); let msg = b"commitment-payload"; @@ -291,8 +285,26 @@ mod tests { // Negative control: //Bob's signature must NOT verify under //Alice. // Cheap protection against a future refactor that accidentally // stops checking the message or the key. - let alice = ProviderState::with_seed(test_storage(), "//Alice").unwrap(); - let bob = ProviderState::with_seed(test_storage(), "//Bob").unwrap(); + let alice_deps = ProviderDeps { + storage: Arc::new(storage::Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(auth::MembershipCache::new( + Box::new(auth::StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let bob_deps = ProviderDeps { + storage: Arc::new(storage::Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(auth::MembershipCache::new( + Box::new(auth::StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let alice = ProviderState::with_seed(alice_deps, "//Alice").unwrap(); + let bob = ProviderState::with_seed(bob_deps, "//Bob").unwrap(); let alice_pub = keypair_for("//Alice").public(); let msg = b"checkpoint payload"; @@ -307,7 +319,16 @@ mod tests { #[test] fn provider_state_chain_defaults_on_new() { use std::sync::atomic::Ordering; - let state = ProviderState::with_provider_id(test_storage(), "test-provider".to_string()); + let deps = ProviderDeps { + storage: Arc::new(storage::Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(auth::MembershipCache::new( + Box::new(auth::StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_provider_id(deps, "test-provider".to_string()); assert_eq!(state.chain_state.current_block.load(Ordering::Relaxed), 0); assert!(state.chain_state.provider_info.read().is_none()); } diff --git a/provider-node/tests/api_integration.rs b/provider-node/tests/api_integration.rs index 9073adba..6d002ac9 100644 --- a/provider-node/tests/api_integration.rs +++ b/provider-node/tests/api_integration.rs @@ -4,23 +4,27 @@ //! //! These tests spin up a real HTTP server and test the full request/response cycle. +mod common; + use axum::http::StatusCode; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use codec::Encode; -use reqwest::Client; +use common::SignedClient; +use reqwest::Method; use serde_json::{json, Value}; use sp_core::crypto::Ss58Codec; use sp_core::{sr25519, ByteArray, Pair, H256}; use std::net::SocketAddr; use std::sync::Arc; -use storage_primitives::{Commitment, CommitmentPayload}; -use storage_provider_node::{create_router, ProviderState, Storage}; -use tokio::net::TcpListener; +use std::time::Duration; +use storage_primitives::{Commitment, CommitmentPayload, Role}; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{NullNonceStore, ProviderDeps, ProviderState, Storage}; /// Test server helper that starts the provider node on a random port. struct TestServer { addr: SocketAddr, - client: Client, + client: SignedClient, } pub const PROVIDER_SEED: &str = "//Alice"; @@ -31,14 +35,21 @@ impl TestServer { /// Endpoints that sign commitments (`/commit`, `/commitment`, ...) work /// because a real sr25519 keypair is available. async fn new() -> Self { - // These tests exercise endpoint behavior, not auth, so run with auth - // disabled (auth is enforced by default). Auth is covered end-to-end in - // `auth_integration.rs`. - Self::with_state(Arc::new( - ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED) - .expect("//Alice is a valid SURI") - .with_auth_disabled(), - )) + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![( + common::test_member_account(), + Role::Admin, + )])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + Self::with_state( + ProviderState::with_seed(deps, PROVIDER_SEED).expect("//Alice is a valid SURI"), + ) .await } @@ -47,35 +58,28 @@ impl TestServer { /// Used to verify that signing-bound endpoints return 503 rather than /// silently emitting zero-byte placeholder signatures. async fn new_unsigned() -> Self { - Self::with_state(Arc::new( - ProviderState::with_provider_id( - Arc::new(Storage::new()), - "0xtest_provider".to_string(), - ) - .with_auth_disabled(), + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![( + common::test_member_account(), + Role::Admin, + )])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + Self::with_state(ProviderState::with_provider_id( + deps, + "0xtest_provider".to_string(), )) .await } - async fn with_state(state: Arc) -> Self { - let app = create_router(state); - - // Bind to port 0 to get a random available port - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - - // Spawn the server - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - // Give the server a moment to start - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - - Self { - addr, - client: Client::new(), - } + async fn with_state(state: ProviderState) -> Self { + let (addr, client) = common::serve(state).await; + Self { addr, client } } fn url(&self, path: &str) -> String { @@ -451,7 +455,7 @@ async fn upload_and_commit(server: &TestServer, bucket_id: u64) -> (String, Valu let resp = server .client - .put(server.url("/node")) + .request_bucket(Method::PUT, server.url("/node"), bucket_id) .json(&json!({ "bucket_id": bucket_id, "hash": hash_hex, @@ -465,7 +469,7 @@ async fn upload_and_commit(server: &TestServer, bucket_id: u64) -> (String, Valu let commit_resp = server .client - .post(server.url("/commit")) + .request_bucket(Method::POST, server.url("/commit"), bucket_id) .json(&json!({ "bucket_id": bucket_id, "data_roots": [hash_hex], diff --git a/provider-node/tests/auth_integration.rs b/provider-node/tests/auth_integration.rs index 2a632da8..86778248 100644 --- a/provider-node/tests/auth_integration.rs +++ b/provider-node/tests/auth_integration.rs @@ -2,76 +2,29 @@ //! Integration tests for auth-enabled HTTP endpoints. //! -//! These tests spin up a real HTTP server with `auth_enabled = true` and a -//! `MockResolver` that returns configurable roles for test accounts. All -//! assertions go through real HTTP requests — the auth middleware, signature -//! verification, membership cache lookup, and role check are exercised as a -//! single end-to-end path. +//! These tests spin up a real HTTP server whose membership is a fixed member +//! set with configurable roles per test account. All assertions go through +//! real HTTP requests — the auth middleware, signature verification, +//! membership cache lookup, and role check are exercised as a single +//! end-to-end path. + +mod common; use axum::http::StatusCode; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use common::{current_timestamp, make_auth_header}; use reqwest::Client; use serde_json::Value; use sp_core::{sr25519, Pair}; use std::sync::Arc; use std::time::Duration; use storage_primitives::Role; -use storage_provider_node::auth::{MembershipCache, MembershipResolver}; -use storage_provider_node::{create_router, ProviderState, Storage}; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{create_router, NullNonceStore, ProviderDeps, ProviderState, Storage}; use tokio::net::TcpListener; type AccountId32 = sp_core::crypto::AccountId32; -// ───────────────────────────────────────────────────────────────────────────── -// Mock resolver (returns configurable roles, no chain needed) -// ───────────────────────────────────────────────────────────────────────────── - -struct MockResolver { - members: std::sync::Mutex>, -} - -impl MockResolver { - fn new(members: Vec<(AccountId32, Role)>) -> Self { - Self { - members: std::sync::Mutex::new(members), - } - } -} - -#[async_trait::async_trait] -impl MembershipResolver for MockResolver { - async fn fetch_members(&self, _bucket_id: u64) -> Result, String> { - Ok(self.members.lock().unwrap().clone()) - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// Auth header helpers -// ───────────────────────────────────────────────────────────────────────────── - -fn current_timestamp() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs() -} - -fn make_auth_header( - keypair: &sr25519::Pair, - method: &str, - bucket_id: u64, - timestamp: u64, -) -> String { - let message = format!("web3storage:{method}:{bucket_id}:{timestamp}"); - let signature = keypair.sign(message.as_bytes()); - format!( - "Web3Storage 0x{}:0x{}:{}", - hex_encode(&keypair.public().0), - hex_encode(&signature.0), - timestamp - ) -} - // ───────────────────────────────────────────────────────────────────────────── // Test server // ───────────────────────────────────────────────────────────────────────────── @@ -87,16 +40,17 @@ impl AuthTestServer { let alice_kp = sr25519::Pair::from_string("//Alice", None).unwrap(); let alice_account = AccountId32::new(alice_kp.public().0); - let resolver = MockResolver::new(vec![(alice_account, alice_role)]); - let cache = Arc::new(MembershipCache::new( - Box::new(resolver), - Duration::from_secs(60), - )); - - let mut state = ProviderState::with_seed(Arc::new(Storage::new()), "//Alice") - .expect("//Alice is valid"); - // 300s skew keeps the default the `*_expired_timestamp` tests assume. - state.set_auth_config(cache, Duration::from_secs(300)); + // The 300s skew keeps the default the `*_expired_timestamp` tests assume. + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![(alice_account, alice_role)])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, "//Alice").expect("//Alice is valid"); let app = create_router(Arc::new(state)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -532,7 +486,7 @@ fn node_body(bucket_id: u64, data: &[u8]) -> Value { let hash = storage_primitives::blake2_256(data); serde_json::json!({ "bucket_id": bucket_id, - "hash": format!("0x{}", hex_encode(hash.as_bytes())), + "hash": format!("0x{}", hex::encode(hash.as_bytes())), "data": BASE64.encode(data), }) } @@ -599,7 +553,7 @@ async fn commit_writer_can_commit() { let data = b"committed chunk"; let hash_hex = format!( "0x{}", - hex_encode(storage_primitives::blake2_256(data).as_bytes()) + hex::encode(storage_primitives::blake2_256(data).as_bytes()) ); let ts = current_timestamp(); let header = make_auth_header(&alice, "PUT", 1, ts); @@ -646,11 +600,3 @@ async fn commit_reader_blocked() { assert_eq!(resp.status(), StatusCode::FORBIDDEN); } - -// ───────────────────────────────────────────────────────────────────────────── -// Helpers -// ───────────────────────────────────────────────────────────────────────────── - -fn hex_encode(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} diff --git a/provider-node/tests/common/mod.rs b/provider-node/tests/common/mod.rs new file mode 100644 index 00000000..a61a4d8f --- /dev/null +++ b/provider-node/tests/common/mod.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-3.0-only + +//! Shared test helpers for the provider-node integration suites. +//! +//! Tests use [`SignedClient`] to sign every request as `//Alice`. + +// Each integration suite compiles this module in its own test crate and exercises only +// a subset of these helpers, so per-crate dead-code analysis flags the rest. +#![allow(dead_code)] + +use provider_negotiation::build_auth_header; +use reqwest::{Method, RequestBuilder}; +use sp_core::{sr25519, Pair}; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use storage_provider_node::{create_router, ProviderState}; + +type AccountId32 = sp_core::crypto::AccountId32; + +/// The account every test signs as. +pub const TEST_MEMBER_SEED: &str = "//Alice"; + +pub fn test_member_pair() -> sr25519::Pair { + sr25519::Pair::from_string(TEST_MEMBER_SEED, None).expect("//Alice is a valid SURI") +} + +pub fn test_member_account() -> AccountId32 { + AccountId32::new(test_member_pair().public().0) +} + +/// Spawn the provider on a random port and return its address plus a +/// [`SignedClient`]. +pub async fn serve(state: ProviderState) -> (SocketAddr, SignedClient) { + let app = create_router(Arc::new(state)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + while tokio::net::TcpStream::connect(addr).await.is_err() { + tokio::task::yield_now().await; + } + (addr, SignedClient::new()) +} + +pub fn current_timestamp() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is after the Unix epoch") + .as_secs() +} + +/// `Authorization` value signed by `keypair` over +/// `web3storage:::`. +pub fn make_auth_header( + keypair: &sr25519::Pair, + method: &str, + bucket_id: u64, + timestamp: u64, +) -> String { + build_auth_header(&keypair.public().0, method, bucket_id, timestamp, |msg| { + keypair.sign(msg).0 + }) +} + +/// Bucket id from the URL path (`/s3/{id}/`, `/fs/{id}/`) or a `?bucket_id=` +/// query param. Matched per-component so a key containing a marker can't mislead. +fn parse_bucket_id(url: &str) -> Option { + let (path, query) = url.split_once('?').unwrap_or((url, "")); + let rest = path + .split_once("/s3/") + .or_else(|| path.split_once("/fs/")) + .or_else(|| query.split_once("bucket_id=")) + .map(|(_, rest)| rest)?; + rest.split(|c: char| !c.is_ascii_digit()) + .find(|t| !t.is_empty())? + .parse() + .ok() +} + +/// `reqwest::Client` that signs every request as the test member. +/// +/// Verb methods infer the bucket from the URL, defaulting to `1`. For an +/// endpoint (`/node`, `/commit`, `/delete`) targeting a bucket other than `1`, +/// sign explicitly with [`SignedClient::request_bucket`]. +pub struct SignedClient { + inner: reqwest::Client, + keypair: sr25519::Pair, +} + +impl Default for SignedClient { + fn default() -> Self { + Self::new() + } +} + +impl SignedClient { + pub fn new() -> Self { + Self { + inner: reqwest::Client::new(), + keypair: test_member_pair(), + } + } + + /// Sign for an explicit bucket id (L0 endpoints whose bucket is in the body). + pub fn request_bucket(&self, method: Method, url: String, bucket_id: u64) -> RequestBuilder { + let header = make_auth_header( + &self.keypair, + method.as_str(), + bucket_id, + current_timestamp(), + ); + self.inner + .request(method, url) + .header(reqwest::header::AUTHORIZATION, header) + } + + fn auto(&self, method: Method, url: String) -> RequestBuilder { + let bucket_id = parse_bucket_id(&url).unwrap_or(1); + self.request_bucket(method, url, bucket_id) + } + + pub fn get(&self, url: String) -> RequestBuilder { + self.auto(Method::GET, url) + } + + pub fn put(&self, url: String) -> RequestBuilder { + self.auto(Method::PUT, url) + } + + pub fn post(&self, url: String) -> RequestBuilder { + self.auto(Method::POST, url) + } + + pub fn delete(&self, url: String) -> RequestBuilder { + self.auto(Method::DELETE, url) + } + + pub fn head(&self, url: String) -> RequestBuilder { + self.auto(Method::HEAD, url) + } +} diff --git a/provider-node/tests/coordinators/challenge.rs b/provider-node/tests/coordinators/challenge.rs index 67321cf5..5519a17b 100644 --- a/provider-node/tests/coordinators/challenge.rs +++ b/provider-node/tests/coordinators/challenge.rs @@ -7,9 +7,11 @@ use sp_core::H256; use std::sync::{Arc, Mutex}; use std::time::Duration; use storage_primitives::{blake2_256, BucketId}; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; use storage_provider_node::{ build_padded_merkle_tree, ChallengeChainClient, ChallengeResponder, ChallengeResponderConfig, - ChallengeResponseResult, DetectedChallenge, Error, ProviderState, Storage, + ChallengeResponseResult, DetectedChallenge, Error, NullNonceStore, ProviderDeps, ProviderState, + Storage, }; struct MockChallengeChainClient { @@ -176,8 +178,17 @@ fn test_state_with_data() -> (Arc, DetectedChallenge) { challenger: ALICE_SS58.to_string(), }; - let state = Arc::new(ProviderState::with_provider_id( + let deps = ProviderDeps { storage, + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = Arc::new(ProviderState::with_provider_id( + deps, ALICE_SS58.to_string(), )); (state, challenge) diff --git a/provider-node/tests/coordinators/checkpoint.rs b/provider-node/tests/coordinators/checkpoint.rs index 8eaae975..6689096f 100644 --- a/provider-node/tests/coordinators/checkpoint.rs +++ b/provider-node/tests/coordinators/checkpoint.rs @@ -7,10 +7,11 @@ use sp_core::H256; use std::sync::{Arc, Mutex}; use std::time::Duration; use storage_primitives::BucketId; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; use storage_provider_node::checkpoint_coordinator::SignProposalRequest; use storage_provider_node::{ CheckpointChainClient, CheckpointCoordinator, CheckpointCoordinatorConfig, CheckpointDuty, - CheckpointResult, Error, ProviderState, Storage, + CheckpointResult, Error, NullNonceStore, ProviderDeps, ProviderState, Storage, }; struct MockCheckpointChainClient { @@ -79,7 +80,16 @@ fn test_state_with_bucket(bucket_id: BucketId) -> Arc { let data_root = H256::from(hash); let _ = storage.store_node(bucket_id, data_root, data, None); storage.commit(bucket_id, vec![data_root]).unwrap(); - Arc::new(ProviderState::with_seed(storage, "//Alice").unwrap()) + let deps = ProviderDeps { + storage, + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + Arc::new(ProviderState::with_seed(deps, "//Alice").unwrap()) } #[test] diff --git a/provider-node/tests/coordinators/main.rs b/provider-node/tests/coordinators/main.rs index 7eeb5a21..1890896a 100644 --- a/provider-node/tests/coordinators/main.rs +++ b/provider-node/tests/coordinators/main.rs @@ -9,7 +9,9 @@ mod checkpoint; mod replica_sync; use std::sync::Arc; -use storage_provider_node::{ProviderState, Storage}; +use std::time::Duration; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{NullNonceStore, ProviderDeps, ProviderState, Storage}; /// Full Alice SS58 address (substrate prefix 42). pub const ALICE_SS58: &str = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"; @@ -17,17 +19,33 @@ pub const ALICE_SEED: &str = "//Alice"; /// Create a standard test `ProviderState` for coordinator tests. pub fn test_state() -> Arc { - let storage = Arc::new(Storage::new()); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; Arc::new(ProviderState::with_provider_id( - storage, + deps, ALICE_SS58.to_string(), )) } /// Create a test `ProviderState` with a keypair derived from the given seed. pub fn test_state_with_seed(seed: &str) -> Arc { - let storage = Arc::new(Storage::new()); - Arc::new(ProviderState::with_seed(storage, seed).unwrap()) + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + Arc::new(ProviderState::with_seed(deps, seed).unwrap()) } /// Poll a condition with timeout. Returns `true` if the condition was met diff --git a/provider-node/tests/coordinators/replica_sync.rs b/provider-node/tests/coordinators/replica_sync.rs index 141896fe..62c722ee 100644 --- a/provider-node/tests/coordinators/replica_sync.rs +++ b/provider-node/tests/coordinators/replica_sync.rs @@ -8,10 +8,11 @@ use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::Duration; use storage_primitives::BucketId; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; use storage_provider_node::replica_sync_coordinator::{BucketSnapshot, ReplicaAgreementInfo}; use storage_provider_node::{ - Error, ProviderState, ReplicaSyncChainClient, ReplicaSyncCoordinator, - ReplicaSyncCoordinatorConfig, Storage, SyncDuty, SyncResult, + Error, NullNonceStore, ProviderDeps, ProviderState, ReplicaSyncChainClient, + ReplicaSyncCoordinator, ReplicaSyncCoordinatorConfig, Storage, SyncDuty, SyncResult, }; struct MockReplicaSyncChainClient { @@ -156,7 +157,16 @@ async fn test_already_synced() { let _ = storage.store_node(1, data_root, data, None); let (mmr_root, _, _) = storage.commit(1, vec![data_root]).unwrap(); - let state = Arc::new(ProviderState::with_provider_id(storage, "test".to_string())); + let deps = ProviderDeps { + storage, + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = Arc::new(ProviderState::with_provider_id(deps, "test".to_string())); let duty = SyncDuty { bucket_id: 1, @@ -364,8 +374,17 @@ async fn test_duties_filter_already_synced() { storage.store_node(1, data_root, data, None).unwrap(); let (mmr_root, _, _) = storage.commit(1, vec![data_root]).unwrap(); - let state = Arc::new(ProviderState::with_provider_id( + let deps = ProviderDeps { storage, + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = Arc::new(ProviderState::with_provider_id( + deps, ALICE_SS58.to_string(), )); diff --git a/provider-node/tests/disk_integration.rs b/provider-node/tests/disk_integration.rs index f57c348c..963d8b68 100644 --- a/provider-node/tests/disk_integration.rs +++ b/provider-node/tests/disk_integration.rs @@ -6,18 +6,23 @@ //! disk backend instead of in-memory storage, ensuring the full RocksDB //! serialization/deserialization path is exercised through real HTTP requests. +mod common; + use axum::http::StatusCode; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; -use reqwest::Client; +use common::SignedClient; +use reqwest::Method; use serde_json::{json, Value}; use std::sync::Arc; -use storage_provider_node::{create_router, DiskStorage, ProviderState}; +use std::time::Duration; +use storage_primitives::Role; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{DiskStorage, NullNonceStore, ProviderDeps, ProviderState}; use tempfile::TempDir; -use tokio::net::TcpListener; struct DiskTestServer { addr: std::net::SocketAddr, - client: Client, + client: SignedClient, // Keep TempDir alive so RocksDB path isn't removed _dir: TempDir, } @@ -26,22 +31,24 @@ impl DiskTestServer { async fn new() -> Self { let dir = TempDir::new().unwrap(); let disk = DiskStorage::new(dir.path()).expect("RocksDB should open"); - // Functional tests, not auth tests: run with auth disabled (auth is - // enforced by default; see auth_integration.rs for the auth coverage). - let state = Arc::new( - ProviderState::with_seed(Arc::new(disk), "//Alice") - .expect("//Alice is valid") - .with_auth_disabled(), - ); - - let app = create_router(state); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - + let deps = ProviderDeps { + storage: Arc::new(disk), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![( + common::test_member_account(), + Role::Admin, + )])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let (addr, client) = + common::serve(ProviderState::with_seed(deps, "//Alice").expect("//Alice is valid")) + .await; Self { addr, - client: Client::new(), + client, _dir: dir, } } @@ -59,7 +66,7 @@ async fn upload_and_commit(server: &DiskTestServer, bucket_id: u64) -> (String, let resp = server .client - .put(server.url("/node")) + .request_bucket(Method::PUT, server.url("/node"), bucket_id) .json(&json!({ "bucket_id": bucket_id, "hash": hash_hex, @@ -73,7 +80,7 @@ async fn upload_and_commit(server: &DiskTestServer, bucket_id: u64) -> (String, let commit_resp = server .client - .post(server.url("/commit")) + .request_bucket(Method::POST, server.url("/commit"), bucket_id) .json(&json!({ "bucket_id": bucket_id, "data_roots": [hash_hex], diff --git a/provider-node/tests/fs_integration.rs b/provider-node/tests/fs_integration.rs index 5ec89538..7ddcb091 100644 --- a/provider-node/tests/fs_integration.rs +++ b/provider-node/tests/fs_integration.rs @@ -2,44 +2,43 @@ //! Integration tests for the file system HTTP API endpoints. +mod common; + use axum::http::StatusCode; -use reqwest::Client; +use common::SignedClient; use serde_json::Value; use std::net::SocketAddr; use std::sync::Arc; -use storage_provider_node::{create_router, ProviderState, Storage}; -use tokio::net::TcpListener; +use std::time::Duration; +use storage_primitives::Role; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{NullNonceStore, ProviderDeps, ProviderState, Storage}; struct TestServer { addr: SocketAddr, - client: Client, + client: SignedClient, } impl TestServer { async fn new() -> Self { - let storage = Arc::new(Storage::new()); - // Functional tests, not auth tests: run with auth disabled (auth is - // enforced by default; see auth_integration.rs for the auth coverage). - let state = Arc::new( - ProviderState::with_provider_id(storage, "0xtest_provider".to_string()) - .with_auth_disabled(), - ); - - let app = create_router(state); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - - Self { - addr, - client: Client::new(), - } + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![( + common::test_member_account(), + Role::Admin, + )])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let (addr, client) = common::serve(ProviderState::with_provider_id( + deps, + "0xtest_provider".to_string(), + )) + .await; + Self { addr, client } } fn url(&self, path: &str) -> String { diff --git a/provider-node/tests/negotiate_integration.rs b/provider-node/tests/negotiate_integration.rs index 92e34544..52d30ac6 100644 --- a/provider-node/tests/negotiate_integration.rs +++ b/provider-node/tests/negotiate_integration.rs @@ -15,11 +15,13 @@ use sp_core::{sr25519, Pair}; use sp_runtime::{AccountId32, MultiSignature}; use std::net::SocketAddr; use std::sync::Arc; +use std::time::Duration; use storage_primitives::ReplicaTerms; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; use storage_provider_node::ProviderInfo; use storage_provider_node::{ create_router, DiskStorage, NegotiateRequest, NonceCounter, NonceStore, NullNonceStore, - PalletConstants, ProviderState, SignedTerms, Storage, + PalletConstants, ProviderDeps, ProviderState, SignedTerms, Storage, }; use tokio::net::TcpListener; @@ -36,8 +38,16 @@ impl TestServer { /// `//Alice`-signed server whose state advertises `info` on-chain and has a /// nonce counter ready, i.e. every `/negotiate` prerequisite satisfied. async fn ready(info: ProviderInfo) -> Self { - let state = ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED) - .expect("//Alice is a valid SURI"); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, PROVIDER_SEED).expect("//Alice is a valid SURI"); // Simulate what the coordinator does once registration lands: publish // constants, bootstrap the nonce counter, then publish provider_info. // Together these satisfy every `/negotiate` prerequisite. @@ -230,8 +240,17 @@ async fn negotiate_accepts_replica_when_sync_price_configured() { #[tokio::test] async fn negotiate_503_when_no_signing_key() { // No keypair configured → the handler refuses before doing any work. + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; let server = TestServer::serve(Arc::new(ProviderState::with_provider_id( - Arc::new(Storage::new()), + deps, "0xtest_provider".to_string(), ))) .await; @@ -247,7 +266,16 @@ async fn negotiate_503_when_provider_info_unavailable() { // Keypair present and chain state ready, but no on-chain registration info // loaded (the reconciler never published it): the node cannot validate terms // it would be bound to, so it must refuse. `provider_info` defaults to `None`. - let state = ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED).unwrap(); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, PROVIDER_SEED).unwrap(); state .chain_state .current_block @@ -333,7 +361,16 @@ async fn negotiate_transitions_to_info_unavailable_after_complete_deregister() { // returns None → clears provider_info. Subsequent negotiate calls should // return provider_info_unavailable, not provider_deregistering (the info is // just gone at that point). - let state = ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED).unwrap(); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, PROVIDER_SEED).unwrap(); state .chain_state .current_block @@ -383,7 +420,16 @@ async fn negotiate_recovers_after_deregister_cancelled() { // coordinator re-fetches storage which now reports deregister_at = None → // negotiate signs again. Mirrors the coordinator clearing the deregistering // state when a provider backs out of winding down. - let state = ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED).unwrap(); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, PROVIDER_SEED).unwrap(); state .chain_state .current_block @@ -426,7 +472,16 @@ async fn negotiate_503_when_nonce_counter_absent() { // Registered (provider_info loaded) but the coordinator has not yet // published any nonce counter (nonce_counter == None). The handler must // refuse so we never sign a nonce not derived from on-chain state. - let state = ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED).unwrap(); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, PROVIDER_SEED).unwrap(); state .chain_state .current_block @@ -450,7 +505,16 @@ async fn negotiate_503_when_nonce_counter_present_but_not_bootstrapped() { // Some counter but bootstrap_from_hsn has not yet been called (e.g. the // chain returned the provider info but replay state was not yet visible). // The handler must refuse until is_bootstrapped() is true. - let state = ProviderState::with_seed(Arc::new(Storage::new()), PROVIDER_SEED).unwrap(); + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let state = ProviderState::with_seed(deps, PROVIDER_SEED).unwrap(); state .chain_state .current_block diff --git a/provider-node/tests/s3_integration.rs b/provider-node/tests/s3_integration.rs index 6664ab4b..488b26ae 100644 --- a/provider-node/tests/s3_integration.rs +++ b/provider-node/tests/s3_integration.rs @@ -2,44 +2,43 @@ //! Integration tests for S3-compatible object storage endpoints. +mod common; + use axum::http::StatusCode; -use reqwest::Client; +use common::SignedClient; use serde_json::Value; use std::net::SocketAddr; use std::sync::Arc; -use storage_provider_node::{create_router, ProviderState, Storage}; -use tokio::net::TcpListener; +use std::time::Duration; +use storage_primitives::Role; +use storage_provider_node::auth::{MembershipCache, StaticMembershipResolver}; +use storage_provider_node::{NullNonceStore, ProviderDeps, ProviderState, Storage}; struct TestServer { addr: SocketAddr, - client: Client, + client: SignedClient, } impl TestServer { async fn new() -> Self { - let storage = Arc::new(Storage::new()); - // Functional tests, not auth tests: run with auth disabled (auth is - // enforced by default; see auth_integration.rs for the auth coverage). - let state = Arc::new( - ProviderState::with_provider_id(storage, "0xtest_provider".to_string()) - .with_auth_disabled(), - ); - - let app = create_router(state); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - - tokio::spawn(async move { - axum::serve(listener, app).await.unwrap(); - }); - - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - - Self { - addr, - client: Client::new(), - } + let deps = ProviderDeps { + storage: Arc::new(Storage::new()), + nonce_store: Arc::new(NullNonceStore), + membership: Arc::new(MembershipCache::new( + Box::new(StaticMembershipResolver(vec![( + common::test_member_account(), + Role::Admin, + )])), + Duration::from_secs(60), + )), + auth_max_skew: Duration::from_secs(300), + }; + let (addr, client) = common::serve(ProviderState::with_provider_id( + deps, + "0xtest_provider".to_string(), + )) + .await; + Self { addr, client } } fn url(&self, path: &str) -> String { diff --git a/provider/negotiation/Cargo.toml b/provider/negotiation/Cargo.toml index a73baf50..8d0b6da2 100644 --- a/provider/negotiation/Cargo.toml +++ b/provider/negotiation/Cargo.toml @@ -17,4 +17,4 @@ sp-core = { workspace = true, features = ["std"] } sp-runtime = { workspace = true, features = ["std"] } [dev-dependencies] -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["std"] } diff --git a/provider/negotiation/src/http_auth.rs b/provider/negotiation/src/http_auth.rs new file mode 100644 index 00000000..70008faa --- /dev/null +++ b/provider/negotiation/src/http_auth.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Provider HTTP authentication — the signed `Authorization` header format +//! shared by the client SDK (which builds it) and the provider node (which +//! verifies it). + +use storage_primitives::BucketId; + +/// The canonical message a client signs for a bucket-scoped request: +/// `web3storage:::` (`METHOD` upper-case, `timestamp` +/// Unix seconds). The provider rebuilds this exact string to verify the signature. +pub fn auth_message(method: &str, bucket_id: BucketId, timestamp: &str) -> String { + format!("web3storage:{method}:{bucket_id}:{timestamp}") +} + +/// Build the provider's `Authorization` header value: the sr25519 signature of +/// [`auth_message`] formatted as `Web3Storage ::`. +/// `sign` returns the 64-byte signature, keeping this keypair-type agnostic. +pub fn build_auth_header( + pubkey: &[u8; 32], + method: &str, + bucket_id: BucketId, + timestamp: u64, + sign: impl FnOnce(&[u8]) -> [u8; 64], +) -> String { + let timestamp = format!("{timestamp}"); + let signature = sign(auth_message(method, bucket_id, ×tamp).as_bytes()); + format!( + "Web3Storage 0x{}:0x{}:{}", + hex::encode(pubkey), + hex::encode(signature), + timestamp + ) +} diff --git a/provider/negotiation/src/lib.rs b/provider/negotiation/src/lib.rs index 6678fe8f..7a1054ff 100644 --- a/provider/negotiation/src/lib.rs +++ b/provider/negotiation/src/lib.rs @@ -14,6 +14,8 @@ //! * [`sign_terms`] — a helper for provider-side code (tests, fixtures) //! that need to sign terms without going through the full provider //! keystore. +//! * [`http_auth`] — the signed `Authorization` header format for +//! bucket-scoped provider HTTP requests (client builds, provider verifies). //! //! The on-chain pallet hashes `blake2_256(TERM_CONTEXT | SCALE(terms))` — //! `primary-term-v1:` or `replica-term-v1:` depending on the redemption @@ -21,6 +23,10 @@ //! public key, so the same payload has to be built on both sides — //! `sign_terms` enforces that via [`AgreementTerms::signing_payload`]. +pub mod http_auth; + +pub use http_auth::{auth_message, build_auth_header}; + use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr, PickFirst}; use sp_core::hashing::blake2_256; diff --git a/storage-interfaces/file-system/client/Cargo.toml b/storage-interfaces/file-system/client/Cargo.toml index 5f2241f9..c8b3c26c 100644 --- a/storage-interfaces/file-system/client/Cargo.toml +++ b/storage-interfaces/file-system/client/Cargo.toml @@ -21,7 +21,6 @@ reqwest = { workspace = true } sp-core = { workspace = true, features = ["std"] } sp-runtime = { workspace = true, features = ["std"] } subxt = { workspace = true } -subxt-signer = { workspace = true } # Utilities tracing = { workspace = true } @@ -30,6 +29,7 @@ thiserror = { workspace = true } hex = { workspace = true, features = ["std"] } [dev-dependencies] +subxt-signer = { workspace = true } tokio-test = { workspace = true } [[example]] diff --git a/storage-interfaces/file-system/client/README.md b/storage-interfaces/file-system/client/README.md index 5b3b86fd..795f4be0 100644 --- a/storage-interfaces/file-system/client/README.md +++ b/storage-interfaces/file-system/client/README.md @@ -51,7 +51,7 @@ Before using the client, you need: ### Basic Usage ```rust -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; use file_system_primitives::CommitStrategy; #[tokio::main] @@ -59,10 +59,9 @@ async fn main() -> Result<(), Box> { // 1. Connect to blockchain and provider let mut fs_client = FileSystemClient::new( "ws://127.0.0.1:2222", // Parachain endpoint - "http://127.0.0.1:3333" // Provider endpoint + "http://127.0.0.1:3333", // Provider endpoint + Signer::dev("alice")?, // Use Alice's key for testing ) - .await? - .with_dev_signer("alice") // Use Alice's key for testing .await?; // 2. Create a drive (10 GB, 500 blocks duration) @@ -123,33 +122,27 @@ The client uses `subxt` for blockchain interaction: // Connect to parachain let fs_client = FileSystemClient::new( "ws://127.0.0.1:2222", // Your parachain WebSocket - "http://127.0.0.1:3333" // Your provider HTTP endpoint + "http://127.0.0.1:3333", // Your provider HTTP endpoint + signer, ) .await?; ``` ### Setting Up a Signer -For development, use dev accounts: +The signer is passed at construction; it signs both on-chain extrinsics and +provider HTTP requests. For development, use dev accounts: ```rust -// Use a development account -let fs_client = fs_client - .with_dev_signer("alice") // alice, bob, charlie, dave, eve, ferdie - .await?; +// A well-known development account +let signer = Signer::dev("alice")?; // alice, bob, charlie, dave, eve, ferdie ``` -For production, use real keypairs: +For production, derive from a secret URI / mnemonic or wrap an existing keypair: ```rust -use subxt_signer::sr25519::Keypair; - -// Load from seed phrase or file -let keypair = Keypair::from_seed("your seed phrase here")?; - -let fs_client = fs_client - .with_signer(keypair) - .await?; +let signer = Signer::from_seed("your seed phrase here")?; +// or: Signer::from_keypair(keypair) ``` ### On-Chain Operations diff --git a/storage-interfaces/file-system/client/examples/basic_usage.rs b/storage-interfaces/file-system/client/examples/basic_usage.rs index d4df46a5..d711cd8b 100644 --- a/storage-interfaces/file-system/client/examples/basic_usage.rs +++ b/storage-interfaces/file-system/client/examples/basic_usage.rs @@ -21,7 +21,7 @@ //! cargo run --example basic_usage [chain_ws] [provider_url] //! ``` -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; use sp_runtime::AccountId32; use std::env; use storage_client::{NegotiateRequest, ProviderClient}; @@ -49,10 +49,8 @@ async fn main() -> Result<(), Box> { // === STEP 1: Create the client === println!("\n📡 Step 1: Connecting to blockchain and provider..."); - let mut fs_client = FileSystemClient::new(chain_ws, provider_url) - .await? - .with_dev_signer("alice") // Use Alice for testing - .await?; + let mut fs_client = + FileSystemClient::new(chain_ws, provider_url, Signer::dev("alice")?).await?; let owner: AccountId32 = dev_signer::alice().public_key().0.into(); let provider = ProviderClient::fetch_provider_id(provider_url).await?; diff --git a/storage-interfaces/file-system/client/examples/ci_integration_test.rs b/storage-interfaces/file-system/client/examples/ci_integration_test.rs index aa409451..cf6d5646 100644 --- a/storage-interfaces/file-system/client/examples/ci_integration_test.rs +++ b/storage-interfaces/file-system/client/examples/ci_integration_test.rs @@ -15,7 +15,7 @@ //! //! Run via justfile (recommended): just fs-demo-ci -use file_system_client::FileSystemClient; +use file_system_client::{FileSystemClient, Signer}; use file_system_primitives::DirectoryEntry; use sp_runtime::AccountId32; use std::env; @@ -65,10 +65,8 @@ async fn main() -> Result<(), Box> { // Step 1: Create the client println!("Step 1: Creating file system client..."); - let mut fs_client = FileSystemClient::new(chain_ws, provider_url) - .await? - .with_dev_signer("alice") - .await?; + let mut fs_client = + FileSystemClient::new(chain_ws, provider_url, Signer::dev("alice")?).await?; println!(" Client connected successfully"); let owner: AccountId32 = dev_signer::alice().public_key().0.into(); diff --git a/storage-interfaces/file-system/client/src/lib.rs b/storage-interfaces/file-system/client/src/lib.rs index bb778fef..e13cebe0 100644 --- a/storage-interfaces/file-system/client/src/lib.rs +++ b/storage-interfaces/file-system/client/src/lib.rs @@ -55,7 +55,7 @@ use thiserror::Error; use tokio::sync::Mutex; pub use file_system_primitives::DriveId; -pub use storage_client::{CheckpointConfig, CheckpointResult}; +pub use storage_client::{CheckpointConfig, CheckpointResult, Signer}; pub use substrate::SubstrateClient; /// File system client errors @@ -131,13 +131,24 @@ impl FileSystemClient { /// /// * `chain_endpoint` - Parachain WebSocket RPC endpoint (e.g., "ws://127.0.0.1:2222") /// * `provider_endpoint` - Storage provider HTTP endpoint - pub async fn new(chain_endpoint: &str, provider_endpoint: &str) -> Result { - let storage_client = StorageUserClient::new(ClientConfig { - provider_urls: vec![provider_endpoint.to_string()], - ..Default::default() - }) + /// + /// The `signer` authenticates provider requests and signs on-chain + /// extrinsics; build it via [`Signer::from_seed`], [`Signer::dev`], or + /// [`Signer::from_keypair`]. + pub async fn new( + chain_endpoint: &str, + provider_endpoint: &str, + signer: Signer, + ) -> Result { + let storage_client = StorageUserClient::new( + ClientConfig { + provider_urls: vec![provider_endpoint.to_string()], + ..Default::default() + }, + signer.clone(), + ) .map_err(|e| FsClientError::Config(e.to_string()))?; - let substrate_client = SubstrateClient::connect(chain_endpoint).await?; + let substrate_client = SubstrateClient::connect(chain_endpoint, signer).await?; Ok(Self { storage_client, @@ -148,18 +159,6 @@ impl FileSystemClient { }) } - /// Create a client with a development signer (for testing). - pub async fn with_dev_signer(mut self, name: &str) -> Result { - self.substrate_client = self.substrate_client.with_dev_signer(name)?; - Ok(self) - } - - /// Set a custom signer for blockchain transactions. - pub fn with_signer(mut self, signer: subxt_signer::sr25519::Keypair) -> Self { - self.substrate_client = self.substrate_client.with_signer(signer); - self - } - /// Create a new drive (USER-FACING API) /// /// This is the primary way for users to create drives. The system automatically: @@ -451,11 +450,7 @@ impl FileSystemClient { let manager = manager.with_providers(provider_endpoints); // Use the same signer as the file system client - let manager = if let Ok(signer) = self.substrate_client.signer_keypair() { - manager.with_signer(signer.clone()) - } else { - manager - }; + let manager = manager.with_signer(self.substrate_client.signer().clone()); // Submit checkpoint Ok(manager.submit_checkpoint(bucket_id).await) @@ -479,11 +474,7 @@ impl FileSystemClient { let manager = manager.with_providers(provider_endpoints); - let manager = if let Ok(signer) = self.substrate_client.signer_keypair() { - manager.with_signer(signer.clone()) - } else { - manager - }; + let manager = manager.with_signer(self.substrate_client.signer().clone()); Ok(manager.submit_checkpoint(bucket_id).await) } @@ -559,11 +550,7 @@ impl FileSystemClient { let manager = manager.with_providers(provider_endpoints); // Use the same signer as the file system client - let manager = if let Ok(signer) = self.substrate_client.signer_keypair() { - manager.with_signer(signer.clone()) - } else { - manager - }; + let manager = manager.with_signer(self.substrate_client.signer().clone()); // Configure batched checkpoint loop let batched_config = BatchedCheckpointConfig { @@ -890,7 +877,7 @@ impl FileSystemClient { let call = substrate::extrinsics::create_drive(name_bytes, provider, terms, sig); // Sign and submit - let signer = self.substrate_client.signer()?; + let signer = self.substrate_client.signer(); let events = self .substrate_client .api() diff --git a/storage-interfaces/file-system/client/src/substrate.rs b/storage-interfaces/file-system/client/src/substrate.rs index 705745da..43549e30 100644 --- a/storage-interfaces/file-system/client/src/substrate.rs +++ b/storage-interfaces/file-system/client/src/substrate.rs @@ -9,12 +9,10 @@ use file_system_primitives::DriveId; use sp_core::H256; use sp_runtime::AccountId32; use std::str::FromStr; -use std::sync::Arc; -use storage_client::{scale_decode, EventParser}; +use storage_client::{scale_decode, EventParser, Signer}; use storage_primitives::Role; use subxt::ext::scale_value::{At, Composite, Value}; use subxt::{OnlineClient, PolkadotConfig}; -use subxt_signer::sr25519::Keypair; /// Pallet name in the runtime configuration. pub const PALLET_NAME: &str = "DriveRegistry"; @@ -23,72 +21,32 @@ pub const PALLET_NAME: &str = "DriveRegistry"; #[derive(Clone)] pub struct SubstrateClient { api: OnlineClient, - signer: Option>, + signer: Signer, endpoint: String, } impl SubstrateClient { /// Connect to a substrate node. - pub async fn connect(ws_url: &str) -> Result { + pub async fn connect(ws_url: &str, signer: Signer) -> Result { let api = OnlineClient::::from_url(ws_url) .await .map_err(|e| FsClientError::Blockchain(format!("Connection failed: {e}")))?; Ok(Self { api, - signer: None, + signer, endpoint: ws_url.to_string(), }) } - /// Set the signer for this client. - pub fn with_signer(mut self, signer: Keypair) -> Self { - self.signer = Some(Arc::new(signer)); - self - } - - /// Create a client with a development keypair (for testing). - pub fn with_dev_signer(mut self, name: &str) -> Result { - use subxt_signer::sr25519::dev; - - let keypair = match name { - "alice" => dev::alice(), - "bob" => dev::bob(), - "charlie" => dev::charlie(), - "dave" => dev::dave(), - "eve" => dev::eve(), - "ferdie" => dev::ferdie(), - _ => { - return Err(FsClientError::Config(format!( - "Unknown dev account: {name}" - ))) - } - }; - self.signer = Some(Arc::new(keypair)); - Ok(self) - } - /// Get the API client. pub fn api(&self) -> &OnlineClient { &self.api } - /// Get the signer if available. - pub fn signer(&self) -> Result<&Keypair, FsClientError> { - self.signer - .as_ref() - .map(|s| s.as_ref()) - .ok_or(FsClientError::NoSigner) - } - - /// Get the signer keypair (cloned) if available. - /// - /// This is useful when you need to pass the keypair to another component. - pub fn signer_keypair(&self) -> Result { - self.signer - .as_ref() - .map(|s| (**s).clone()) - .ok_or(FsClientError::NoSigner) + /// The signer. + pub fn signer(&self) -> &Signer { + &self.signer } /// Get the WebSocket endpoint URL. diff --git a/storage-interfaces/s3/README.md b/storage-interfaces/s3/README.md index 68410aa8..0e9c09aa 100644 --- a/storage-interfaces/s3/README.md +++ b/storage-interfaces/s3/README.md @@ -39,7 +39,7 @@ This module provides an S3-compatible storage interface (Layer 1) on top of the ## Quick Start ```rust -use s3_client::{S3Client, PutObjectOptions}; +use s3_client::{PutObjectOptions, S3Client, Signer}; #[tokio::main] async fn main() -> Result<(), Box> { @@ -47,7 +47,7 @@ async fn main() -> Result<(), Box> { let client = S3Client::new( "ws://127.0.0.1:2222", // Chain URL "http://localhost:3333", // Provider URL - "//Alice", // Seed phrase + Signer::from_seed("//Alice")?, // Signer ).await?; // Create bucket diff --git a/storage-interfaces/s3/client/Cargo.toml b/storage-interfaces/s3/client/Cargo.toml index 3291d830..b5fda16d 100644 --- a/storage-interfaces/s3/client/Cargo.toml +++ b/storage-interfaces/s3/client/Cargo.toml @@ -22,8 +22,6 @@ tokio = { workspace = true } # Subxt for chain interaction subxt = { workspace = true } -subxt-signer = { workspace = true } -bip39 = { workspace = true } # Utilities hex = { workspace = true, features = ["std"] } @@ -32,6 +30,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } [dev-dependencies] +subxt-signer = { workspace = true } tokio = { workspace = true } [[example]] diff --git a/storage-interfaces/s3/client/examples/basic_usage.rs b/storage-interfaces/s3/client/examples/basic_usage.rs index 94f38334..d24126ed 100644 --- a/storage-interfaces/s3/client/examples/basic_usage.rs +++ b/storage-interfaces/s3/client/examples/basic_usage.rs @@ -4,7 +4,7 @@ //! //! Usage: cargo run --example basic_usage [chain_ws] [provider_url] [seed] -use s3_client::{PutObjectOptions, S3Client}; +use s3_client::{PutObjectOptions, S3Client, Signer}; use sp_runtime::AccountId32; use std::collections::HashMap; use std::env; @@ -33,7 +33,7 @@ async fn main() -> Result<(), Box> { println!("Account: {seed}\n"); println!("Creating S3 client..."); - let client = S3Client::new(chain_url, provider_url, seed).await?; + let client = S3Client::new(chain_url, provider_url, Signer::from_seed(seed)?).await?; println!("S3 client created successfully!\n"); // Resolve owner/provider accounts. Owner derives from `seed`; provider is diff --git a/storage-interfaces/s3/client/examples/ci_integration_test.rs b/storage-interfaces/s3/client/examples/ci_integration_test.rs index 3d1562dc..81a24129 100644 --- a/storage-interfaces/s3/client/examples/ci_integration_test.rs +++ b/storage-interfaces/s3/client/examples/ci_integration_test.rs @@ -17,7 +17,7 @@ //! //! Usage: cargo run --example ci_integration_test [chain_ws] [provider_url] -use s3_client::{PutObjectOptions, S3Client}; +use s3_client::{PutObjectOptions, S3Client, Signer}; use sp_runtime::AccountId32; use std::collections::HashMap; use std::env; @@ -47,7 +47,7 @@ async fn main() -> Result<(), Box> { // Step 1: Create the S3 client println!("Step 1: Creating S3 client..."); let owner: AccountId32 = dev_signer::bob().public_key().0.into(); - let client = S3Client::new(chain_ws, provider_url, "//Bob").await?; + let client = S3Client::new(chain_ws, provider_url, Signer::from_seed("//Bob")?).await?; println!(" Client connected successfully"); // Discover the provider's on-chain account from its /info endpoint diff --git a/storage-interfaces/s3/client/src/lib.rs b/storage-interfaces/s3/client/src/lib.rs index 591cd953..8bfb94df 100644 --- a/storage-interfaces/s3/client/src/lib.rs +++ b/storage-interfaces/s3/client/src/lib.rs @@ -6,6 +6,7 @@ mod substrate; +pub use storage_client::Signer; pub use substrate::SubstrateClient; use s3_primitives::{ @@ -134,7 +135,7 @@ pub struct S3Client { impl S3Client { /// Create a new S3 client. - pub async fn new(chain_url: &str, provider_url: &str, seed_phrase: &str) -> Result { + pub async fn new(chain_url: &str, provider_url: &str, signer: Signer) -> Result { info!( "Creating S3 client with chain={}, provider={}", chain_url, provider_url @@ -145,10 +146,10 @@ impl S3Client { provider_urls: vec![provider_url.to_string()], ..Default::default() }; - let storage_client = storage_client::StorageUserClient::new(config) + let storage_client = storage_client::StorageUserClient::new(config, signer.clone()) .map_err(|e| S3ClientError::ProviderError(e.to_string()))?; - let substrate_client = SubstrateClient::new(chain_url, seed_phrase) + let substrate_client = SubstrateClient::new(chain_url, signer) .await .map_err(|e| S3ClientError::ChainError(e.to_string()))?; diff --git a/storage-interfaces/s3/client/src/substrate.rs b/storage-interfaces/s3/client/src/substrate.rs index f5916616..3db70b3e 100644 --- a/storage-interfaces/s3/client/src/substrate.rs +++ b/storage-interfaces/s3/client/src/substrate.rs @@ -6,11 +6,9 @@ use crate::{BucketInfo, S3ClientError}; use s3_primitives::{ListObjectsParams, ListObjectsResponse, S3BucketId}; use sp_core::H256; use sp_runtime::AccountId32; -use std::sync::Arc; -use storage_client::{scale_decode, EventParser}; +use storage_client::{scale_decode, EventParser, Signer}; use subxt::ext::scale_value::{At, Composite, Value, ValueDef}; use subxt::{OnlineClient, PolkadotConfig}; -use subxt_signer::sr25519::Keypair; use tracing::{debug, info}; /// Pallet name in the runtime configuration. @@ -39,8 +37,8 @@ pub struct MetadataEntry { pub struct SubstrateClient { /// Subxt online client. client: OnlineClient, - /// Signer for transactions. - signer: Option>, + /// Signer for extrinsics and provider auth. + signer: Signer, /// Account ID (32 bytes). account_id: [u8; 32], /// Endpoint URL. @@ -50,52 +48,24 @@ pub struct SubstrateClient { impl SubstrateClient { /// Create a new substrate client. - pub async fn new(chain_url: &str, seed_phrase: &str) -> std::result::Result { + pub async fn new(chain_url: &str, signer: Signer) -> std::result::Result { info!("Connecting to chain at {}", chain_url); let client = OnlineClient::::from_url(chain_url) .await .map_err(|e| format!("Failed to connect to chain: {e}"))?; - let keypair = if seed_phrase.starts_with("//") { - // Dev account like //Alice - match seed_phrase { - "//Alice" => subxt_signer::sr25519::dev::alice(), - "//Bob" => subxt_signer::sr25519::dev::bob(), - "//Charlie" => subxt_signer::sr25519::dev::charlie(), - "//Dave" => subxt_signer::sr25519::dev::dave(), - "//Eve" => subxt_signer::sr25519::dev::eve(), - "//Ferdie" => subxt_signer::sr25519::dev::ferdie(), - _ => return Err(format!("Unknown dev account: {seed_phrase}")), - } - } else { - // Mnemonic phrase - parse and create keypair - let mnemonic = bip39::Mnemonic::parse(seed_phrase) - .map_err(|e| format!("Invalid mnemonic: {e:?}"))?; - subxt_signer::sr25519::Keypair::from_phrase(&mnemonic, None) - .map_err(|e| format!("Failed to create keypair: {e:?}"))? - }; - - let public_key = keypair.public_key(); - let account_id: [u8; 32] = public_key.0; + let account_id: [u8; 32] = signer.keypair().public_key().0; info!("Connected to chain, account: 0x{}", hex::encode(account_id)); Ok(Self { client, - signer: Some(Arc::new(keypair)), + signer, account_id, endpoint: chain_url.to_string(), }) } - /// Get the signer keypair. - fn signer(&self) -> std::result::Result<&Keypair, String> { - self.signer - .as_ref() - .map(|s| s.as_ref()) - .ok_or_else(|| "No signer configured".to_string()) - } - /// Sign, submit, and wait for a transaction to finalize successfully. /// /// Retries on stale-nonce (error 1010) which can happen when submitting @@ -105,8 +75,6 @@ impl SubstrateClient { &self, tx: subxt::tx::DynamicPayload>, ) -> std::result::Result, String> { - let signer = self.signer()?; - let mut last_err = String::new(); for attempt in 0..3u32 { let at = match self.client.at_current_block().await { @@ -115,7 +83,7 @@ impl SubstrateClient { }; match at .transactions() - .sign_and_submit_then_watch_default(&tx, signer) + .sign_and_submit_then_watch_default(&tx, &self.signer) .await { Ok(progress) => { diff --git a/user-interfaces/drive-ui/src/lib/drive-client.ts b/user-interfaces/drive-ui/src/lib/drive-client.ts index 3e090267..e739ec4e 100644 --- a/user-interfaces/drive-ui/src/lib/drive-client.ts +++ b/user-interfaces/drive-ui/src/lib/drive-client.ts @@ -11,7 +11,6 @@ import type { PolkadotSigner } from "polkadot-api"; import { parachain } from "@polkadot-api/descriptors"; -import { ss58Decode } from "@polkadot-labs/hdkd-helpers"; import { buildSignedTermsArgs, createDrive as createDriveTx, @@ -19,6 +18,7 @@ import { parseMultiaddrToUrl, toSs58, type ChainSigner, + type Keypair, type NegotiateRequest, type SignedTerms, } from "@web3-storage/sdk"; @@ -111,6 +111,7 @@ export class DriveClient { private api: ParachainApi | null = null; private signer: Signer | null = null; private signerAddress: string | null = null; + private keypair: Keypair | null = null; private fsc: FileSystemClient | null = null; private rebuild(): void { @@ -119,15 +120,12 @@ export class DriveClient { return; } let chainSigner: ChainSigner | null = null; - if (this.signer && this.signerAddress) { - // Wallet flows hand us a PolkadotSigner + address; recover the public - // key from the address. No raw keypair here, so provider requests go - // unsigned (same as this app always behaved). - const [publicKey] = ss58Decode(this.signerAddress); + if (this.signer && this.signerAddress && this.keypair) { chainSigner = { signer: this.signer, address: this.signerAddress, - publicKey, + publicKey: this.keypair.publicKey, + keypair: this.keypair, }; } this.fsc = new FileSystemClient({ api: this.api, signer: chainSigner }); @@ -140,9 +138,14 @@ export class DriveClient { } } - setSigner(signer: Signer | null, address: string | null): void { + setSigner( + signer: Signer | null, + address: string | null, + keypair: Keypair | null, + ): void { this.signer = signer; this.signerAddress = address; + this.keypair = keypair; this.rebuild(); } @@ -352,12 +355,14 @@ export class DriveClient { signed: SignedTerms, ): Promise { const api = this.requireApi(); - if (!this.signer || !this.signerAddress) throw new Error("Signer not set"); - const [publicKey] = ss58Decode(this.signerAddress); + if (!this.signer || !this.signerAddress || !this.keypair) { + throw new Error("Signer not set"); + } const owner: ChainSigner = { signer: this.signer, address: this.signerAddress, - publicKey, + publicKey: this.keypair.publicKey, + keypair: this.keypair, }; // Finalize: the state layer reads the drive list back at the finalized diff --git a/user-interfaces/drive-ui/src/state/drive.state.ts b/user-interfaces/drive-ui/src/state/drive.state.ts index b75dea18..dc521958 100644 --- a/user-interfaces/drive-ui/src/state/drive.state.ts +++ b/user-interfaces/drive-ui/src/state/drive.state.ts @@ -20,7 +20,7 @@ import { type SignedTerms, } from "@/lib/drive-client"; import { api$$, getApi } from "@/state/chain.state"; -import { signer$$, signerAddress$$, getSignerAddress, refreshBalance } from "@/state/wallet.state"; +import { signer$$, keypair$$, signerAddress$$, getSignerAddress, refreshBalance } from "@/state/wallet.state"; // ───────────────────────────────────────────────────────────────────────────── // Types @@ -86,12 +86,11 @@ api$$.subscribe((api) => { client.setApi(api); }); -// Use combineLatest so the client always receives signer + address together. -// With separate subscriptions, signer$ fires before signerAddress$ inside -// setSigner(), leaving client.signerAddress null when refreshDrives() runs. -combineLatest([signer$$, signerAddress$$]).subscribe(([signer, address]) => { - client.setSigner(signer, address); -}); +combineLatest([signer$$, signerAddress$$, keypair$$]).subscribe( + ([signer, address, keypair]) => { + client.setSigner(signer, address, keypair); + }, +); export function getDriveClient(): DriveClient { return client; diff --git a/user-interfaces/drive-ui/src/state/wallet.state.ts b/user-interfaces/drive-ui/src/state/wallet.state.ts index 9363b35d..76c30245 100644 --- a/user-interfaces/drive-ui/src/state/wallet.state.ts +++ b/user-interfaces/drive-ui/src/state/wallet.state.ts @@ -14,7 +14,7 @@ import { BehaviorSubject, combineLatest, map } from "rxjs"; import { bind } from "@react-rxjs/core"; import { getPolkadotSigner } from "polkadot-api/signer"; import { getApi } from "@/state/chain.state"; -import { seedToKeypair, toSs58 } from "@/lib/crypto"; +import { seedToKeypair, toSs58, type Keypair } from "@/lib/crypto"; export type Signer = ReturnType; @@ -41,6 +41,7 @@ const STORAGE_KEY_ACCOUNT_NAME = "drive-ui-account-name"; const signer$ = new BehaviorSubject(null); const signerAddress$ = new BehaviorSubject(null); const signerName$ = new BehaviorSubject(null); +const keypair$ = new BehaviorSubject(null); const balance$ = new BehaviorSubject(null); const settingSigner$ = new BehaviorSubject(false); @@ -64,6 +65,7 @@ export async function setSigner(seed: string, name?: string): Promise { keypair.sign(input), ); + keypair$.next(keypair); signer$.next(newSigner); signerAddress$.next(address); signerName$.next(name ?? null); @@ -88,6 +90,7 @@ export async function selectDevAccount(name: string): Promise { } export function clearSigner(): void { + keypair$.next(null); signer$.next(null); signerAddress$.next(null); signerName$.next(null); @@ -133,6 +136,7 @@ export function getSignerAddress(): string | null { } export const signer$$ = signer$.asObservable(); +export const keypair$$ = keypair$.asObservable(); export const signerAddress$$ = signerAddress$.asObservable(); export const signerInfo$ = combineLatest([signerAddress$, signerName$]).pipe( diff --git a/user-interfaces/s3-ui/src/lib/s3-client.ts b/user-interfaces/s3-ui/src/lib/s3-client.ts index 4cdbfdb7..af5ac649 100644 --- a/user-interfaces/s3-ui/src/lib/s3-client.ts +++ b/user-interfaces/s3-ui/src/lib/s3-client.ts @@ -15,7 +15,6 @@ import { Subscription } from "rxjs"; import type { PolkadotSigner } from "polkadot-api"; import { parachain } from "@polkadot-api/descriptors"; -import { ss58Decode } from "@polkadot-labs/hdkd-helpers"; import { getSs58AddressInfo } from "@polkadot-api/substrate-bindings"; import { buildSignedTermsArgs, @@ -187,16 +186,12 @@ export class S3Client { return; } let chainSigner: ChainSigner | null = null; - if (this.signer && this.signerAddress) { - // s3-ui derives raw dev-account keypairs, so provider requests are - // signed (the SDK's S3Client reads `signer.keypair`). Fall back to the - // address-recovered public key if only a wallet signer is present. - const publicKey = this.keypair?.publicKey ?? ss58Decode(this.signerAddress)[0]; + if (this.signer && this.signerAddress && this.keypair) { chainSigner = { signer: this.signer, address: this.signerAddress, - publicKey, - keypair: this.keypair ?? undefined, + publicKey: this.keypair.publicKey, + keypair: this.keypair, }; } this.owner = chainSigner; @@ -210,13 +205,13 @@ export class S3Client { } } - setSigner(signer: Signer | null, address: string | null): void { + setSigner( + signer: Signer | null, + address: string | null, + keypair: Keypair | null, + ): void { this.signer = signer; this.signerAddress = address; - this.rebuild(); - } - - setKeypair(keypair: Keypair | null): void { this.keypair = keypair; this.rebuild(); } diff --git a/user-interfaces/s3-ui/src/state/s3.state.ts b/user-interfaces/s3-ui/src/state/s3.state.ts index 392f3bd1..965a82c8 100644 --- a/user-interfaces/s3-ui/src/state/s3.state.ts +++ b/user-interfaces/s3-ui/src/state/s3.state.ts @@ -84,13 +84,11 @@ api$$.subscribe((api) => { client.setApi(api); }); -combineLatest([signer$$, signerAddress$$]).subscribe(([signer, address]) => { - client.setSigner(signer, address); -}); - -keypair$$.subscribe((keypair) => { - client.setKeypair(keypair); -}); +combineLatest([signer$$, signerAddress$$, keypair$$]).subscribe( + ([signer, address, keypair]) => { + client.setSigner(signer, address, keypair); + }, +); export function getS3Client(): S3Client { return client;