Added content-defined chunking

CLAUDE.md

@@ -15,6 +15,13 @@
 - ALWAYS run `/format` before creating any git commit
 - This ensures all code follows project formatting standards (Rust, TOML, feature propagation) and passes clippy
 
+**File headers:**
+- Every new source file (Rust, TypeScript, JavaScript, Solidity) MUST start with an SPDX license header on the first line, matching the surrounding crate / package. Check a sibling file in the same crate to copy the exact identifier (`Apache-2.0`, `GPL-3.0-only`, etc.).
+- Examples: `// SPDX-License-Identifier: Apache-2.0` for the `primitives` crate, `// SPDX-License-Identifier: GPL-3.0-only` for `provider-node` and the `user-interfaces/` workspace.
+
+**Dependencies:**
+- New external dependencies go in the workspace `Cargo.toml` under `[workspace.dependencies]`, then each consuming crate references them via `{ workspace = true }`. Don't inline version literals in per-crate manifests; that includes `[dev-dependencies]`.
+
 ## Project Overview
 
 Scalable Web3 Storage is a decentralized storage system built on Substrate with game-theoretic guarantees. Storage providers lock stake and face slashing for data loss, while the chain acts as a credible threat rather than the hot path.

Cargo.toml

@@ -140,17 +140,22 @@ tracing-subscriber = { version = "=0.3.19" }
 axum = "0.7"
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
 tokio = { version = "1.0", features = ["full"] }
+tokio-stream = "0.1"
 tower = "0.5"
 tower-http = { version = "0.6", features = ["cors", "trace"] }
 
 # Crypto
 blake2 = { version = "0.10", default-features = false }
 
+# Chunking
+fastcdc = { version = "3.1" }
+
 # Subxt (chain interaction for off-chain clients)
 subxt = { version = "0.44.3" }
 subxt-signer = { version = "0.44.3", features = ["sr25519"] }
 
 # Testing
+rand = "0.8"
 sp-keyring = { version = "46.0.0", default-features = false }
 sp-keystore = { version = "0.46.0", default-features = false }
 

client/src/base.rs

@@ -211,25 +211,18 @@ pub struct AgreementInfo {
 }
 
 /// Chunking strategy for data upload.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, Default)]
 pub enum ChunkingStrategy {
-    /// Fixed-size chunks (default: 256 KiB)
+    /// Fixed-size chunks of the given size. A byte-shifting edit invalidates
+    /// every chunk after the edit point, so cross-version dedup only survives
+    /// in-place overwrites and appends.
     Fixed(usize),
-    /// TODO: Content-defined chunking (not yet implemented)
-    ///
-    /// ContentDefined {
-    ///     min_size: usize,
-    ///     target_size: usize,
-    ///     max_size: usize,
-    /// },
+    /// Content-defined chunks via FastCDC (parameters in
+    /// `storage_primitives::chunking`). Boundaries track content, so an
+    /// insertion only changes the chunks straddling the edit — the default.
+    #[default]
     ContentDefined,
 }
 
-impl Default for ChunkingStrategy {
-    fn default() -> Self {
-        Self::Fixed(256 * 1024)
-    }
-}
-
 /// Result type for client operations.
 pub type ClientResult<T> = Result<T, ClientError>;

client/src/storage_user.rs

@@ -14,7 +14,7 @@ use crate::encryption::{Cipher, EncryptionKey, XChaCha20Poly1305Cipher};
 use crate::verification::ClientVerifier;
 use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
 use sp_core::H256;
-use storage_primitives::{blake2_256, BucketId};
+use storage_primitives::{blake2_256, verify_merkle_proof, BucketId, MerkleProof};
 
 /// Client for storage users (end users who store/retrieve data).
 pub struct StorageUserClient {
@@ -169,61 +169,124 @@ impl StorageUserClient {
         offset: u64,
         length: u64,
     ) -> ClientResult<Vec<u8>> {
+        // Walk chunks by index, verifying each against `data_root`, until we've
+        // covered the requested range or the provider signals end-of-file.
+        // Content-defined chunking makes chunk sizes variable, so a byte offset
+        // can't be mapped to a chunk index up front — we walk from chunk 0. In
+        // practice callers read from offset 0 (whole object or prefix), so this
+        // is a single forward pass; `spot_check` fetches a single chunk by index
+        // directly rather than going through here.
+        let end = offset.saturating_add(length);
+        let mut data = Vec::new();
+        let mut chunk_index = 0u64;
+        while (data.len() as u64) < end {
+            match self.fetch_chunk_verified(data_root, chunk_index).await? {
+                Some(chunk) => data.extend_from_slice(&chunk),
+                None => break, // no more chunks
+            }
+            chunk_index += 1;
+        }
+
+        // Slice to the requested range (clamped — a short object yields fewer
+        // bytes rather than erroring).
+        let start = (offset as usize).min(data.len());
+        let stop = (end as usize).min(data.len());
+        let ranged = data[start..stop].to_vec();
+
+        // Decrypt after reassembly if encryption is enabled.
+        if let Some(cipher) = &self.cipher {
+            cipher.decrypt(&ranged)
+        } else {
+            Ok(ranged)
+        }
+    }
+
+    /// Fetch a single chunk by index and verify its Merkle proof against
+    /// `data_root`. Returns `Ok(None)` when `chunk_index` is past the last
+    /// chunk (the provider's 404 end-of-file signal), so callers can iterate
+    /// `0..` until they hit it.
+    ///
+    /// Uses `/chunk_proof`, which is index-based and therefore works for any
+    /// chunking strategy — unlike `/read`, whose fixed-size offset math the
+    /// provider rejects for content-defined roots. This is also strictly
+    /// stronger than the old `/read` path: it verifies the chunk actually sits
+    /// under `data_root`, not just that the bytes match a hash the provider
+    /// itself supplied.
+    async fn fetch_chunk_verified(
+        &self,
+        data_root: &H256,
+        chunk_index: u64,
+    ) -> ClientResult<Option<Vec<u8>>> {
         let provider_url = self.base.get_provider_url()?;
 
         let response = self
             .base
             .http
-            .get(format!("{provider_url}/read"))
+            .get(format!("{provider_url}/chunk_proof"))
             .query(&[
                 ("data_root", BaseClient::hex_encode(data_root.as_bytes())),
-                ("offset", offset.to_string()),
-                ("length", length.to_string()),
+                ("chunk_index", chunk_index.to_string()),
             ])
             .send()
             .await?;
 
+        // 404 means `chunk_index` ran past the last leaf — the loop's stop
+        // condition, not an error.
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
         if !response.status().is_success() {
             return Err(ClientError::Api(format!(
                 "Provider returned error: {}",
                 response.status()
             )));
         }
 
-        let read_response: ReadResponse = response.json().await?;
-
-        let mut data = Vec::new();
-        for chunk in read_response.chunks {
-            let chunk_data = BASE64
-                .decode(&chunk.data)
-                .map_err(|e| ClientError::Serialization(e.to_string()))?;
-
-            // Verify chunk hash
-            let expected_hash = BaseClient::hex_decode(&chunk.hash)?;
-            let actual_hash = blake2_256(&chunk_data);
-            if actual_hash.as_bytes() != expected_hash.as_slice() {
-                return Err(ClientError::VerificationFailed);
-            }
+        let proof_response: ChunkProofResponse = response
+            .json()
+            .await
+            .map_err(|e| ClientError::Serialization(e.to_string()))?;
 
-            data.extend_from_slice(&chunk_data);
+        let chunk_data = proof_response
+            .chunk_data
+            .ok_or(ClientError::VerificationFailed)
+            .and_then(|d| {
+                BASE64
+                    .decode(d)
+                    .map_err(|e| ClientError::Serialization(e.to_string()))
+            })?;
+
+        // 1. The leaf hash must be the hash of the bytes we actually received.
+        let leaf_hash = blake2_256(&chunk_data);
+        let claimed_hash = BaseClient::hex_decode(&proof_response.chunk_hash)?;
+        if leaf_hash.as_bytes() != claimed_hash.as_slice() {
+            return Err(ClientError::VerificationFailed);
         }
 
-        // Trim to requested range
-        let chunk_size = 256 * 1024;
-        let start = (offset % chunk_size) as usize;
-        let end = start + length as usize;
-        let trimmed = if end <= data.len() {
-            data[start..end].to_vec()
-        } else {
-            data[start..].to_vec()
+        // 2. That leaf must sit under `data_root` at `chunk_index`.
+        let siblings = proof_response
+            .proof
+            .siblings
+            .iter()
+            .map(|h| {
+                let bytes = BaseClient::hex_decode(h)?;
+                if bytes.len() != 32 {
+                    return Err(ClientError::Serialization(
+                        "proof sibling is not 32 bytes".to_string(),
+                    ));
+                }
+                Ok(H256::from_slice(&bytes))
+            })
+            .collect::<ClientResult<Vec<H256>>>()?;
+        let proof = MerkleProof {
+            siblings,
+            path: proof_response.proof.path,
         };
-
-        // Decrypt after reassembly if encryption is enabled
-        if let Some(cipher) = &self.cipher {
-            cipher.decrypt(&trimmed)
-        } else {
-            Ok(trimmed)
+        if !verify_merkle_proof(leaf_hash, chunk_index, &proof, data_root) {
+            return Err(ClientError::VerificationFailed);
         }
+
+        Ok(Some(chunk_data))
     }
 
     /// Download entire file by data root.
@@ -403,22 +466,22 @@ impl StorageUserClient {
     pub async fn spot_check(&mut self, data_root: &H256, chunk_index: u64) -> ClientResult<bool> {
         use std::time::Instant;
 
-        let chunk_size = 256 * 1024u64; // 256 KiB
-        let offset = chunk_index * chunk_size;
         let provider_url = self.base.get_provider_url()?.to_string();
 
+        // Fetch the chunk by index and verify its Merkle proof against
+        // `data_root`. Index-based, so it's correct regardless of chunk size.
         let start = Instant::now();
-        let result = self.download(data_root, offset, chunk_size).await;
+        let result = self.fetch_chunk_verified(data_root, chunk_index).await;
         let duration = start.elapsed();
 
         match result {
-            Ok(data) if !data.is_empty() => {
+            Ok(Some(_)) => {
                 self.verifier.record_request(&provider_url, duration, true);
                 Ok(true)
             }
             _ => {
-                // Either download failed or the provider returned no data for this chunk,
-                // which means the data is unavailable.
+                // Missing chunk, failed verification, or transport error — the
+                // provider can't prove it holds this chunk.
                 self.verifier.record_request(&provider_url, duration, false);
                 Ok(false)
             }
@@ -467,12 +530,9 @@ impl StorageUserClient {
     fn chunk_data(data: &[u8], strategy: ChunkingStrategy) -> Vec<Vec<u8>> {
         match strategy {
             ChunkingStrategy::Fixed(chunk_size) => {
-                data.chunks(chunk_size).map(|c| c.to_vec()).collect()
-            }
-            ChunkingStrategy::ContentDefined => {
-                // TODO: Implement content-defined chunking
-                Self::chunk_data(data, ChunkingStrategy::Fixed(256 * 1024))
+                storage_primitives::chunking::chunk_fixed(data, chunk_size)
             }
+            ChunkingStrategy::ContentDefined => storage_primitives::chunking::chunk_cdc(data),
         }
     }
 
@@ -686,16 +746,16 @@ pub struct CommitResponse {
 }
 
 #[derive(serde::Deserialize)]
-struct ReadResponse {
-    chunks: Vec<ChunkWithProof>,
+struct ChunkProofResponse {
+    chunk_hash: String,
+    chunk_data: Option<String>,
+    proof: MerkleProofResponse,
 }
 
 #[derive(serde::Deserialize)]
-#[allow(dead_code)]
-struct ChunkWithProof {
-    hash: String,
-    data: String,
-    proof: Vec<String>,
+struct MerkleProofResponse {
+    siblings: Vec<String>,
+    path: Vec<bool>,
 }
 
 #[derive(serde::Deserialize)]