From de7254338f1b2280a95778f82b8b563f525f9687 Mon Sep 17 00:00:00 2001 From: Nick Furfaro Date: Mon, 27 Oct 2025 11:56:41 -0600 Subject: [PATCH 1/3] docs: add inline docs and README Also adds a perf test --- apps/increment/README.md | 395 ++++++++++++++++++++++++++++++++++ apps/increment/src/data.rs | 52 +++++ apps/increment/src/db.rs | 25 +++ apps/increment/src/lib.rs | 210 +++++++++++++++++- apps/increment/src/main.rs | 35 +++ apps/increment/src/proof.rs | 15 ++ apps/increment/src/server.rs | 20 ++ apps/increment/src/state.rs | 19 ++ apps/increment/tests/tests.rs | 232 ++++++++++++++++++++ 9 files changed, 994 insertions(+), 9 deletions(-) create mode 100644 apps/increment/README.md diff --git a/apps/increment/README.md b/apps/increment/README.md new file mode 100644 index 0000000..fb88de9 --- /dev/null +++ b/apps/increment/README.md @@ -0,0 +1,395 @@ +# Increment + +A VOID Framework application demonstrating on-chain increment tracking with cryptographic proofs using sparse merkle trees. Features both in-memory and persistent database storage options. + +## Overview + +Increment is a minimal example showcasing the VOID Framework's capabilities for building verifiable off-chain state machines with flexible storage backends. The application: + +- Listens to `Incremented` events from an on-chain Solidity contract +- Maintains a counter and tracks which addresses triggered each increment +- Stores address-to-count mappings in a sparse merkle tree (in-memory or SQLite) +- Generates and signs cryptographic witnesses of the current state +- Provides HTTP API for querying state and generating merkle proofs +- Supports both Publisher (full node) and Observer (light client) modes +- Offers pluggable storage: in-memory or persistent SQLite database + +Users can increment the counter on-chain, then use the off-chain proofs to claim ownership of specific increment counts back on-chain. + +## Architecture + +### Storage Options + +**In-Memory Mode** (Default) +- Fast, ephemeral storage +- Suitable for development and testing +- State is lost when the application stops + +**Database Mode** (SQLite) +- Persistent storage across restarts +- Production-ready with ACID guarantees +- Efficient queries with indexed lookups +- Stores merkle tree nodes, proofs, and metadata + +### Components + +**State Management** (`state.rs`, `data.rs`, `db.rs`) +- `State` trait: Abstract interface for both memory and database backends +- `MemoryState`: In-memory implementation using `SparseMerkleTree` +- `Db` + `Tx`: SQLite-backed implementation with transactional guarantees +- Merkle tree maps count → address (20-byte leaf values) +- `Witness`: Signed commitment to current state (count + merkle root + block height) +- State transitions process `Incremented` events from the oracle + +**Modes** (`lib.rs`) +- **Publisher Mode**: Full node that processes events, signs witnesses, and replicates state + - Runs oracle to fetch blockchain events + - Signs state commitments with private key + - Broadcasts proofs to observers via network channel + - Persists state to database (if enabled) +- **Observer Mode**: Light client that receives signed state from publishers + - Connects to publisher's oracle and network endpoints + - Validates signed witnesses + - Maintains local state replica + - Provides same query API as publisher + +**HTTP Server** (`server.rs`) +- Server-Sent Events (SSE) stream of current count +- REST endpoints for proofs and state queries +- All responses include block height for consistency +- CORS enabled for browser access + +**Proof System** (`proof.rs`) +- Manages current signed witness with block height +- Serializes proofs for network replication +- Custom binary format for efficient transmission +- Coordinates with storage layer for persistence + +## Building + +### Compile Contracts + +Before running the application, compile the Solidity contracts: + +```bash +nix run .#compile-contracts +``` + +This generates the ABI file at `contracts/build/increment_abi.json`. + +### Build the Application + +Enter the Nix development shell and build: + +```bash +nix develop +cargo build --package increment +``` + +## Running + +### Publisher Mode + +Run a full node that processes events and signs state: + +```bash +# In-memory (ephemeral) +cargo run --package increment -- publisher \ + --key PRIVATE_KEY_ENV_VAR \ + \ + \ + \ + + +# With persistent database +cargo run --package increment -- publisher \ + --key PRIVATE_KEY_ENV_VAR \ + --db-path ./increment.db \ + --oracle-db-path ./oracle.db \ + \ + \ + \ + +``` + +Example: +```bash +export SIGNER_KEY="0x..." +cargo run --package increment -- publisher \ + --key SIGNER_KEY \ + --db-path increment.db \ + --oracle-db-path oracle.db \ + oracle_config.yaml \ + 127.0.0.1:3500 \ + 127.0.0.1:5000 \ + 127.0.0.1:4000 +``` + +### Observer Mode + +Run a light client that receives state from a publisher: + +```bash +# In-memory +cargo run --package increment -- observer \ + \ + \ + + +# With persistent database +cargo run --package increment -- observer \ + --db-path ./increment.db \ + --oracle-db-path ./oracle.db \ + \ + \ + +``` + +Example: +```bash +cargo run --package increment -- observer \ + --db-path increment_observer.db \ + --oracle-db-path oracle_observer.db \ + observer_config.yaml \ + 127.0.0.1:3600 \ + http://localhost:5000 +``` + +### Oracle Configuration + +The oracle config specifies which blockchain events to monitor. + +**Publisher Config** (full oracle config): +```yaml +query: + stream_config: + - ChainContractLogs: + rpc_url: !Ws "ws://localhost:8546" + api_key_env: null + contract_addresses: + - "0x5FbDB2315678afecb367f032d93F642f64180aa3" + event_signatures: + - "0x..." # Incremented event signature + +block: + max_block_size: 1000 + block_duration: 200ms +``` + +**Observer Config** (blocks config): +```yaml +endpoint: "http://localhost:4000" +height: 0 +public_key: "0x..." # Publisher's address for signature verification +``` + +## API Endpoints + +The HTTP server exposes the following endpoints: + +### `GET /` +Health check endpoint. Returns `"OK"`. + +### `GET /current-count` +Server-Sent Events (SSE) stream that emits the current count whenever it updates. + +**Response**: SSE stream with `Height` values (includes block height) + +### `GET /get-current-count` +Get the current count as a single value with block height. + +**Response**: +```json +{ + "block_height": 42, + "data": 100 +} +``` + +Returns `null` if no blocks have been processed yet. + +### `GET /get-app-proof/{height}` +Get the signed witness (state commitment) at a specific block height. + +**Parameters**: +- `height`: Block height to query + +**Response**: +```json +{ + "signature": [/* bytes */], + "data": { + "block_height": 42, + "data": { + "count": 100, + "root": [/* 32 bytes */] + } + } +} +``` + +Returns `null` if no proof exists at that height. + +### `GET /get-merkle-proof/{count}` +Generate a merkle proof for a specific count at the latest block height. + +**Parameters**: +- `count`: The increment count to prove + +**Response**: +```json +{ + "block_height": 42, + "data": [ + [/* 32 bytes - sibling hash at depth 0 */], + [/* 32 bytes - sibling hash at depth 1 */], + ... + ] +} +``` + +The proof array length equals the tree height (64 levels). Returns `null` if no state exists yet. + +### `GET /get-counts/{address}` +Get all counts associated with an address. + +**Parameters**: +- `address`: Ethereum address (hex string with 0x prefix) + +**Response**: +```json +[1, 5, 42] +``` + +Returns an empty array if the address has no counts. + +## Example Workflow + +### 1. Increment On-Chain + +Call the `increment()` function on the Increment contract: + +```solidity +// Emits Incremented(msg.sender) +increment.increment(); +``` + +### 2. Query Off-Chain State + +Wait for the oracle to process the event, then query the current count: + +```bash +curl http://localhost:3500/get-current-count +# Returns: {"block_height": 1, "data": 1} +``` + +### 3. Get Proof + +Fetch the signed witness and merkle proof: + +```bash +# Get signed state commitment at block height 1 +curl http://localhost:3500/get-app-proof/1 > witness.json + +# Get merkle proof for count 1 +curl http://localhost:3500/get-merkle-proof/1 > proof.json +``` + +### 4. Claim On-Chain + +Use the witness and merkle proof to claim ownership on-chain: + +```solidity +// Extract from witness.json and proof.json +increment.claim_count( + count, // From witness.data.data.count + signature, // From witness.signature + root, // From witness.data.data.root + proof // From merkle proof.data array +); +``` + +The contract verifies: +1. The signature is from the trusted publisher +2. The merkle proof is valid +3. The proof shows msg.sender at the specified count +4. Stores the claim on-chain in the `counts` mapping + +## Testing + +### Unit Tests + +Run the integration test suite: + +```bash +cargo test --package increment +``` + +### E2E Tests + +The main test spins up a local Reth node and tests the full workflow: + +```bash +# Requires Reth on PATH +cargo test --package increment -- --ignored test_api +``` + +This test: +- Deploys the Increment contract +- Runs publisher and observer nodes with database storage +- Performs increments +- Verifies state synchronization +- Tests on-chain claim verification + +### Performance Tests + +Run the performance benchmark: + +```bash +cargo test --package increment -- --ignored test_performance --nocapture +``` + +This test: +- Performs 100 increments (configurable) +- Measures on-chain and off-chain processing times +- Calculates transactions per second (TPS) +- Displays detailed timing metrics +- Verifies state consistency and merkle proof generation + +## Database Schema + +When using SQLite storage (`--db-path`), the following tables are created: + +- `increment`: Stores the current count +- `tree_leaf`: Merkle tree leaf nodes (count → address mappings) +- `tree_node`: Merkle tree internal nodes (computed hashes) +- `owner`: Derived table of addresses that triggered increments +- `count`: Join table for efficient address → counts queries +- `current_proof`: Latest signed witness for each block height (limited buffer) +- `latest_header`: Tracks the latest processed block height and hash + +All database operations are transactional, ensuring consistency even if the application crashes mid-block. + +## Contract Interface + +The Solidity contract provides: + +- `increment()`: Emit an increment event +- `claim_count(count, signature, root, proof)`: Claim ownership with proof +- `counts[count]`: Mapping of claimed counts to owners +- `user_counts[address]`: Array of counts claimed by each user + +The contract uses SHA-256 for merkle tree hashing to match the Rust implementation. + +## Dependencies + +Key dependencies from `void-toolkit`: +- `merkle`: Sparse merkle tree implementation (in-memory and database-backed) +- `app`: Stream processing, block height tracking, and notifications +- `network-channel`: State replication between nodes +- `oracle`: Blockchain event sourcing with optional persistence + +Additional dependencies: +- `rusqlite`: SQLite database bindings +- `tokio`: Async runtime +- `axum`: HTTP server framework diff --git a/apps/increment/src/data.rs b/apps/increment/src/data.rs index c6fa9a8..950fab4 100644 --- a/apps/increment/src/data.rs +++ b/apps/increment/src/data.rs @@ -1,3 +1,12 @@ +//! Storage abstraction layer for the Increment application. +//! +//! This module provides a unified interface (`Data`) over two storage backends: +//! - **Memory**: Fast, ephemeral in-memory storage using `SparseMerkleTree` +//! - **Database**: Persistent SQLite storage with transactional guarantees +//! +//! The `Data` enum allows the rest of the application to be storage-agnostic, +//! with the backend chosen at startup via `DataType`. + use std::collections::HashMap; use std::convert::Infallible; @@ -11,46 +20,85 @@ use crate::{ state::{MemoryState, Witness}, }; +/// Storage backend abstraction. +/// +/// `Data` provides a unified interface over both memory and database storage, +/// allowing the application logic to be storage-agnostic. #[derive(Clone)] pub enum Data { + /// SQLite database backend (persistent) Db(Db), + /// In-memory backend (ephemeral) Memory(Memory), } +/// In-memory storage implementation. +/// +/// Contains all application state in memory with no persistence. +/// Suitable for development, testing, or temporary deployments. #[derive(Clone)] pub struct Memory { + /// The core application state (count and merkle tree) pub state: Lock, + /// Ring buffer of recent signed proofs (last 10 block heights) pub current_proofs: Lock>>>, + /// Index mapping addresses to all their increment counts pub counts_per_owner_index: Lock>>, + /// Latest processed block header (height and hash) pub latest_header: Lock, } +/// Storage backend type selector. +/// +/// Used at initialization to choose between memory and database storage. #[derive(Clone)] pub enum DataType { + /// Use SQLite database at the specified path Db(String), + /// Use in-memory storage (ephemeral) Memory, } +/// Tracks the latest processed block header in memory. #[derive(Default)] pub enum LatestHeaderMem { + /// No blocks have been processed yet #[default] Empty, + /// Last processed block header Header { + /// Block height height: u64, + /// Block hash hash: [u8; 32], }, } +/// Fixed-size ring buffer for key-value storage. +/// +/// Maintains the most recent `LENGTH` entries, automatically evicting +/// the oldest entry when full. Used to store recent proofs in memory. pub struct KeyValueBuffer { map: std::collections::BTreeMap, } +/// Mutable pair of memory state and header. +/// +/// Used during state transitions to update both the state and block header +/// atomically within the same lock access. pub struct MemoryStatePair<'a> { + /// Reference to the in-memory state pub memory_state: &'a mut MemoryState, + /// Reference to the latest header pub header: &'a mut LatestHeaderMem, } impl Data { + /// Creates a new storage backend of the specified type. + /// + /// # Arguments + /// + /// * `data_type` - The storage backend to create (Memory or Db) pub fn new(data_type: DataType) -> Self { match data_type { DataType::Db(path) => Data::Db(Db::new(&path)), @@ -63,6 +111,10 @@ impl Data { } } + /// Updates the derived owner→counts index. + /// + /// This maintains a mapping of addresses to all their increment counts + /// for efficient API queries. pub async fn update_owners_index(&self) -> anyhow::Result<()> { match self { Data::Db(db) => db.update_owners_index().await, diff --git a/apps/increment/src/db.rs b/apps/increment/src/db.rs index b93a4c4..df6dbb1 100644 --- a/apps/increment/src/db.rs +++ b/apps/increment/src/db.rs @@ -1,3 +1,15 @@ +//! SQLite database backend for persistent storage. +//! +//! This module provides a transaction-based SQLite implementation of the application state, +//! including: +//! - Increment counter storage +//! - Database-backed sparse merkle tree +//! - Signed proof history +//! - Owner→count index tables +//! - Latest block header tracking +//! +//! All operations are performed within transactions to ensure ACID guarantees. + use std::sync::Arc; use std::sync::Mutex; @@ -30,15 +42,28 @@ macro_rules! decl_const_sql_str { }; } +/// SQLite database handle with connection pooling. +/// +/// Provides async-safe access to the database through a semaphore-protected +/// connection. All database operations are performed within transactions. #[derive(Clone)] pub struct Db { + /// Semaphore limiting concurrent database access (single writer) pub permit: Arc, + /// Shared database connection pub conn: Arc>, + /// Merkle tree hash computation cache pub tree: Arc, } +/// Database transaction handle. +/// +/// Wraps a rusqlite transaction with access to the merkle tree implementation. +/// All state modifications must occur within a transaction. pub struct Tx<'conn> { + /// The active SQLite transaction pub tx: rusqlite::Transaction<'conn>, + /// Reference to merkle tree hasher pub tree: &'conn merkle::MerkleTreeHashes, } diff --git a/apps/increment/src/lib.rs b/apps/increment/src/lib.rs index 626d5db..612f863 100644 --- a/apps/increment/src/lib.rs +++ b/apps/increment/src/lib.rs @@ -1,3 +1,49 @@ +//! Increment: A VOID Framework application for tracking on-chain increments with merkle proofs. +//! +//! This application demonstrates the core capabilities of the VOID Framework with support +//! for both in-memory and persistent database storage: +//! - Processing blockchain events through an oracle +//! - Maintaining verifiable off-chain state with merkle trees +//! - Signing state commitments with block height tracking +//! - Replicating state between publisher and observer nodes +//! - Providing HTTP API for state queries and proof generation +//! - Flexible storage: in-memory for development, SQLite for production +//! +//! # Architecture +//! +//! The application can run in two modes: +//! - **Publisher**: Full node that processes events, maintains state, and signs commitments +//! - **Observer**: Light client that receives signed state from a publisher +//! +//! Both modes support pluggable storage backends via the `Data` abstraction. +//! +//! # Storage Options +//! +//! - **Memory**: Fast, ephemeral storage using `SparseMerkleTree` +//! - **Database**: Persistent SQLite storage with transactional guarantees +//! +//! # Example +//! +//! ```no_run +//! use increment::{App, Mode, ModeType, Publisher, DataType}; +//! use std::net::SocketAddr; +//! +//! #[tokio::main] +//! async fn main() -> anyhow::Result<()> { +//! // Configuration would be loaded from files in practice +//! # /* +//! let mode = Mode { +//! data_type: DataType::Db("increment.db".to_string()), +//! server_bind_address: "127.0.0.1:3500".parse().unwrap(), +//! mode_type: ModeType::Publisher(/* ... */), +//! oracle_db_path: Some("oracle.db".to_string()), +//! }; +//! increment::run_app(mode).await +//! # */ +//! # Ok(()) +//! } +//! ``` + use std::net::SocketAddr; use std::path::PathBuf; @@ -20,6 +66,7 @@ use crate::state::Witness; pub use data::DataType; +// Generate Rust bindings for the Increment Solidity contract sol!( #[allow(missing_docs)] #[sol(rpc)] @@ -35,38 +82,70 @@ pub mod state; mod data; mod db; +/// The main application state container. +/// +/// `App` holds all shared state for the Increment application, including: +/// - Storage abstraction (memory or database) +/// - Notification channel for state updates +/// - Current signed proof for replication +/// +/// The storage backend is determined at initialization time via `DataType`. #[derive(Clone)] pub struct App { + /// Storage layer (either in-memory or database-backed) pub data: Data, + /// Notification channel that signals when state is updated pub state_notification: Notification, + /// Current signed witness with block height, used for replication to observers pub current_proof: Proof, } +/// Configuration for running the application. pub struct Mode { + /// Storage backend type (Memory or Db with path) pub data_type: DataType, + /// The type of node to run (Publisher or Observer) pub mode_type: ModeType, + /// Address to bind the HTTP API server pub server_bind_address: SocketAddr, + /// Optional path to oracle database (for persisting oracle state) pub oracle_db_path: Option, } +/// The node operation mode. pub enum ModeType { + /// Full node that processes events, signs state, and replicates to observers Publisher(Box), + /// Light client that receives signed state from a publisher Observer(Observer), } +/// Configuration for running in Publisher mode (full node). pub struct Publisher { + /// Private key for signing state witnesses pub signer: PrivateKeySigner, + /// Oracle configuration for blockchain event sourcing pub oracle_config: OracleConfig, + /// Address to bind the oracle's HTTP endpoint pub oracle_bind_address: SocketAddr, + /// Address to bind the app network replication endpoint pub app_network_bind_address: SocketAddr, } +/// Configuration for running in Observer mode (light client). pub struct Observer { + /// Oracle configuration for receiving blocks from publisher pub oracle_config: OracleBlocksConfig, + /// HTTP endpoint of the publisher's app network server pub app_network_endpoint: String, } impl App { + /// Creates a new `App` with the specified storage backend. + /// + /// # Arguments + /// + /// * `data_type` - The storage backend to use (Memory or Db) pub fn new(data_type: DataType) -> Self { let data = Data::new(data_type); Self { @@ -78,13 +157,30 @@ impl App { } impl Default for App { + /// Creates an `App` with in-memory storage (for testing/development). fn default() -> Self { Self::new(DataType::Memory) } } +/// Runs the Increment application in the specified mode. +/// +/// This is the main entry point for the application. It: +/// 1. Initializes the app with the specified storage backend +/// 2. Spawns the HTTP API server +/// 3. Runs the oracle stream (publisher or observer mode) +/// +/// # Arguments +/// +/// * `mode` - Configuration specifying storage backend, server address, and node mode +/// +/// # Returns +/// +/// Never returns under normal operation. Returns an error if initialization fails. pub async fn run_app(mode: Mode) -> anyhow::Result<()> { let app = App::new(mode.data_type); + + // Spawn the HTTP server in the background tokio::spawn({ let app = app.clone(); let server_bind_address = mode.server_bind_address; @@ -92,6 +188,8 @@ pub async fn run_app(mode: Mode) -> anyhow::Result<()> { server::run(app, server_bind_address).await; } }); + + // Run the oracle stream (blocks until shutdown) match mode.mode_type { ModeType::Publisher(full) => { run_publisher( @@ -117,34 +215,73 @@ pub async fn run_app(mode: Mode) -> anyhow::Result<()> { Ok(()) } -/// Update any information that is specific to the app but not part of the state or proof. +/// Updates app-specific indices after state transitions. +/// +/// This maintains derived data structures (like owner→counts mappings) that are used +/// for efficient API queries. The implementation is delegated to the storage layer. async fn update_app(app: &App) -> anyhow::Result<()> { app.data.update_owners_index().await } +/// Applies a state transition using the appropriate storage backend. +/// +/// This function abstracts over the storage backend, applying the transition either +/// to the database within a transaction or to the in-memory state. +/// +/// # Arguments +/// +/// * `block` - The block of events to process +/// * `data` - The storage backend (database or memory) +/// +/// # Returns +/// +/// A tuple of (processed block, witness with block height) or an error async fn state_transition_with_data( block: Block, data: &Data, ) -> anyhow::Result<(Block, Height)> { match data { Data::Db(db) => { + // Database: run transition within a transaction for atomicity db.apply(move |tx| -> anyhow::Result<_> { apply_transition(block, tx, state::state_transition) }) .await } - Data::Memory(mem) => mem.state.access(|s| { - mem.latest_header.access(|h| { - let mut s = MemoryStatePair { - memory_state: s, - header: h, - }; - apply_transition(block, &mut s, state::state_transition) + Data::Memory(mem) => { + // Memory: access in-memory state with locking + mem.state.access(|s| { + mem.latest_header.access(|h| { + let mut s = MemoryStatePair { + memory_state: s, + header: h, + }; + apply_transition(block, &mut s, state::state_transition) + }) }) - }), + } } } +/// Runs the application in Publisher mode (full node). +/// +/// Publisher mode performs the following tasks: +/// 1. Spawns a network replication server to broadcast proofs to observers +/// 2. Spawns the oracle stream processing task +/// 3. Coordinates both tasks until shutdown +/// +/// # Arguments +/// +/// * `app` - The application state +/// * `signer` - Private key for signing state witnesses +/// * `oracle_config` - Configuration for blockchain event sourcing +/// * `oracle_bind_address` - Address to bind the oracle's HTTP endpoint +/// * `app_network_bind_address` - Address to bind the replication server +/// * `oracle_db_path` - Optional path to persist oracle state +/// +/// # Returns +/// +/// Never returns under normal operation. Returns an error if oracle connection fails. pub async fn run_publisher( app: App, signer: PrivateKeySigner, @@ -171,6 +308,18 @@ pub async fn run_publisher( Ok(()) } +/// Internal function that runs the publisher's oracle stream. +/// +/// This processes blockchain events, applies state transitions, signs witnesses, +/// and updates the replication proof. +/// +/// # Arguments +/// +/// * `app` - The application state +/// * `signer` - Private key for signing +/// * `oracle_config` - Oracle configuration +/// * `oracle_bind_address` - Oracle HTTP endpoint address +/// * `oracle_db_path` - Optional path for oracle database async fn run_publisher_stream( app: App, signer: PrivateKeySigner, @@ -210,6 +359,27 @@ async fn run_publisher_stream( Ok(()) } +/// Runs the application in Observer mode (light client). +/// +/// Observer mode performs the following tasks: +/// 1. Connects to a publisher's replication endpoint to receive signed proofs +/// 2. Connects to the publisher's oracle to receive event blocks +/// 3. Validates and applies state transitions locally +/// 4. Updates derived indices for API queries +/// +/// Observers do not sign state commitments themselves; they rely on the publisher's +/// signatures. This allows for lighter-weight nodes that can serve the same query API. +/// +/// # Arguments +/// +/// * `app` - The application state +/// * `oracle_config` - Configuration for connecting to publisher's oracle +/// * `app_network_endpoint` - HTTP endpoint of publisher's replication server +/// * `oracle_db_path` - Optional path to persist oracle state +/// +/// # Returns +/// +/// Never returns under normal operation. Returns an error if connection fails. pub async fn run_observer( app: App, oracle_config: OracleBlocksConfig, @@ -279,14 +449,36 @@ async fn run_observer_stream( Ok(()) } +/// Loads a full oracle configuration for Publisher mode. +/// +/// # Arguments +/// +/// * `config_path` - Path to the configuration file (YAML or JSON) +/// +/// # Returns +/// +/// The parsed `OracleConfig` or an error if the file cannot be read or parsed. pub fn load_config(config_path: &PathBuf) -> anyhow::Result { load_config_inner(config_path) } +/// Loads an oracle blocks configuration for Observer mode. +/// +/// # Arguments +/// +/// * `config_path` - Path to the configuration file (YAML or JSON) +/// +/// # Returns +/// +/// The parsed `OracleBlocksConfig` or an error if the file cannot be read or parsed. pub fn load_blocks_config(config_path: &PathBuf) -> anyhow::Result { load_config_inner(config_path) } +/// Internal helper to load and parse configuration files. +/// +/// Supports both YAML (.yaml, .yml) and JSON (.json) formats. +/// The type `T` is inferred from the caller's context. fn load_config_inner(config_path: &PathBuf) -> anyhow::Result { let config_str = std::fs::read_to_string(config_path) .map_err(|e| anyhow::anyhow!("Failed to read config file {:?}: {}", config_path, e))?; diff --git a/apps/increment/src/main.rs b/apps/increment/src/main.rs index 502f08e..8153dee 100644 --- a/apps/increment/src/main.rs +++ b/apps/increment/src/main.rs @@ -1,8 +1,43 @@ +//! Command-line interface for the Increment application. +//! +//! Provides two modes: +//! - `publisher`: Run a full node that processes events and signs state +//! - `observer`: Run a light client that receives state from a publisher +//! +//! # New Features +//! +//! - `--db-path`: Optional SQLite database for persistent storage +//! - `--oracle-db-path`: Optional SQLite database for oracle state persistence +//! - Defaults to in-memory storage if no database paths provided +//! +//! # Examples +//! +//! Run a publisher with persistent storage: +//! ```bash +//! increment publisher \ +//! --key SIGNER_KEY \ +//! --db-path increment.db \ +//! --oracle-db-path oracle.db \ +//! oracle_config.yaml \ +//! 127.0.0.1:3500 \ +//! 127.0.0.1:5000 \ +//! 127.0.0.1:4000 +//! ``` +//! +//! Run an observer with in-memory storage: +//! ```bash +//! increment observer \ +//! observer_config.yaml \ +//! 127.0.0.1:3600 \ +//! http://localhost:5000 +//! ``` + use std::{net::SocketAddr, path::PathBuf}; use clap::{Args, Parser, Subcommand}; use increment::{Mode, signing::get_signer}; +/// Increment - A VOID Framework application for tracking on-chain increments #[derive(Parser)] #[command(version, about, long_about = None)] struct Cli { diff --git a/apps/increment/src/proof.rs b/apps/increment/src/proof.rs index 459affa..70ed50f 100644 --- a/apps/increment/src/proof.rs +++ b/apps/increment/src/proof.rs @@ -1,3 +1,18 @@ +//! Proof replication and serialization for network transmission. +//! +//! This module handles: +//! - Maintaining the current signed witness with block height +//! - Serializing proofs for network transmission (custom binary format) +//! - Deserializing proofs received from publishers +//! - Coordinating with storage layer for persistence +//! - Notifying observers of proof location updates +//! +//! # Key Changes from Previous Version +//! +//! - Witnesses now include `Height` with block height +//! - Proof storage is delegated to the `Data` layer +//! - Observer synchronization uses proof location streams + use std::io::Read; use futures::StreamExt; diff --git a/apps/increment/src/server.rs b/apps/increment/src/server.rs index e8fce79..a4b0f60 100644 --- a/apps/increment/src/server.rs +++ b/apps/increment/src/server.rs @@ -1,3 +1,23 @@ +//! HTTP API server for the Increment application. +//! +//! Provides a REST API and Server-Sent Events (SSE) interface for querying application +//! state and generating merkle proofs. +//! +//! # Key Changes from Previous Version +//! +//! - All responses now include block height via `Height` wrapper +//! - `/get-app-proof/{height}` now requires a height parameter +//! - All endpoints are now async and return `Result` types +//! +//! # Endpoints +//! +//! - `GET /` - Health check +//! - `GET /current-count` - SSE stream of count updates +//! - `GET /get-current-count` - Current count with block height +//! - `GET /get-app-proof/{height}` - Signed witness at specific height +//! - `GET /get-merkle-proof/{count}` - Merkle proof for a count +//! - `GET /get-counts/{address}` - All counts for an address + use std::net::SocketAddr; use alloy::primitives::Address; diff --git a/apps/increment/src/state.rs b/apps/increment/src/state.rs index 26941e3..3fc0d2f 100644 --- a/apps/increment/src/state.rs +++ b/apps/increment/src/state.rs @@ -1,3 +1,14 @@ +//! State management for the Increment application. +//! +//! This module defines: +//! - `State` trait: Abstract interface for state transitions (works with both memory and DB) +//! - `MemoryState`: In-memory implementation using `SparseMerkleTree` +//! - `Witness`: Cryptographic commitment to current state (count + merkle root) +//! - State transition logic for processing `Incremented` events +//! +//! State transitions are deterministic and idempotent. Given the same sequence of events, +//! the state will always converge to the same final state. + use std::convert::Infallible; use alloy::{ @@ -10,11 +21,19 @@ use void_toolkit::oracle_types::Event; use crate::Increment::Incremented; +/// Type alias for in-memory sparse merkle tree with 64-bit indices. pub type InMemoryMerkle = SparseMerkleTree<{ get_hashes_height(64) }, Sha256>; +/// Abstract state interface supporting both memory and database backends. +/// +/// This trait allows state transitions to work with either `MemoryState` or +/// database transactions (`Tx`), enabling storage-agnostic application logic. pub trait State { + /// Error type for state operations type Error; + /// Increments the count by one fn increment_count(&mut self) -> Result<(), Self::Error>; + /// Adds an owner at the current count position fn add_owner(&mut self, owner: [u8; 20]) -> Result<(), Self::Error>; } /// The state of the increment application. diff --git a/apps/increment/tests/tests.rs b/apps/increment/tests/tests.rs index 5f2ef21..4ab0dda 100644 --- a/apps/increment/tests/tests.rs +++ b/apps/increment/tests/tests.rs @@ -418,3 +418,235 @@ where Err(e) => panic!("Failed to parse JSON from {endpoint}/{path}: {e}"), } } + +#[tokio::test] +#[ignore = "Must be run in a shell that has a Reth node on path. Performance test for benchmarking."] +async fn test_performance() { + // Configuration + const NUM_INCREMENTS: usize = 100; + const PROGRESS_INTERVAL: usize = 10; + + println!("\n=== Increment Performance Test ==="); + println!("Configuration:"); + println!(" - Number of increments: {}", NUM_INCREMENTS); + println!( + " - Progress updates every: {} increments", + PROGRESS_INTERVAL + ); + println!(); + + // Setup: Start Reth node + println!("[Setup] Starting local Reth node..."); + let path = tempfile::tempdir().unwrap(); + let db_path = tempfile::tempdir().unwrap(); + let reth = Reth::new() + .dev() + .block_time("2s") + .data_dir(path.path()) + .disable_discovery() + .instance(1) + .ws_port(9000) + .spawn(); + println!("[Setup] Reth node started on {}", reth.endpoint()); + + // Setup: Create signer + let mnemonic = "test test test test test test test test test test test junk"; + let signer = MnemonicBuilder::::default() + .phrase(mnemonic) + .index(0) + .unwrap() + .derivation_path("m/44'/60'/0'/0/0") + .unwrap() + .build() + .unwrap(); + println!("[Setup] Using signer address: {}", signer.address()); + + // Setup: Deploy contract + println!("[Setup] Deploying Increment contract..."); + let provider = ProviderBuilder::new() + .wallet(signer.clone()) + .connect_http(reth.endpoint().parse().unwrap()); + let increment_contract = Increment::deploy(provider, signer.address()).await.unwrap(); + println!( + "[Setup] Contract deployed at: {}", + increment_contract.address() + ); + + // Setup: Configure oracle + let query = void_toolkit::oracle_types::config::QueryConfig { + stream_config: vec![ + void_toolkit::oracle_types::config::StreamConfig::ChainContractLogs( + void_toolkit::oracle_types::config::ChainContractLogsConfig { + rpc_url: void_toolkit::oracle_types::config::ConnectionType::Ws( + reth.ws_endpoint(), + ), + api_key_env: None, + contract_addresses: vec![increment_contract.address().to_string()], + event_signatures: vec![Increment::Incremented::SIGNATURE.to_string()], + }, + ), + ], + }; + let block = void_toolkit::oracle_types::config::BlockConfig { + max_block_size: 1000, + block_duration: std::time::Duration::from_millis(200), + }; + let config = void_toolkit::oracle_types::config::Config { query, block }; + + // Setup: Start Increment app with database + println!("[Setup] Starting Increment publisher with database..."); + let increment_endpoint = "127.0.0.1:3500"; + let db_file = db_path.path().join("increment.db"); + let oracle_db_file = db_path.path().join("oracle.db"); + let data_type = increment::DataType::Db(db_file.to_str().unwrap().to_string()); + let oracle_db_path = oracle_db_file.to_str().unwrap().to_string(); + + tokio::spawn({ + let increment_endpoint = increment_endpoint.to_string(); + let signer = signer.clone(); + async move { + let mode = Mode { + data_type, + server_bind_address: increment_endpoint.parse().unwrap(), + mode_type: ModeType::Publisher( + Publisher { + oracle_config: config, + signer, + oracle_bind_address: SocketAddr::from(([127, 0, 0, 1], 4000)), + app_network_bind_address: SocketAddr::from(([127, 0, 0, 1], 5000)), + } + .into(), + ), + oracle_db_path: Some(oracle_db_path), + }; + increment::run_app(mode).await.unwrap(); + } + }); + + // Wait for server to be ready + wait_for_ok(&format!("http://{increment_endpoint}")).await; + println!("[Setup] Publisher started on http://{}", increment_endpoint); + + // Subscribe to count updates + let mut rx = subscribe(increment_endpoint.to_string()).await; + println!("[Setup] Subscribed to count updates"); + println!(); + + // Performance test: Send increments + println!("=== Performance Test Started ==="); + println!(); + + let start_time = std::time::Instant::now(); + let mut on_chain_times = Vec::new(); + let mut off_chain_times = Vec::new(); + + for i in 1..=NUM_INCREMENTS { + let increment_start = std::time::Instant::now(); + + // Send increment transaction on-chain + increment_contract + .increment() + .send() + .await + .unwrap() + .watch() + .await + .unwrap(); + + let on_chain_duration = increment_start.elapsed(); + on_chain_times.push(on_chain_duration); + + // Wait for off-chain processing + let off_chain_start = std::time::Instant::now(); + let count = rx.recv().await.unwrap(); + let off_chain_duration = off_chain_start.elapsed(); + off_chain_times.push(off_chain_duration); + + assert_eq!(count, i as u64, "Count mismatch at iteration {}", i); + + // Progress reporting + if i % PROGRESS_INTERVAL == 0 || i == NUM_INCREMENTS { + let elapsed = start_time.elapsed(); + let current_tps = i as f64 / elapsed.as_secs_f64(); + println!( + "[Progress] {}/{} increments | Elapsed: {:.2}s | Current TPS: {:.2}", + i, + NUM_INCREMENTS, + elapsed.as_secs_f64(), + current_tps + ); + } + } + + let total_duration = start_time.elapsed(); + println!(); + println!("=== Performance Test Complete ==="); + println!(); + + // Calculate statistics + let total_tps = NUM_INCREMENTS as f64 / total_duration.as_secs_f64(); + + let avg_on_chain = + on_chain_times.iter().sum::() / on_chain_times.len() as u32; + let min_on_chain = on_chain_times.iter().min().unwrap(); + let max_on_chain = on_chain_times.iter().max().unwrap(); + + let avg_off_chain = + off_chain_times.iter().sum::() / off_chain_times.len() as u32; + let min_off_chain = off_chain_times.iter().min().unwrap(); + let max_off_chain = off_chain_times.iter().max().unwrap(); + + // Detailed metrics + println!("=== Detailed Metrics ==="); + println!(); + println!("Total Performance:"); + println!(" - Total increments: {}", NUM_INCREMENTS); + println!(" - Total duration: {:.2}s", total_duration.as_secs_f64()); + println!(" - Average TPS: {:.2} transactions/second", total_tps); + println!(); + + println!("On-Chain Transaction Times:"); + println!(" - Average: {}ms", avg_on_chain.as_millis()); + println!(" - Minimum: {}ms", min_on_chain.as_millis()); + println!(" - Maximum: {}ms", max_on_chain.as_millis()); + println!(); + + println!("Off-Chain Processing Times (Oracle + State Update + DB Write):"); + println!(" - Average: {}ms", avg_off_chain.as_millis()); + println!(" - Minimum: {}ms", min_off_chain.as_millis()); + println!(" - Maximum: {}ms", max_off_chain.as_millis()); + println!(); + + // Verify final state + println!("=== State Verification ==="); + let final_count = get_current_count(increment_endpoint).await.unwrap(); + println!(" - Final count: {}", final_count.data); + println!(" - Final block height: {}", final_count.block_height); + assert_eq!(final_count.data, NUM_INCREMENTS as u64); + + let witness = get_app_proof(increment_endpoint, final_count.block_height) + .await + .unwrap(); + println!(" - Witness count: {}", witness.data.data.count); + println!( + " - Witness root: 0x{:x}", + alloy::primitives::FixedBytes::<32>::from(witness.data.data.root) + ); + assert_eq!(witness.data.data.count, NUM_INCREMENTS as u64); + + // Test merkle proof generation for a random count + let test_count = NUM_INCREMENTS as u64 / 2; + let merkle_proof = get_merkle_proof(increment_endpoint, test_count) + .await + .unwrap(); + println!(" - Merkle proof depth: {} levels", merkle_proof.data.len()); + assert_eq!(merkle_proof.data.len(), 64, "Expected 64-level tree"); + + let counts = get_counts(increment_endpoint, &signer.address()).await; + println!(" - Counts for signer: {} increments", counts.len()); + assert_eq!(counts.len(), NUM_INCREMENTS); + + println!(); + println!("=== Test Passed ==="); + println!("Note: Database-backed storage was used for this test."); +} From 399f419053b5796180d0a0d65fa10b9605fe598c Mon Sep 17 00:00:00 2001 From: Nick Furfaro Date: Mon, 27 Oct 2025 16:16:47 -0600 Subject: [PATCH 2/3] fixup --- apps/increment/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/increment/tests/tests.rs b/apps/increment/tests/tests.rs index 4ab0dda..3e240bc 100644 --- a/apps/increment/tests/tests.rs +++ b/apps/increment/tests/tests.rs @@ -217,7 +217,7 @@ async fn test_api() { witness.data.block_height, witness.signature.into(), witness.data.data.root.into(), - merkle_proof.data.into_iter().map(Into::into).collect(), + merkle_proof.data.into_iter().map(|x| x.into()).collect(), ) .send() .await From dcdea6fd255d72e674df13a5de20308ed17e965a Mon Sep 17 00:00:00 2001 From: Nick Furfaro Date: Wed, 29 Oct 2025 10:05:02 -0600 Subject: [PATCH 3/3] minor tweaks --- apps/increment/README.md | 2 +- apps/increment/src/data.rs | 7 +++++-- apps/increment/tests/tests.rs | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/increment/README.md b/apps/increment/README.md index fb88de9..79aea29 100644 --- a/apps/increment/README.md +++ b/apps/increment/README.md @@ -25,7 +25,7 @@ Users can increment the counter on-chain, then use the off-chain proofs to claim - Suitable for development and testing - State is lost when the application stops -**Database Mode** (SQLite) +**Database Mode** (SQLite) - Persistent storage across restarts - Production-ready with ACID guarantees - Efficient queries with indexed lookups diff --git a/apps/increment/src/data.rs b/apps/increment/src/data.rs index 950fab4..26ca202 100644 --- a/apps/increment/src/data.rs +++ b/apps/increment/src/data.rs @@ -1,8 +1,11 @@ //! Storage abstraction layer for the Increment application. //! //! This module provides a unified interface (`Data`) over two storage backends: -//! - **Memory**: Fast, ephemeral in-memory storage using `SparseMerkleTree` -//! - **Database**: Persistent SQLite storage with transactional guarantees +//! - **Memory**: Fast, ephemeral storage where `SparseMerkleTree` uses in-memory backend +//! - **Database**: Persistent storage where `SparseMerkleTree` uses SQLite backend with transactional guarantees +//! +//! Both modes use `SparseMerkleTree` for state management and proof generation - the difference +//! is whether the merkle tree data is stored in memory or persisted to SQLite. //! //! The `Data` enum allows the rest of the application to be storage-agnostic, //! with the backend chosen at startup via `DataType`. diff --git a/apps/increment/tests/tests.rs b/apps/increment/tests/tests.rs index 3e240bc..e44878d 100644 --- a/apps/increment/tests/tests.rs +++ b/apps/increment/tests/tests.rs @@ -441,7 +441,7 @@ async fn test_performance() { let db_path = tempfile::tempdir().unwrap(); let reth = Reth::new() .dev() - .block_time("2s") + .block_time("100ms") .data_dir(path.path()) .disable_discovery() .instance(1)