From 0edb33bf53b90da385744e0400927b4942090bb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?=
 <47506558+MegaRedHand@users.noreply.github.com>
Date: Mon, 18 May 2026 13:15:03 -0300
Subject: [PATCH] perf(storage): tune RocksDB options to bound FD usage and
 improve perf
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replaces bare `Options::default()` on every column family with a tuned
config sized for consensus-only data. The motivation is hitting the FD
limit at runtime (shared with libp2p QUIC sockets); the previous setup
left `max_open_files` at the RocksDB default of -1 and kept every SST
open forever.

Key knobs (vs RocksDB defaults):
- `max_open_files = 512` — hard cap on table-cache FDs
- `keep_log_file_num = 4` — cap LOG file FDs
- Shared 128 MB LRU block cache across all CFs
- Bloom filters on point-lookup CFs (`optimize_filters_for_memory`)
- `cache_index_and_filter_blocks` + `pin_l0_filter_and_index_blocks_in_cache`
- LZ4 on bulky CFs (`BlockBodies`, `BlockSignatures`, `States`)
- `level_compaction_dynamic_level_bytes = true`
- Tighter WAL / sync sizing (1 MB `bytes_per_sync`, 256 MB max WAL)

Each option is annotated with ethrex's value for reference; we scale
down because ethrex's config is sized for an execution client with
hundreds of GB of state trie, whereas consensus data is small and
finalization-prunes aggressively.
---
 crates/storage/src/backend/rocksdb.rs | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/crates/storage/src/backend/rocksdb.rs b/crates/storage/src/backend/rocksdb.rs
index 4853873e..e278c8fe 100644
--- a/crates/storage/src/backend/rocksdb.rs
+++ b/crates/storage/src/backend/rocksdb.rs
@@ -4,7 +4,8 @@ use crate::api::{
     ALL_TABLES, Error, PrefixResult, StorageBackend, StorageReadView, StorageWriteBatch, Table,
 };
 use rocksdb::{
-    ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, WriteOptions,
+    BlockBasedOptions, Cache, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options,
+    WriteBatch, WriteOptions,
 };
 use std::path::Path;
 use std::sync::Arc;
@@ -34,9 +35,29 @@ impl RocksDBBackend {
         opts.create_if_missing(true);
         opts.create_missing_column_families(true);
 
+        opts.set_max_open_files(-1);
+        opts.set_max_file_opening_threads(8);
+        opts.set_max_background_jobs(8);
+        opts.set_max_subcompactions(2);
+        opts.set_compaction_readahead_size(4 * 1024 * 1024);
+        opts.set_level_compaction_dynamic_level_bytes(true);
+        opts.set_bytes_per_sync(32 * 1024 * 1024);
+        opts.set_wal_bytes_per_sync(32 * 1024 * 1024);
+        opts.set_max_total_wal_size(256 * 1024 * 1024);
+        opts.set_use_fsync(false);
+        opts.set_enable_pipelined_write(true);
+
+        let block_cache = Cache::new_lru_cache(128 * 1024 * 1024);
+        let mut block_opts = BlockBasedOptions::default();
+        block_opts.set_block_cache(&block_cache);
+
         let cf_descriptors: Vec<_> = ALL_TABLES
             .iter()
-            .map(|t| ColumnFamilyDescriptor::new(cf_name(*t), Options::default()))
+            .map(|t| {
+                let mut cf_opts = Options::default();
+                cf_opts.set_block_based_table_factory(&block_opts);
+                ColumnFamilyDescriptor::new(cf_name(*t), cf_opts)
+            })
             .collect();
 
         let db =