Adds chain snapshotting for benchmark runs

Introduces the ability to run benchmarks from a pre-existing chain state using snapshots. This allows restoring a datadir via a user-defined command and loading a corresponding genesis file, enabling more flexible test scenarios.

Key changes include:
-   Mechanism to define and execute snapshot creation/restoration.
-   Integration of snapshot handling into the test setup and execution flow.
-   Loading genesis configuration from a file when using snapshots.
-   Updates engine API calls from V3 to V4.
-   Refactors mempool to distinguish between regular and sequencer-specific transactions.
-   Implements funding of test accounts using deposit transactions.
-   Updates default devnet genesis configuration.

Author: meyer9

configs/snapshot.yml

@@ -0,0 +1,41 @@
+- name: Sepolia Alpha snapshot test
+  description: Sepolia Alpha snapshot test
+  snapshot:
+    # skip non-empty for testing so we don't copy every time we run this
+    # just delete the snapshot directory to force a full copy
+    command: ./scripts/setup-snapshot.sh --skip-if-nonempty
+    genesis_file: ../../sepolia-alpha/sepolia-alpha-genesis.json
+    # force_clean is true by default to ensure consistency, but we can skip it for testing
+    force_clean: false
+  variables:
+    - type: transaction_workload
+      values:
+        - transfer-only
+    - type: node_type
+      values:
+        - reth
+    - type: num_blocks
+      value: 10
+    - type: gas_limit
+      values:
+        - 15000000
+        - 30000000
+        - 60000000
+        - 90000000
+- name: Devnet snapshot test
+  description: Devnet snapshot test
+  variables:
+    - type: transaction_workload
+      values:
+        - transfer-only
+    - type: node_type
+      values:
+        - reth
+    - type: num_blocks
+      value: 10
+    - type: gas_limit
+      values:
+        - 15000000
+        - 30000000
+        - 60000000
+        - 90000000

runner/benchmark/benchmark.go

@@ -2,12 +2,12 @@ package benchmark
 
 import (
 	"fmt"
+	"math"
 	"math/big"
 	"strings"
 	"time"
 
 	"github.com/base/base-bench/runner/config"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
 	"github.com/ethereum/go-ethereum/core"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
@@ -111,22 +111,19 @@ func (p Params) ClientOptions(prevClientOptions config.ClientOptions) config.Cli
 	return prevClientOptions
 }
 
-// Genesis returns the genesis block for a given genesis time.
-func (p Params) Genesis(genesisTime time.Time) core.Genesis {
+const MAX_GAS_LIMIT = math.MaxUint64
+
+// DefaultGenesis returns the genesis block for a devnet.
+func DefaultDevnetGenesis(genesisTime time.Time) *core.Genesis {
 	zero := uint64(0)
 	fifty := uint64(50)
 
 	allocs := make(gethTypes.GenesisAlloc)
-	// private key: 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
-	allocs[common.HexToAddress("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266")] = gethTypes.Account{
-		Balance: new(big.Int).Mul(big.NewInt(1e6), big.NewInt(params.Ether)), // 100,000 ETH
-	}
-
-	return core.Genesis{
+	return &core.Genesis{
 		Nonce:      0,
 		Timestamp:  uint64(genesisTime.Unix()),
 		ExtraData:  eip1559.EncodeHoloceneExtraData(50, 1),
-		GasLimit:   p.GasLimit,
+		GasLimit:   MAX_GAS_LIMIT, // adjust gas limit in FCU call, not here
 		Difficulty: big.NewInt(1),
 		Alloc:      allocs,
 		Config: &params.ChainConfig{
@@ -150,7 +147,7 @@ func (p Params) Genesis(genesisTime time.Time) core.Genesis {
 			TerminalTotalDifficulty: big.NewInt(1),
 			ShanghaiTime:            new(uint64),
 			CancunTime:              new(uint64),
-			PragueTime:              nil,
+			PragueTime:              new(uint64),
 			VerkleTime:              nil,
 			// OP-Stack forks are disabled, since we use this for L1.
 			BedrockBlock: big.NewInt(0),
@@ -160,9 +157,8 @@ func (p Params) Genesis(genesisTime time.Time) core.Genesis {
 			FjordTime:    &zero,
 			GraniteTime:  &zero,
 			HoloceneTime: &zero,
-			// Disabled due to reth/geth mismatch
-			IsthmusTime: nil,
-			InteropTime: nil,
+			IsthmusTime:  &zero,
+			InteropTime:  nil,
 			Optimism: &params.OptimismConfig{
 				EIP1559Elasticity:        1,
 				EIP1559Denominator:       50,

runner/benchmark/matrix.go

@@ -3,6 +3,8 @@ package benchmark
 import (
 	"errors"
 	"fmt"
+	"os"
+	"os/exec"
 	"path"
 	"regexp"
 	"strings"
@@ -95,12 +97,60 @@ func (bp *Param) Check() error {
 	return nil
 }
 
+// SnapshotDefinition is the user-facing YAML configuration for specifying
+// a snapshot to be restored before running a benchmark.
+type SnapshotDefinition struct {
+	Command     string `yaml:"command"`
+	GenesisFile string `yaml:"genesis_file"`
+	ForceClean  *bool  `yaml:"force_clean"`
+}
+
+// CreateSnapshot copies the snapshot to the output directory for the given
+// node type.
+func (s SnapshotDefinition) CreateSnapshot(nodeType string, outputDir string) error {
+	// default to true if not set
+	forceClean := s.ForceClean == nil || *s.ForceClean
+	if _, err := os.Stat(outputDir); err == nil && forceClean {
+		// TODO: we could reuse it here potentially
+		if err := os.RemoveAll(outputDir); err != nil {
+			return fmt.Errorf("failed to remove existing snapshot: %w", err)
+		}
+	}
+
+	// get absolute path of outputDir
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("failed to get absolute path of outputDir: %w", err)
+	}
+
+	outputDir = path.Join(currentDir, outputDir)
+
+	var cmdBin string
+	var args []string
+	// split out default args from command
+	parts := strings.SplitN(s.Command, " ", 2)
+	if len(parts) < 2 {
+		cmdBin = parts[0]
+	} else {
+		cmdBin = parts[0]
+		args = strings.Split(parts[1], " ")
+	}
+
+	args = append(args, nodeType, outputDir)
+
+	cmd := exec.Command(cmdBin, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
 // TestDefinition is the user-facing YAML configuration for specifying a
 // matrix of benchmark runs.
 type TestDefinition struct {
-	Name        string  `yaml:"name"`
-	Description string  `yaml:"desciption"`
-	Variables   []Param `yaml:"variables"`
+	Name        string              `yaml:"name"`
+	Snapshot    *SnapshotDefinition `yaml:"snapshot"`
+	Description string              `yaml:"description"`
+	Variables   []Param             `yaml:"variables"`
 }
 
 func (bc *TestDefinition) Check() error {
@@ -120,20 +170,21 @@ func (bc *TestDefinition) Check() error {
 }
 
 // TestPlan represents a list of test runs to be executed.
-type TestPlan []TestRun
-
-func NewTestPlanFromConfig(c []TestDefinition, testFileName string) (TestPlan, error) {
-	testPlan := make(TestPlan, 0, len(c))
+type TestPlan struct {
+	Runs     []TestRun
+	Snapshot *SnapshotDefinition
+}
 
-	for _, m := range c {
-		params, err := ResolveTestRunsFromMatrix(m, testFileName)
-		if err != nil {
-			return nil, err
-		}
-		testPlan = append(testPlan, params...)
+func NewTestPlanFromConfig(c TestDefinition, testFileName string) (*TestPlan, error) {
+	testRuns, err := ResolveTestRunsFromMatrix(c, testFileName)
+	if err != nil {
+		return nil, err
 	}
 
-	return testPlan, nil
+	return &TestPlan{
+		Runs:     testRuns,
+		Snapshot: c.Snapshot,
+	}, nil
 }
 
 var alphaNumericRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)

runner/benchmark/result_metadata.go

@@ -48,20 +48,22 @@ func (runs *BenchmarkRuns) AddResult(testIdx int, runResult BenchmarkRunResult)
 	runs.Runs[testIdx].Result = &runResult
 }
 
-func BenchmarkMetadataFromTestPlan(testPlan TestPlan) BenchmarkRuns {
+func BenchmarkMetadataFromTestPlans(testPlans []TestPlan) BenchmarkRuns {
 	metadata := BenchmarkRuns{
-		Runs:      make([]BenchmarkRun, 0, len(testPlan)),
+		Runs:      make([]BenchmarkRun, 0),
 		CreatedAt: time.Now(),
 	}
 
-	for _, params := range testPlan {
-		metadata.Runs = append(metadata.Runs, BenchmarkRun{
-			SourceFile:      params.TestFile,
-			TestName:        params.Name,
-			TestDescription: params.Description,
-			TestConfig:      params.Params.ToConfig(),
-			OutputDir:       params.OutputDir,
-		})
+	for _, testPlan := range testPlans {
+		for _, params := range testPlan.Runs {
+			metadata.Runs = append(metadata.Runs, BenchmarkRun{
+				SourceFile:      params.TestFile,
+				TestName:        params.Name,
+				TestDescription: params.Description,
+				TestConfig:      params.Params.ToConfig(),
+				OutputDir:       params.OutputDir,
+			})
+		}
 	}
 
 	return metadata

runner/benchmark/snapshots.go

@@ -0,0 +1,83 @@
+package benchmark
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"path/filepath"
+)
+
+// SnapshotManager is an interface that manages snapshots for different node types
+// and roles.
+type SnapshotManager interface {
+	// EnsureSnapshot ensures that a snapshot exists for the given node type and
+	// role. If it does not exist, it will create it using the given snapshot
+	// definition. It returns the path to the snapshot.
+	EnsureSnapshot(definition SnapshotDefinition, nodeType string, role string) (string, error)
+}
+
+type snapshotStoragePath struct {
+	// nodeType is the type of node that is using this snapshot.
+	nodeType string
+
+	// role is "validator" or "sequencer". Each must have their own datadir
+	// because we need to re-execute blocks from scratch on the validator.
+	role string
+
+	// command is the command that created this snapshot.
+	command string
+}
+
+func (s *snapshotStoragePath) Equals(other *snapshotStoragePath) bool {
+	if s.nodeType != other.nodeType {
+		return false
+	}
+	if s.role != other.role {
+		return false
+	}
+	if s.command != other.command {
+		return false
+	}
+	return true
+}
+
+type benchmarkDatadirState struct {
+	// currentDataDirs is a map of node types to their datadir. Datadirs can be
+	// reused by multiple tests ro reduce the amount of copying that needs to be
+	// done.
+	currentDataDirs map[snapshotStoragePath]string
+
+	// snapshotsDir is the directory where all the snapshots are stored. Each
+	// file will have the format <nodeType>_<role>_<hash_command>.
+	snapshotsDir string
+}
+
+func NewSnapshotManager(snapshotsDir string) SnapshotManager {
+	return &benchmarkDatadirState{
+		currentDataDirs: make(map[snapshotStoragePath]string),
+		snapshotsDir:    snapshotsDir,
+	}
+}
+
+func (b *benchmarkDatadirState) EnsureSnapshot(definition SnapshotDefinition, nodeType string, role string) (string, error) {
+	snapshotDatadir := snapshotStoragePath{
+		nodeType: nodeType,
+		role:     role,
+		command:  definition.Command,
+	}
+
+	if datadir, ok := b.currentDataDirs[snapshotDatadir]; ok {
+		return datadir, nil
+	}
+
+	hashCommand := sha256.New().Sum([]byte(definition.Command))
+
+	snapshotPath := filepath.Join(b.snapshotsDir, fmt.Sprintf("%s_%s_%x", nodeType, role, hashCommand[:12]))
+
+	// Create a new datadir for this snapshot.
+	err := definition.CreateSnapshot(nodeType, snapshotPath)
+	if err != nil {
+		return "", err
+	}
+	b.currentDataDirs[snapshotDatadir] = snapshotPath
+	return snapshotPath, nil
+}

runner/clients/reth/client.go

@@ -69,6 +69,14 @@ func (r *RethClient) Run(ctx context.Context, cfg *types.RuntimeConfig) error {
 	args = append(args, "--txpool.pending-max-size", "100")
 	args = append(args, "--txpool.queued-max-size", "100")
 
+	// delete datadir/txpool-transactions-backup.rlp if it exists
+	txpoolBackupPath := fmt.Sprintf("%s/txpool-transactions-backup.rlp", r.options.DataDirPath)
+	if _, err := os.Stat(txpoolBackupPath); err == nil {
+		if err := os.Remove(txpoolBackupPath); err != nil {
+			return errors.Wrap(err, "failed to remove txpool backup")
+		}
+	}
+
 	// read jwt secret
 	jwtSecretStr, err := os.ReadFile(r.options.JWTSecretPath)
 	if err != nil {