feat(inprocess): in-process N-validator harness

app/app.go

@@ -478,6 +478,14 @@ type App struct {
 	httpServerStartSignalSent bool
 	wsServerStartSignalSent   bool
 
+	// evmHTTPServer/evmWSServer hold the EVM JSON-RPC HTTP and WebSocket listeners
+	// constructed in RegisterLocalServices so an embedding orchestrator (the
+	// in-process harness) can Stop() them at teardown. Nil when the respective
+	// listener is disabled. Production seid does not read these; its process exit
+	// reaps the listeners.
+	evmHTTPServer evmrpc.EVMServer
+	evmWSServer   evmrpc.EVMServer
+
 	txPrioritizer sdk.TxPrioritizer
 
 	benchmarkManager *benchmark.Manager
@@ -2726,6 +2734,7 @@ func (app *App) RegisterLocalServices(node client.LocalClient, txConfig client.T
 		if err != nil {
 			panic(err)
 		}
+		app.evmHTTPServer = evmHTTPServer
 		go func() {
 			<-app.httpServerStartSignal
 			if err := evmHTTPServer.Start(); err != nil {
@@ -2740,6 +2749,7 @@ func (app *App) RegisterLocalServices(node client.LocalClient, txConfig client.T
 		if err != nil {
 			panic(err)
 		}
+		app.evmWSServer = evmWSServer
 		go func() {
 			<-app.wsServerStartSignal
 			if err := evmWSServer.Start(); err != nil {

app/app_inprocess.go

@@ -0,0 +1,20 @@
+//go:build inprocess
+
+package app
+
+import "github.com/sei-protocol/sei-chain/evmrpc"
+
+// This file holds the harness-only accessors for App's EVM serve plumbing. They
+// are gated behind the `inprocess` build tag so production App's public surface
+// does not widen — only the in-process harness (which builds with that tag) sees
+// them. The backing handle fields stay in untagged app.go because the production
+// serve goroutines construct them.
+
+// EVMHTTPServer returns the EVM JSON-RPC HTTP listener constructed in
+// RegisterLocalServices, or nil if HTTP serving is disabled. An embedding
+// orchestrator calls Stop() on it at teardown.
+func (app *App) EVMHTTPServer() evmrpc.EVMServer { return app.evmHTTPServer }
+
+// EVMWebSocketServer returns the EVM JSON-RPC WebSocket listener, or nil if WS
+// serving is disabled.
+func (app *App) EVMWebSocketServer() evmrpc.EVMServer { return app.evmWSServer }

inprocess/appoptions.go

@@ -0,0 +1,54 @@
+//go:build inprocess
+
+package inprocess
+
+import (
+	"time"
+
+	"github.com/sei-protocol/sei-chain/app"
+)
+
+// appOptions is the per-node servertypes.AppOptions the harness injects into
+// app.New. It enables EVM HTTP/WS on distinct per-node ports (the EVM-enable
+// injection invariant — app.TestAppOpts disables them to dodge a fixed-port
+// clash), sets the per-run chain-id, and disables the EVM stats tracker. The
+// SeiDB flags are pinned explicitly rather than delegated to app.TestAppOpts:
+// delegating would also adopt its giga-OFF default, flipping the in-process app
+// off the production Giga execution engine (which an unset giga flag selects).
+// Unknown keys return nil (the servertypes.AppOptions "unset, use default"
+// contract).
+type appOptions struct {
+	chainID  string
+	httpPort int
+	wsPort   int
+}
+
+func (o appOptions) Get(key string) interface{} {
+	switch key {
+	case "chain-id":
+		return o.chainID
+	case "evm.http_enabled":
+		return true
+	case "evm.http_port":
+		return o.httpPort
+	case "evm.ws_enabled":
+		return true
+	case "evm.ws_port":
+		return o.wsPort
+	case "evm.rpc_stats_interval":
+		// Disable the stats tracker. A positive interval (the unset default is 10s)
+		// makes each EVM server spawn a reporter goroutine on a package-global that
+		// EVMServer.Stop never cancels and that's reassigned per node — so N nodes
+		// would orphan N-1. 0 skips it, keeping teardown deterministic.
+		return time.Duration(0)
+	case app.FlagSCEnable:
+		return true
+	case app.FlagSCSnapshotInterval:
+		return uint32(0)
+	case app.FlagSSEnable:
+		return true
+	case app.FlagSSBackend:
+		return "pebbledb"
+	}
+	return nil
+}

inprocess/doc.go

@@ -0,0 +1,93 @@
+//go:build inprocess
+
+// Package inprocess stands up N sei-chain validators in a single Go process —
+// real CometBFT consensus, each node serving its own Tendermint RPC + EVM
+// JSON-RPC (HTTP/WS), with deterministic teardown. It is the in-process
+// provisioning foundation for the SDK "local" provider (design:
+// bdchatham-designs/designs/test-harness/sdk-local-provider-lld.md).
+//
+// Use Validators = 1 or Validators >= 3; Start rejects 2 (see "Choosing the
+// validator count"). The package is gated behind the `inprocess` build tag so
+// its heavy sei-tendermint/sei-cosmos bring-up never leaks into a normal `seid`
+// build.
+//
+// # Usage
+//
+//	net, err := inprocess.Start(ctx, inprocess.Options{Validators: 4})
+//	if err != nil { ... }
+//	defer net.Close()
+//	if err := net.WaitReady(ctx); err != nil { ... }
+//	rpc := net.Node(0).TendermintRPC() // http://127.0.0.1:PORT
+//
+// # Choosing the validator count
+//
+// Pick 1 or >= 3 — never 2. The constraint is CometBFT's block-sync→consensus
+// handoff, not a voting-power quorum:
+//
+//   - N=1: the sole validator skips block-sync and proposes blocks solo
+//     (sei-tendermint onlyValidatorIsUs in node/setup.go gates
+//     `blockSync := !onlyValidatorIsUs` in node/node.go). That decision reads
+//     the genesis-derived valset before InitChain, so the harness pins the
+//     single validator into genesis for N=1 — an empty valset would leave size
+//     0, defeat onlyValidatorIsUs, and hang the solo node in block-sync (see
+//     startNode).
+//   - N=2 deadlocks: each node has exactly one peer, but BlockPool.IsCaughtUp
+//     (internal/blocksync/pool.go) requires len(peers) > 1 to ever report
+//     caught-up, so neither node leaves block-sync. It is a peer-count
+//     deadlock, not a stake threshold — Start rejects 2 loudly rather than hang.
+//   - N>=3: every node has >= 2 peers, so IsCaughtUp can fire and hand off to
+//     consensus. N=3 is the smallest real multi-node topology.
+//
+// # Bring-up invariants
+//
+// These are the load-bearing deltas vs sei-cosmos/testutil/network.New. Each is
+// named and referenced by name at its point of use in the code — there is no
+// central numbered list to drift:
+//
+//   - empty-valset: set genDoc.Validators = nil and let CometBFT derive the
+//     valset from the app's InitChain response. testutil/network sets it to
+//     []{self}, which fails consensus replay for N>1. (N=1 is the exception —
+//     it pins the validator into genesis; see "Choosing the validator count".)
+//   - gentx-derived peer mesh: the harness never wires the P2P mesh. Each
+//     validator's gentx memo carries nodeID@127.0.0.1:p2pPort, and
+//     collectGentxs → genutil.GenAppStateFromConfig (sei-cosmos x/genutil)
+//     mutates P2P.PersistentPeers in place on the same *config.Config the
+//     harness holds in node.tmCfg and later hands to tmnode.New. Without it
+//     nodes never gossip and consensus never forms for N>1. The in-place
+//     mutation is invisible at the harness layer and fragile — cloning tmCfg
+//     before collectGentxs, or building nodes before collecting, silently
+//     breaks consensus for all N — so Start asserts PersistentPeers is
+//     non-empty (N>=2) right after collectGentxs and fails loudly otherwise.
+//   - EVM-enable injection: injected AppOptions enable EVM HTTP/WS on per-node
+//     ports. Without them app.TestAppOpts hard-disables the listeners and no
+//     node serves EVM.
+//   - metrics-off: set tmCfg.Instrumentation.Prometheus = false to avoid the
+//     dup-registry panic from the process-wide registries. Metrics must stay
+//     off until the evmrpc/EVM-keeper metrics are de-globalized — re-enabling
+//     Prometheus before then reintroduces the panic.
+//   - loopback bind scope: scope TM RPC and P2P to 127.0.0.1 (they default to
+//     [::]/0.0.0.0), or the harness publishes externally reachable
+//     consensus/RPC listeners. The EVM HTTP/WS listeners are the accepted
+//     exception: they bind all interfaces (0.0.0.0) because evmrpc has no
+//     bind-host option yet, but run on free ephemeral ports dialed via
+//     127.0.0.1. A rare port-bind collision — the free port is taken between
+//     freePort's probe-close (net.Listen on 127.0.0.1:0) and the listener's bind —
+//     panics the node's serve goroutine (the production fail-loud path,
+//     intentionally not diverted). If that ever flakes, harden the freePort
+//     probe-to-bind window rather than re-add a serve-error diversion.
+//   - loopback conn-tracker ceiling: raise MaxIncomingConnectionAttempts.
+//     Loopback collapses every peer onto 127.0.0.1, so the router's IP-keyed
+//     conn-tracker counts the whole startup burst against one key; without the
+//     raise the burst trips the per-IP cap and peers are rejected.
+//
+// # Why a native API, not the SDK sei.Provider interface
+//
+// The LLD's eventual target is for Start to back the SDK's sei.Provider so
+// suites written against sei.Open(ctx, "local") run unchanged. That wiring is
+// deferred: the SDK lives in the github.com/sei-protocol/sei-k8s-controller
+// module, which declares `go >= 1.26.0`, while sei-chain runs go 1.25.6 — so
+// importing it would force a chain-wide toolchain bump and pull the controller's
+// controller-runtime/AWS dep graph into the seid build. The handle methods here
+// intentionally mirror sei.NodeHandle / sei.NetworkHandle so a thin adapter can
+// satisfy the SDK interface once the skew is resolved — see Node and Network.
+package inprocess

inprocess/genesis.go

@@ -0,0 +1,194 @@
+//go:build inprocess
+
+package inprocess
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/sei-protocol/sei-chain/sei-cosmos/client"
+	"github.com/sei-protocol/sei-chain/sei-cosmos/client/tx"
+	"github.com/sei-protocol/sei-chain/sei-cosmos/codec"
+	"github.com/sei-protocol/sei-chain/sei-cosmos/crypto/keyring"
+	cryptotypes "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/types"
+	"github.com/sei-protocol/sei-chain/sei-cosmos/testutil"
+	sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
+	authtypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/auth/types"
+	banktypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/bank/types"
+	"github.com/sei-protocol/sei-chain/sei-cosmos/x/genutil"
+	genutiltypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/genutil/types"
+	stakingtypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/staking/types"
+	tmtime "github.com/sei-protocol/sei-chain/sei-tendermint/libs/time"
+	tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types"
+)
+
+// genesisBuilder accumulates per-validator accounts, balances, and gentxs across
+// the key-generation pass, then assembles a shared genesis whose validator set
+// is left EMPTY so every node derives the consensus valset from its InitChain
+// response (the empty-valset invariant), the load-bearing delta from
+// testutil/network.
+//
+// This is a self-contained reimplementation of the unexported initGenFiles /
+// collectGenFiles / writeFile helpers in sei-cosmos/testutil/network: lifting
+// them verbatim would require exporting them from a production cosmos package.
+// They use only exported cosmos APIs, so reimplementing keeps the harness free
+// of any sei-cosmos source change.
+type genesisBuilder struct {
+	codec     codec.Codec
+	txConfig  client.TxConfig
+	chainID   string
+	bondDenom string
+
+	accounts []authtypes.GenesisAccount
+	balances []banktypes.Balance
+}
+
+// fundValidator stores a validator operator key in kb, funds its genesis account
+// + balances, and writes its self-delegation gentx to gentxsDir keyed by moniker.
+// It returns the operator address for downstream client wiring.
+func (b *genesisBuilder) fundValidator(
+	kb keyring.Keyring,
+	moniker string,
+	pubKey cryptotypes.PubKey,
+	algo keyring.SignatureAlgo,
+	accountTokens, stakingTokens, bondedTokens sdk.Int,
+	p2pHost, p2pPort, nodeID, gentxsDir string,
+) (sdk.AccAddress, error) {
+	addr, _, err := testutil.GenerateSaveCoinKey(kb, moniker, "", true, algo)
+	if err != nil {
+		return nil, fmt.Errorf("generate key for %s: %w", moniker, err)
+	}
+
+	balances := sdk.NewCoins(
+		sdk.NewCoin(fmt.Sprintf("%stoken", moniker), accountTokens),
+		sdk.NewCoin(b.bondDenom, stakingTokens),
+	)
+	b.balances = append(b.balances, banktypes.Balance{Address: addr.String(), Coins: balances.Sort()})
+	b.accounts = append(b.accounts, authtypes.NewBaseAccount(addr, nil, 0, 0))
+
+	commission, err := sdk.NewDecFromStr("0.5")
+	if err != nil {
+		return nil, err
+	}
+	createValMsg, err := stakingtypes.NewMsgCreateValidator(
+		sdk.ValAddress(addr), pubKey,
+		sdk.NewCoin(b.bondDenom, bondedTokens),
+		stakingtypes.NewDescription(moniker, "", "", "", ""),
+		stakingtypes.NewCommissionRates(commission, sdk.OneDec(), sdk.OneDec()),
+		sdk.OneInt(),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	memo := fmt.Sprintf("%s@%s:%s", nodeID, p2pHost, p2pPort)
+	txb := b.txConfig.NewTxBuilder()
+	if err := txb.SetMsgs(createValMsg); err != nil {
+		return nil, err
+	}
+	txb.SetFeeAmount(sdk.NewCoins(sdk.NewCoin(fmt.Sprintf("%stoken", moniker), sdk.NewInt(0))))
+	txb.SetGasLimit(1_000_000)
+	txb.SetMemo(memo)
+	txf := tx.Factory{}.WithChainID(b.chainID).WithMemo(memo).WithKeybase(kb).WithTxConfig(b.txConfig)
+	if err := tx.Sign(txf, moniker, txb, true); err != nil {
+		return nil, err
+	}
+	txBz, err := b.txConfig.TxJSONEncoder()(txb.GetTx())
+	if err != nil {
+		return nil, err
+	}
+	if err := writeFile(moniker+".json", gentxsDir, txBz); err != nil {
+		return nil, err
+	}
+	return addr, nil
+}
+
+// fundAccount stores a non-validator key in kb and funds its genesis account +
+// balance. Unlike fundValidator it writes no gentx (the account never stakes) —
+// it is the genesis-funded signing account a suite spends from (e.g. `admin`).
+func (b *genesisBuilder) fundAccount(
+	kb keyring.Keyring,
+	name string,
+	algo keyring.SignatureAlgo,
+	coins sdk.Coins,
+) error {
+	addr, _, err := testutil.GenerateSaveCoinKey(kb, name, "", true, algo)
+	if err != nil {
+		return fmt.Errorf("generate key for %s: %w", name, err)
+	}
+	b.accounts = append(b.accounts, authtypes.NewBaseAccount(addr, nil, 0, 0))
+	if !coins.Empty() {
+		b.balances = append(b.balances, banktypes.Balance{Address: addr.String(), Coins: coins.Sort()})
+	}
+	return nil
+}
+
+// writeBaseGenesis writes a base genesis file (accounts + balances, empty
+// validator set) to every validator's genesis path. Mirrors initGenFiles.
+func (b *genesisBuilder) writeBaseGenesis(baseState map[string]json.RawMessage, genFiles []string) error {
+	var authGenState authtypes.GenesisState
+	b.codec.MustUnmarshalJSON(baseState[authtypes.ModuleName], &authGenState)
+	packed, err := authtypes.PackAccounts(b.accounts)
+	if err != nil {
+		return err
+	}
+	authGenState.Accounts = append(authGenState.Accounts, packed...)
+	baseState[authtypes.ModuleName] = b.codec.MustMarshalJSON(&authGenState)
+
+	var bankGenState banktypes.GenesisState
+	b.codec.MustUnmarshalJSON(baseState[banktypes.ModuleName], &bankGenState)
+	bankGenState.Balances = append(bankGenState.Balances, b.balances...)
+	baseState[banktypes.ModuleName] = b.codec.MustMarshalJSON(&bankGenState)
+
+	appStateJSON, err := json.MarshalIndent(baseState, "", "  ")
+	if err != nil {
+		return err
+	}
+	genDoc := tmtypes.GenesisDoc{
+		ChainID:    b.chainID,
+		AppState:   appStateJSON,
+		Validators: nil, // empty-valset invariant: derive valset from InitChain.
+	}
+	for _, gf := range genFiles {
+		if err := genDoc.SaveAs(gf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// collectGentxs folds every validator's gentx into each node's genesis app state
+// under one canonical genesis time (consensus timestamp validation diverges if
+// the nodes disagree on GenesisTime). Mirrors collectGenFiles.
+func (b *genesisBuilder) collectGentxs(nodes []*node, gentxsDir string) error {
+	genTime := tmtime.Now()
+	for _, n := range nodes {
+		initCfg := genutiltypes.NewInitConfig(b.chainID, gentxsDir, n.nodeID, n.pubKey)
+		genFile := n.tmCfg.GenesisFile()
+		genDoc, err := tmtypes.GenesisDocFromFile(genFile)
+		if err != nil {
+			return err
+		}
+		appState, err := genutil.GenAppStateFromConfig(
+			b.codec, b.txConfig, n.tmCfg, initCfg, *genDoc, banktypes.GenesisBalancesIterator{},
+		)
+		if err != nil {
+			return err
+		}
+		if err := genutil.ExportGenesisFileWithTime(genFile, b.chainID, nil, appState, genTime); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// writeFile writes contents under dir/name, creating dir. Mirrors the network
+// package's unexported writeFile.
+func writeFile(name, dir string, contents []byte) error {
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(dir, name), contents, 0o600)
+}