Skip to content

Commit 814fea5

Browse files
author
kant
committed
feat: indexer indexes blocks and txs of mev-commit chain to any pluggable storage
1 parent bf20b62 commit 814fea5

14 files changed

Lines changed: 1748 additions & 0 deletions

File tree

‎indexer/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# indexer

‎indexer/cmd/indexer/main.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"syscall"
9+
"time"
10+
11+
"github.com/primev/mev-commit/indexer/internal/indexer"
12+
"github.com/primev/mev-commit/indexer/pkg/ethclient"
13+
"github.com/primev/mev-commit/indexer/pkg/logutil"
14+
"github.com/primev/mev-commit/indexer/pkg/store"
15+
"github.com/primev/mev-commit/indexer/pkg/store/elasticsearch"
16+
17+
"log/slog"
18+
19+
"github.com/urfave/cli/v2"
20+
)
21+
22+
func main() {
23+
app := &cli.App{
24+
Name: "blockchain-indexer",
25+
Usage: "Index blockchain data into Elasticsearch",
26+
Flags: []cli.Flag{
27+
&cli.StringFlag{
28+
Name: "ethereum-endpoint",
29+
EnvVars: []string{"ETHEREUM_ENDPOINT"},
30+
Value: "http://localhost:8545",
31+
Usage: "Ethereum node endpoint",
32+
},
33+
&cli.StringFlag{
34+
Name: "elasticsearch-endpoint",
35+
EnvVars: []string{"ELASTICSEARCH_ENDPOINT"},
36+
Value: "http://localhost:9200",
37+
Usage: "Elasticsearch endpoint",
38+
},
39+
&cli.StringFlag{
40+
Name: "es-username",
41+
EnvVars: []string{"ES_USERNAME"},
42+
Value: "",
43+
Usage: "Elasticsearch username",
44+
},
45+
&cli.StringFlag{
46+
Name: "es-password",
47+
EnvVars: []string{"ES_PASSWORD"},
48+
Value: "",
49+
Usage: "Elasticsearch password",
50+
},
51+
&cli.DurationFlag{
52+
Name: "index-interval",
53+
EnvVars: []string{"INDEX_INTERVAL"},
54+
Value: 15 * time.Second,
55+
Usage: "Interval between indexing operations",
56+
},
57+
&cli.StringFlag{
58+
Name: "log-level",
59+
EnvVars: []string{"LOG_LEVEL"},
60+
Value: "info",
61+
Usage: "Log level (debug, info, warn, error)",
62+
},
63+
},
64+
Action: run,
65+
}
66+
67+
if err := app.Run(os.Args); err != nil {
68+
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
69+
os.Exit(1)
70+
}
71+
}
72+
73+
func run(cliCtx *cli.Context) error {
74+
ctx, cancel := context.WithCancel(context.Background())
75+
defer cancel()
76+
77+
ethClient, err := ethclient.NewW3EthereumClient(cliCtx.String("ethereum-endpoint"))
78+
if err != nil {
79+
slog.Error("Failed to create Ethereum client", "error", err)
80+
return err
81+
}
82+
83+
esClient, err := elasticsearch.NewESClient(cliCtx.String("elasticsearch-endpoint"), cliCtx.String("es-username"), cliCtx.String("es-password"))
84+
if err != nil {
85+
slog.Error("Failed to create Elasticsearch client", "error", err)
86+
return err
87+
}
88+
defer func() {
89+
if err := esClient.Close(ctx); err != nil {
90+
slog.Error("Failed to close Elasticsearch client", "error", err)
91+
}
92+
}()
93+
94+
var esStorage store.Storage = esClient
95+
96+
blockchainIndexer := indexer.NewBlockchainIndexer(
97+
ethClient,
98+
esStorage,
99+
cliCtx.Duration("index-interval"),
100+
)
101+
102+
// Set log level
103+
logutil.SetLogLevel(cliCtx.String("log-level"))
104+
105+
if err := blockchainIndexer.Start(ctx); err != nil {
106+
slog.Error("Failed to start blockchain indexer", "error", err)
107+
return err
108+
}
109+
110+
// Set up graceful shutdown
111+
c := make(chan os.Signal, 1)
112+
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
113+
114+
// Wait for interrupt signal
115+
<-c
116+
slog.Info("Shutting down gracefully...")
117+
cancel()
118+
// Wait for some time to allow ongoing operations to complete
119+
time.Sleep(5 * time.Second)
120+
return nil
121+
}

‎indexer/cmd/main.go‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"syscall"
9+
"time"
10+
11+
"github.com/primev/mev-commit/indexer/pkg/indexer"
12+
"github.com/urfave/cli/v2"
13+
"log/slog"
14+
)
15+
16+
func main() {
17+
app := &cli.App{
18+
Name: "blockchain-indexer",
19+
Usage: "Index blockchain data into Elasticsearch",
20+
Flags: []cli.Flag{
21+
&cli.StringFlag{
22+
Name: "ethereum-endpoint",
23+
EnvVars: []string{"ETHEREUM_ENDPOINT"},
24+
Value: "http://localhost:8545",
25+
Usage: "Ethereum node endpoint",
26+
},
27+
&cli.StringFlag{
28+
Name: "elasticsearch-endpoint",
29+
EnvVars: []string{"ELASTICSEARCH_ENDPOINT"},
30+
Value: "http://localhost:9200",
31+
Usage: "Elasticsearch endpoint",
32+
},
33+
&cli.DurationFlag{
34+
Name: "index-interval",
35+
EnvVars: []string{"INDEX_INTERVAL"},
36+
Value: 15 * time.Second,
37+
Usage: "Interval between indexing operations",
38+
},
39+
&cli.StringFlag{
40+
Name: "log-level",
41+
EnvVars: []string{"LOG_LEVEL"},
42+
Value: "info",
43+
Usage: "Log level (debug, info, warn, error)",
44+
},
45+
},
46+
Action: run,
47+
}
48+
49+
if err := app.Run(os.Args); err != nil {
50+
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
51+
os.Exit(1)
52+
}
53+
}
54+
55+
func run(cliCtx *cli.Context) error {
56+
ctx, cancel := context.WithCancel(context.Background())
57+
defer cancel()
58+
59+
ethClient, err := indexer.NewW3EthereumClient(cliCtx.String("ethereum-endpoint"))
60+
if err != nil {
61+
slog.Error("Failed to create Ethereum client", "error", err)
62+
return err
63+
}
64+
65+
esClient, err := indexer.NewESClient(cliCtx.String("elasticsearch-endpoint"))
66+
if err != nil {
67+
slog.Error("Failed to create Elasticsearch client", "error", err)
68+
return err
69+
}
70+
defer func() {
71+
if err := esClient.Close(ctx); err != nil {
72+
slog.Error("Failed to close Elasticsearch client", "error", err)
73+
}
74+
}()
75+
76+
blockchainIndexer := indexer.NewBlockchainIndexer(
77+
ethClient,
78+
esClient,
79+
cliCtx.Duration("index-interval"),
80+
)
81+
82+
// Set log level
83+
indexer.SetLogLevel(cliCtx.String("log-level"))
84+
85+
if err := blockchainIndexer.Start(ctx); err != nil {
86+
slog.Error("Failed to start blockchain indexer", "error", err)
87+
return err
88+
}
89+
90+
// Set up graceful shutdown
91+
c := make(chan os.Signal, 1)
92+
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
93+
94+
// Wait for interrupt signal
95+
<-c
96+
slog.Info("Shutting down gracefully...")
97+
cancel()
98+
// Wait for some time to allow ongoing operations to complete
99+
time.Sleep(5 * time.Second)
100+
return nil
101+
}

‎indexer/find_missing_block_num.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
from elasticsearch import Elasticsearch, helpers
2+
3+
# Initialize Elasticsearch client with authentication
4+
es = Elasticsearch(
5+
["http://localhost:9200"], # Replace with your Elasticsearch host if different
6+
basic_auth=("elastic", "mev-commit")
7+
)
8+
9+
# Function to get all numbers using scroll API
10+
def get_all_numbers():
11+
numbers = []
12+
scroll_size = 10000
13+
14+
# Initial search request
15+
response = es.search(
16+
index="blocks",
17+
body={
18+
"size": scroll_size,
19+
"_source": ["number"],
20+
"sort": [{"number": "asc"}]
21+
},
22+
scroll='2m'
23+
)
24+
25+
# Get the scroll ID
26+
scroll_id = response['_scroll_id']
27+
28+
# Get the first batch of numbers
29+
numbers.extend([hit['_source']['number'] for hit in response['hits']['hits']])
30+
31+
# Continue scrolling until no more hits
32+
while len(response['hits']['hits']):
33+
response = es.scroll(scroll_id=scroll_id, scroll='2m')
34+
numbers.extend([hit['_source']['number'] for hit in response['hits']['hits']])
35+
36+
return numbers
37+
38+
# Get all numbers
39+
all_numbers = get_all_numbers()
40+
41+
# Find missing numbers
42+
missing_numbers = []
43+
for i in range(len(all_numbers) - 1):
44+
current_number = all_numbers[i]
45+
next_number = all_numbers[i + 1]
46+
if next_number != current_number + 1:
47+
missing_numbers.extend(range(current_number + 1, next_number))
48+
49+
print("Missing numbers:", missing_numbers)

‎indexer/go.mod‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
module github.com/primev/mev-commit/indexer
2+
3+
go 1.22
4+
5+
require (
6+
github.com/elastic/go-elasticsearch/v8 v8.14.0
7+
github.com/ethereum/go-ethereum v1.14.6
8+
github.com/lmittmann/tint v1.0.5
9+
github.com/lmittmann/w3 v0.16.7
10+
github.com/urfave/cli/v2 v2.27.1
11+
)
12+
13+
require (
14+
github.com/DataDog/zstd v1.5.2 // indirect
15+
github.com/Microsoft/go-winio v0.6.2 // indirect
16+
github.com/StackExchange/wmi v1.2.1 // indirect
17+
github.com/bits-and-blooms/bitset v1.10.0 // indirect
18+
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
19+
github.com/consensys/bavard v0.1.13 // indirect
20+
github.com/consensys/gnark-crypto v0.12.1 // indirect
21+
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
22+
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
23+
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
24+
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
25+
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
26+
github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
27+
github.com/go-logr/logr v1.4.1 // indirect
28+
github.com/go-logr/stdr v1.2.2 // indirect
29+
github.com/go-ole/go-ole v1.3.0 // indirect
30+
github.com/gorilla/websocket v1.5.3 // indirect
31+
github.com/holiman/uint256 v1.2.4 // indirect
32+
github.com/klauspost/compress v1.17.8 // indirect
33+
github.com/mattn/go-runewidth v0.0.14 // indirect
34+
github.com/mmcloughlin/addchain v0.4.0 // indirect
35+
github.com/prometheus/client_golang v1.19.1 // indirect
36+
github.com/prometheus/client_model v0.6.1 // indirect
37+
github.com/rivo/uniseg v0.4.2 // indirect
38+
github.com/rogpeppe/go-internal v1.10.0 // indirect
39+
github.com/russross/blackfriday/v2 v2.1.0 // indirect
40+
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
41+
github.com/supranational/blst v0.3.11 // indirect
42+
github.com/tklauser/go-sysconf v0.3.13 // indirect
43+
github.com/tklauser/numcpus v0.7.0 // indirect
44+
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
45+
go.opentelemetry.io/otel v1.24.0 // indirect
46+
go.opentelemetry.io/otel/metric v1.24.0 // indirect
47+
go.opentelemetry.io/otel/sdk v1.22.0 // indirect
48+
go.opentelemetry.io/otel/trace v1.24.0 // indirect
49+
golang.org/x/crypto v0.23.0 // indirect
50+
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
51+
golang.org/x/sync v0.7.0 // indirect
52+
golang.org/x/sys v0.20.0 // indirect
53+
golang.org/x/time v0.5.0 // indirect
54+
google.golang.org/protobuf v1.34.1 // indirect
55+
rsc.io/tmplfunc v0.0.3 // indirect
56+
)

0 commit comments

Comments
 (0)