-
Notifications
You must be signed in to change notification settings - Fork 879
Restructure sei data folder for Giga #3155
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
7c6ac8f
88daf32
0df74b4
23d7a91
c20f733
b4d61bb
022fd22
3ebd0ae
343d67d
240c627
c7e0ed3
8ebf801
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,69 @@ | ||
| package utils | ||
|
|
||
| import "path/filepath" | ||
| import ( | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| // PathExists returns true if the given path exists on disk. | ||
| func PathExists(path string) bool { | ||
| _, err := os.Stat(path) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Harden this by requiring the type of path? e.g. directory? file etc. This is racy in that it doesn't atomically check and it repeatedly checks but hopefully the probability of race is low enough that we can accept the race condition as negligible.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make sense, will do |
||
| return err == nil | ||
| } | ||
|
|
||
| // GetCommitStorePath returns the path for the memiavl state commitment store. | ||
| // New nodes use data/state_commit/memiavl; existing nodes with data/committer.db | ||
| // continue using the legacy path for backward compatibility. | ||
| func GetCommitStorePath(homePath string) string { | ||
| return filepath.Join(homePath, "data", "committer.db") | ||
| legacyPath := filepath.Join(homePath, "data", "committer.db") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_commit", "memiavl") | ||
| } | ||
|
|
||
| // GetFlatKVPath returns the path for the FlatKV EVM commit store. | ||
| // New nodes use data/state_commit/flatkv; existing nodes with data/flatkv | ||
| // continue using the legacy path for backward compatibility. | ||
| func GetFlatKVPath(homePath string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "flatkv") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_commit", "flatkv") | ||
| } | ||
|
|
||
| // GetStateStorePath returns the path for the Cosmos state store (SS). | ||
| // New nodes use data/state_store/cosmos_ss/{backend}; existing nodes with | ||
| // data/{backend} continue using the legacy path for backward compatibility. | ||
| func GetStateStorePath(homePath string, backend string) string { | ||
| return filepath.Join(homePath, "data", backend) | ||
| legacyPath := filepath.Join(homePath, "data", backend) | ||
| if PathExists(legacyPath) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor: Is there ever a case where an abrupt node failure could cause the creation of an empty directory here? If so, for robustness i recommend checking if this dir is empty and proceed with legacy path iff it is not empty.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If the creation of empty dir happens, then it will continue proceed with the old path, which is fine. I don't think there's any case where it is started with the new path first and then suddenly crash and create an old path |
||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_store", "cosmos_ss", backend) | ||
| } | ||
|
|
||
| // GetEVMStateStorePath returns the path for the EVM state store. | ||
| // New nodes use data/state_store/evm_ss; existing nodes with data/evm_ss | ||
| // continue using the legacy path for backward compatibility. | ||
| func GetEVMStateStorePath(homePath string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "evm_ss") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_store", "evm_ss") | ||
| } | ||
|
|
||
| // GetReceiptStorePath returns the path for the receipt store. | ||
| // New nodes use data/ledger/receipt.db; existing nodes with data/receipt.db | ||
| // continue using the legacy path for backward compatibility. | ||
| func GetReceiptStorePath(homePath string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "receipt.db") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "ledger", "receipt.db") | ||
| } | ||
|
|
||
| func GetChangelogPath(dbPath string) string { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| package utils | ||
|
|
||
| import ( | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestPathExists(t *testing.T) { | ||
| dir := t.TempDir() | ||
| assert.True(t, PathExists(dir)) | ||
| assert.False(t, PathExists(filepath.Join(dir, "nonexistent"))) | ||
|
|
||
| f := filepath.Join(dir, "file.txt") | ||
| require.NoError(t, os.WriteFile(f, []byte("hi"), 0644)) | ||
| assert.True(t, PathExists(f)) | ||
| } | ||
|
|
||
| // --- GetCommitStorePath --- | ||
|
|
||
| func TestGetCommitStorePath_NewNode(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetCommitStorePath(home) | ||
| assert.Equal(t, filepath.Join(home, "data", "state_commit", "memiavl"), got) | ||
| } | ||
|
|
||
| func TestGetCommitStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "committer.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetCommitStorePath(home) | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetFlatKVPath --- | ||
|
|
||
| func TestGetFlatKVPath_NewNode(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetFlatKVPath(home) | ||
| assert.Equal(t, filepath.Join(home, "data", "state_commit", "flatkv"), got) | ||
| } | ||
|
|
||
| func TestGetFlatKVPath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "flatkv") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetFlatKVPath(home) | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetStateStorePath --- | ||
|
|
||
| func TestGetStateStorePath_NewNode_Pebble(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "cosmos_ss", "pebbledb"), got) | ||
| } | ||
|
|
||
| func TestGetStateStorePath_NewNode_RocksDB(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetStateStorePath(home, "rocksdb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "cosmos_ss", "rocksdb"), got) | ||
| } | ||
|
|
||
| func TestGetStateStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "pebbledb") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| func TestGetStateStorePath_LegacyForDifferentBackend(t *testing.T) { | ||
| home := t.TempDir() | ||
| // Legacy rocksdb dir exists but we ask for pebbledb — no legacy match | ||
| require.NoError(t, os.MkdirAll(filepath.Join(home, "data", "rocksdb"), 0755)) | ||
|
|
||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "cosmos_ss", "pebbledb"), got) | ||
| } | ||
|
|
||
| // --- GetEVMStateStorePath --- | ||
|
|
||
| func TestGetEVMStateStorePath_NewNode(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetEVMStateStorePath(home) | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "evm_ss"), got) | ||
| } | ||
|
|
||
| func TestGetEVMStateStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "evm_ss") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetEVMStateStorePath(home) | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetReceiptStorePath --- | ||
|
|
||
| func TestGetReceiptStorePath_NewNode(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetReceiptStorePath(home) | ||
| assert.Equal(t, filepath.Join(home, "data", "ledger", "receipt.db"), got) | ||
| } | ||
|
|
||
| func TestGetReceiptStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "receipt.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetReceiptStorePath(home) | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetChangelogPath (unchanged, but verify) --- | ||
|
|
||
| func TestGetChangelogPath(t *testing.T) { | ||
| assert.Equal(t, "/foo/bar/changelog", GetChangelogPath("/foo/bar")) | ||
| } | ||
|
|
||
| // --- Edge: new path already has data (second run of new node) --- | ||
|
|
||
| func TestGetCommitStorePath_NewDataAlreadyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| newPath := filepath.Join(home, "data", "state_commit", "memiavl") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetCommitStorePath(home) | ||
| assert.Equal(t, newPath, got, "should use new path when legacy is absent even if new path already exists") | ||
| } | ||
|
|
||
| func TestGetStateStorePath_NewDataAlreadyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| newPath := filepath.Join(home, "data", "state_store", "cosmos_ss", "pebbledb") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, newPath, got, "should use new path when legacy is absent even if new path already exists") | ||
| } | ||
|
|
||
| // --- Edge: both legacy and new exist (legacy wins) --- | ||
|
|
||
| func TestGetCommitStorePath_BothExist(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "committer.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
| newPath := filepath.Join(home, "data", "state_commit", "memiavl") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetCommitStorePath(home) | ||
| assert.Equal(t, legacy, got, "legacy should take precedence when both exist") | ||
| } | ||
|
|
||
| func TestGetReceiptStorePath_BothExist(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "receipt.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
| newPath := filepath.Join(home, "data", "ledger", "receipt.db") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetReceiptStorePath(home) | ||
| assert.Equal(t, legacy, got, "legacy should take precedence when both exist") | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -149,23 +149,23 @@ func loadEventSinks(cfg *tmcfg.Config) ([]indexer.EventSink, error) { | |
| func loadStateAndBlockStore(cfg *tmcfg.Config) (*store.BlockStore, state.Store, error) { | ||
| dbType := dbm.BackendType(cfg.DBBackend) | ||
|
|
||
| if !os.FileExists(filepath.Join(cfg.DBDir(), "blockstore.db")) { | ||
| blockstoreDir := tmcfg.ResolveDBDir("blockstore", cfg.DBDir()) | ||
| if !os.FileExists(filepath.Join(blockstoreDir, "blockstore.db")) { | ||
| return nil, nil, fmt.Errorf("no blockstore found in %v", cfg.DBDir()) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the error msg seems to should be against blockstoreDir instead of cfg.DBDir()
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good catch! |
||
| } | ||
|
|
||
| // Get BlockStore | ||
| blockStoreDB, err := dbm.NewDB("blockstore", dbType, cfg.DBDir()) | ||
| blockStoreDB, err := dbm.NewDB("blockstore", dbType, blockstoreDir) | ||
| if err != nil { | ||
| return nil, nil, err | ||
| } | ||
| blockStore := store.NewBlockStore(blockStoreDB) | ||
|
|
||
| if !os.FileExists(filepath.Join(cfg.DBDir(), "state.db")) { | ||
| return nil, nil, fmt.Errorf("no blockstore found in %v", cfg.DBDir()) | ||
| stateDir := tmcfg.ResolveDBDir("state", cfg.DBDir()) | ||
| if !os.FileExists(filepath.Join(stateDir, "state.db")) { | ||
| return nil, nil, fmt.Errorf("no state store found in %v", cfg.DBDir()) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same as above, error msg against stateDir instead of cfg.DBDir()
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good catch! |
||
| } | ||
|
|
||
| // Get StateStore | ||
| stateDB, err := dbm.NewDB("state", dbType, cfg.DBDir()) | ||
| stateDB, err := dbm.NewDB("state", dbType, stateDir) | ||
| if err != nil { | ||
| return nil, nil, err | ||
| } | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
readReceiptStoreConfig() was changed to default to data/ledger/receipt.db, but the emitted config template still says /data/receipt.db
need to update the related toml.go
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fixed