-
Notifications
You must be signed in to change notification settings - Fork 879
Restructure sei data folder for Giga #3155
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
7c6ac8f
88daf32
0df74b4
23d7a91
c20f733
b4d61bb
022fd22
3ebd0ae
343d67d
240c627
c7e0ed3
8ebf801
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,69 @@ | ||
| package utils | ||
|
|
||
| import "path/filepath" | ||
| import ( | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| func GetCommitStorePath(homePath string) string { | ||
| return filepath.Join(homePath, "data", "committer.db") | ||
| // PathExists returns true if the given path exists on disk. | ||
| func PathExists(path string) bool { | ||
| _, err := os.Stat(path) | ||
| return err == nil | ||
| } | ||
|
|
||
| // GetCosmosSCStorePath returns the path for the memiavl state commitment store. | ||
| // New nodes use data/state_commit/memiavl; existing nodes with data/committer.db | ||
| // continue using the legacy path for backward compatibility. | ||
| func GetCosmosSCStorePath(homePath string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "committer.db") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_commit", "memiavl") | ||
| } | ||
|
|
||
| // GetFlatKVPath returns the path for the FlatKV EVM commit store. | ||
| // New nodes use data/state_commit/flatkv; existing nodes with data/flatkv | ||
| // continue using the legacy path for backward compatibility. | ||
| func GetFlatKVPath(homePath string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "flatkv") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_commit", "flatkv") | ||
| } | ||
|
|
||
| // GetStateStorePath returns the path for the Cosmos state store (SS). | ||
| // New nodes use data/state_store/cosmos/{backend}; existing nodes with | ||
| // data/{backend} continue using the legacy path for backward compatibility. | ||
| func GetStateStorePath(homePath string, backend string) string { | ||
| return filepath.Join(homePath, "data", backend) | ||
| legacyPath := filepath.Join(homePath, "data", backend) | ||
| if PathExists(legacyPath) { | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Minor: Is there ever a case where an abrupt node failure could cause the creation of an empty directory here? If so, for robustness i recommend checking if this dir is empty and proceed with legacy path iff it is not empty.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If the creation of empty dir happens, then it will continue proceed with the old path, which is fine. I don't think there's any case where it is started with the new path first and then suddenly crash and create an old path |
||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_store", "cosmos", backend) | ||
| } | ||
|
|
||
| // GetEVMStateStorePath returns the path for the EVM state store. | ||
| // New nodes use data/state_store/evm/{backend}; existing nodes with | ||
| // data/evm_ss continue using the legacy path for backward compatibility. | ||
| func GetEVMStateStorePath(homePath string, backend string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "evm_ss") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "state_store", "evm", backend) | ||
| } | ||
|
|
||
| // GetReceiptStorePath returns the path for the receipt store. | ||
| // New nodes use data/ledger/receipt/{backend}; existing nodes with | ||
| // data/receipt.db continue using the legacy path for backward compatibility. | ||
| func GetReceiptStorePath(homePath string, backend string) string { | ||
| legacyPath := filepath.Join(homePath, "data", "receipt.db") | ||
| if PathExists(legacyPath) { | ||
| return legacyPath | ||
| } | ||
| return filepath.Join(homePath, "data", "ledger", "receipt", backend) | ||
| } | ||
|
|
||
| func GetChangelogPath(dbPath string) string { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| package utils | ||
|
|
||
| import ( | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestPathExists(t *testing.T) { | ||
| dir := t.TempDir() | ||
| assert.True(t, PathExists(dir)) | ||
| assert.False(t, PathExists(filepath.Join(dir, "nonexistent"))) | ||
|
|
||
| f := filepath.Join(dir, "file.txt") | ||
| require.NoError(t, os.WriteFile(f, []byte("hi"), 0644)) | ||
| assert.True(t, PathExists(f)) | ||
| } | ||
|
|
||
| // --- GetCosmosSCStorePath --- | ||
|
|
||
| func TestGetCosmosSCStorePath_NewNode(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetCosmosSCStorePath(home) | ||
| assert.Equal(t, filepath.Join(home, "data", "state_commit", "memiavl"), got) | ||
| } | ||
|
|
||
| func TestGetCosmosSCStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "committer.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetCosmosSCStorePath(home) | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetFlatKVPath --- | ||
|
|
||
| func TestGetFlatKVPath_NewNode(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetFlatKVPath(home) | ||
| assert.Equal(t, filepath.Join(home, "data", "state_commit", "flatkv"), got) | ||
| } | ||
|
|
||
| func TestGetFlatKVPath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "flatkv") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetFlatKVPath(home) | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetStateStorePath --- | ||
|
|
||
| func TestGetStateStorePath_NewNode_Pebble(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "cosmos", "pebbledb"), got) | ||
| } | ||
|
|
||
| func TestGetStateStorePath_NewNode_RocksDB(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetStateStorePath(home, "rocksdb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "cosmos", "rocksdb"), got) | ||
| } | ||
|
|
||
| func TestGetStateStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "pebbledb") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| func TestGetStateStorePath_LegacyForDifferentBackend(t *testing.T) { | ||
| home := t.TempDir() | ||
| // Legacy rocksdb dir exists but we ask for pebbledb — no legacy match | ||
| require.NoError(t, os.MkdirAll(filepath.Join(home, "data", "rocksdb"), 0755)) | ||
|
|
||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "cosmos", "pebbledb"), got) | ||
| } | ||
|
|
||
| // --- GetEVMStateStorePath --- | ||
|
|
||
| func TestGetEVMStateStorePath_NewNode_Pebble(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetEVMStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "evm", "pebbledb"), got) | ||
| } | ||
|
|
||
| func TestGetEVMStateStorePath_NewNode_RocksDB(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetEVMStateStorePath(home, "rocksdb") | ||
| assert.Equal(t, filepath.Join(home, "data", "state_store", "evm", "rocksdb"), got) | ||
| } | ||
|
|
||
| func TestGetEVMStateStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "evm_ss") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetEVMStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetReceiptStorePath --- | ||
|
|
||
| func TestGetReceiptStorePath_NewNode_Pebble(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetReceiptStorePath(home, "pebbledb") | ||
| assert.Equal(t, filepath.Join(home, "data", "ledger", "receipt", "pebbledb"), got) | ||
| } | ||
|
|
||
| func TestGetReceiptStorePath_NewNode_Parquet(t *testing.T) { | ||
| home := t.TempDir() | ||
| got := GetReceiptStorePath(home, "parquet") | ||
| assert.Equal(t, filepath.Join(home, "data", "ledger", "receipt", "parquet"), got) | ||
| } | ||
|
|
||
| func TestGetReceiptStorePath_LegacyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "receipt.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
|
|
||
| got := GetReceiptStorePath(home, "pebbledb") | ||
| assert.Equal(t, legacy, got) | ||
| } | ||
|
|
||
| // --- GetChangelogPath (unchanged, but verify) --- | ||
|
|
||
| func TestGetChangelogPath(t *testing.T) { | ||
| assert.Equal(t, "/foo/bar/changelog", GetChangelogPath("/foo/bar")) | ||
| } | ||
|
|
||
| // --- Edge: new path already has data (second run of new node) --- | ||
|
|
||
| func TestGetCosmosSCStorePath_NewDataAlreadyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| newPath := filepath.Join(home, "data", "state_commit", "memiavl") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetCosmosSCStorePath(home) | ||
| assert.Equal(t, newPath, got, "should use new path when legacy is absent even if new path already exists") | ||
| } | ||
|
|
||
| func TestGetStateStorePath_NewDataAlreadyExists(t *testing.T) { | ||
| home := t.TempDir() | ||
| newPath := filepath.Join(home, "data", "state_store", "cosmos", "pebbledb") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetStateStorePath(home, "pebbledb") | ||
| assert.Equal(t, newPath, got, "should use new path when legacy is absent even if new path already exists") | ||
| } | ||
|
|
||
| // --- Edge: both legacy and new exist (legacy wins) --- | ||
|
|
||
| func TestGetCosmosSCStorePath_BothExist(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "committer.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
| newPath := filepath.Join(home, "data", "state_commit", "memiavl") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetCosmosSCStorePath(home) | ||
| assert.Equal(t, legacy, got, "legacy should take precedence when both exist") | ||
| } | ||
|
|
||
| func TestGetReceiptStorePath_BothExist(t *testing.T) { | ||
| home := t.TempDir() | ||
| legacy := filepath.Join(home, "data", "receipt.db") | ||
| require.NoError(t, os.MkdirAll(legacy, 0755)) | ||
| newPath := filepath.Join(home, "data", "ledger", "receipt", "pebbledb") | ||
| require.NoError(t, os.MkdirAll(newPath, 0755)) | ||
|
|
||
| got := GetReceiptStorePath(home, "pebbledb") | ||
| assert.Equal(t, legacy, got, "legacy should take precedence when both exist") | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Harden this by requiring the type of path? e.g. directory? file etc.
This is racy in that it doesn't atomically check and it repeatedly checks but hopefully the probability of race is low enough that we can accept the race condition as negligible.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Make sense, will do