MemIAVL should only only keep 1 snapshot (#110)

* MemIAVL should only keep 1 current snapshot

* Fix comment

* Fix locking issue

* Use numcpu as default

Author: yzang2019

config/config.go

@@ -1,16 +1,15 @@
 package config
 
 const (
-	DefaultSnapshotInterval    = 10000
-	DefaultSnapshotKeepRecent  = 1
-	DefaultSnapshotWriterLimit = 1
-	DefaultAsyncCommitBuffer   = 100
-	DefaultCacheSize           = 100000
-	DefaultSSKeepRecent        = 100000
-	DefaultSSPruneInterval     = 600
-	DefaultSSImportWorkers     = 1
-	DefaultSSAsyncBuffer       = 100
-	DefaultSSHashRange         = 1000000
+	DefaultSnapshotInterval   = 10000
+	DefaultSnapshotKeepRecent = 0 // set to 0 to only keep one current snapshot
+	DefaultAsyncCommitBuffer  = 100
+	DefaultCacheSize          = 100000
+	DefaultSSKeepRecent       = 100000
+	DefaultSSPruneInterval    = 600
+	DefaultSSImportWorkers    = 1
+	DefaultSSAsyncBuffer      = 100
+	DefaultSSHashRange        = 1000000
 )
 
 type StateCommitConfig struct {

sc/memiavl/db.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -17,7 +18,6 @@ import (
 	errorutils "github.com/sei-protocol/sei-db/common/errors"
 	"github.com/sei-protocol/sei-db/common/logger"
 	"github.com/sei-protocol/sei-db/common/utils"
-	"github.com/sei-protocol/sei-db/config"
 	"github.com/sei-protocol/sei-db/proto"
 	"github.com/sei-protocol/sei-db/stream/changelog"
 	"github.com/sei-protocol/sei-db/stream/types"
@@ -209,8 +209,10 @@ func OpenDB(logger logger.Logger, targetVersion int64, opts Options) (*DB, error
 			return nil, errorutils.Join(err, db.Close())
 		}
 	}
-	if db.streamHandler == nil {
-		fmt.Println("[Debug] DB steam handler is nil??")
+
+	// We need to prune snapshots during start up to avoid snapshot leaks
+	if !db.readOnly {
+		db.pruneSnapshots()
 	}
 	return db, nil
 }
@@ -617,7 +619,8 @@ func (db *DB) Close() error {
 	db.mtx.Lock()
 	defer db.mtx.Unlock()
 	errs := []error{}
-
+	db.pruneSnapshotLock.Lock()
+	defer db.pruneSnapshotLock.Unlock()
 	// Close stream handler
 	if db.streamHandler != nil {
 		err := db.streamHandler.Close()
@@ -801,7 +804,8 @@ func initEmptyDB(dir string, initialVersion uint32) error {
 	tmp := NewEmptyMultiTree(initialVersion, 0)
 	snapshotDir := snapshotName(0)
 	// create tmp worker pool
-	pool := pond.New(config.DefaultSnapshotWriterLimit, config.DefaultSnapshotWriterLimit*10)
+	concurrency := runtime.NumCPU()
+	pool := pond.New(concurrency, concurrency*10)
 	defer pool.Stop()
 
 	if err := tmp.WriteSnapshot(context.Background(), filepath.Join(dir, snapshotDir), pool); err != nil {

sc/memiavl/export.go

@@ -5,10 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	"runtime"
 
 	errorutils "github.com/sei-protocol/sei-db/common/errors"
 	"github.com/sei-protocol/sei-db/common/logger"
-	"github.com/sei-protocol/sei-db/config"
 	"github.com/sei-protocol/sei-db/sc/types"
 )
 
@@ -36,7 +36,7 @@ func NewMultiTreeExporter(dir string, version uint32, onlyAllowExportOnSnapshotV
 			Dir:                 dir,
 			ZeroCopy:            true,
 			ReadOnly:            true,
-			SnapshotWriterLimit: config.DefaultSnapshotWriterLimit,
+			SnapshotWriterLimit: runtime.NumCPU(),
 		})
 		if err != nil {
 			return nil, fmt.Errorf("invalid height: %d, %w", version, err)

sc/memiavl/opts.go

@@ -2,6 +2,7 @@ package memiavl
 
 import (
 	"errors"
+	"runtime"
 
 	"github.com/sei-protocol/sei-db/config"
 )
@@ -55,10 +56,12 @@ func (opts *Options) FillDefaults() {
 	}
 
 	if opts.SnapshotWriterLimit <= 0 {
-		opts.SnapshotWriterLimit = config.DefaultSnapshotWriterLimit
+		opts.SnapshotWriterLimit = runtime.NumCPU()
 	}
 
 	if opts.CacheSize < 0 {
 		opts.CacheSize = config.DefaultCacheSize
 	}
+
+	opts.SnapshotKeepRecent = config.DefaultSnapshotKeepRecent
 }