diff --git a/cmd/tesseract/posix/main.go b/cmd/tesseract/posix/main.go index 97bdf3a3..5135416f 100644 --- a/cmd/tesseract/posix/main.go +++ b/cmd/tesseract/posix/main.go @@ -36,6 +36,7 @@ import ( "time" "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v4/options" "github.com/dustin/go-humanize" "github.com/transparency-dev/tessera" tposix "github.com/transparency-dev/tessera/storage/posix" @@ -100,9 +101,12 @@ var ( clientHTTPMaxIdlePerHost = flag.Int("client_http_max_idle_per_host", 10, "Maximum number of idle HTTP connections per host for outgoing requests.") garbageCollectionInterval = flag.Duration("garbage_collection_interval", 10*time.Second, "Interval between scans to remove obsolete partial tiles and entry bundles. Set to 0 to disable.") antispamBatchSize = flag.Uint("antispam_batch_size", 10000, "Maximum number of antispam rows to insert per batch update.") - antispamBlockCacheSize = flag.String("antispam_block_cache_size", "768MB", "Amount of RAM to allocate for antispam block cache, set to zero to disable.") + antispamBlockCacheSize = flag.String("antispam_block_cache_size", "0MB", "Amount of RAM to allocate for antispam block cache, set to zero to disable. Default disabled since compression is off.") antispamIndexCacheSize = flag.String("antispam_index_cache_size", "768MB", "Amount of RAM to allocate for antispam index cache, set to zero for unlimited.") antispamCompactionInterval = flag.Duration("antispam_compaction_interval", tposix_as.DefaultCompactionInterval, "Interval between GC/compaction runs on the antispam index.") + antispamNumCompactors = flag.Int("antispam_num_compactors", 2, "Number of BadgerDB compactors to run.") + antispamMemTableSize = flag.Int64("antispam_mem_table_size", 256<<20, "Size of BadgerDB memtables.") + antispamBaseTableSize = flag.Int64("antispam_base_table_size", 16<<20, "Size of BadgerDB base tables.") awaiterPollInterval = flag.Duration("awaiter_poll_interval", storage.DefaultAwaiterPollInterval, "Interval between two checkpoint polls by the awaiter. Used for antispam, and if enable_publication_awaiter is set, to block add-* requests responses. Must be strictly positive or defaults to DefaultAwaiterPollInterval.") // Infrastructure setup flags @@ -245,6 +249,10 @@ func newStorage(ctx context.Context, signer note.Signer) (st *storage.CTStorage, CompactionInterval: *antispamCompactionInterval, BadgerOptions: func(o badger.Options) badger.Options { return o. + WithCompression(options.None). // Off as this appears to cause memory issues when compacting large indices. + WithMemTableSize(*antispamMemTableSize). // Default tunes memtables for high write throughput + WithBaseTableSize(*antispamBaseTableSize). // Default tunes to reduce file count + WithNumCompactors(*antispamNumCompactors). // Default tunes to be able keep up with high throughput of LSM merges WithIndexCacheSize(int64(antispamIndexCacheBytes)). WithBlockCacheSize(int64(antispamBlockCacheBytes)) },