Skip to content

Commit 641f93b

Browse files
adamretteradamretter
committed
Add scripts for running rocksdbjni Samples and Benchmark
1 parent d982aba commit 641f93b

7 files changed

Lines changed: 258 additions & 157 deletions

File tree

‎java/benchmark/pom.xml‎

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,23 +68,54 @@
6868
<dependency>
6969
<groupId>org.rocksdb</groupId>
7070
<artifactId>rocksdbjni</artifactId>
71+
7172
<!-- TODO(AR) make sure this is set to the latest version you want to benchmark -->
72-
<version>10.4.2</version>
73+
<version>10.5.1</version>
74+
7375
</dependency>
7476
</dependencies>
7577

7678
<build>
7779
<plugins>
7880
<plugin>
79-
<groupId>org.apache.maven.plugins</groupId>
80-
<artifactId>maven-compiler-plugin</artifactId>
81-
<version>3.15.0</version>
82-
<configuration>
83-
<source>${project.build.source}</source>
84-
<target>${project.build.target}</target>
85-
<encoding>${project.build.sourceEncoding}</encoding>
86-
</configuration>
87-
</plugin>
81+
<groupId>org.apache.maven.plugins</groupId>
82+
<artifactId>maven-compiler-plugin</artifactId>
83+
<version>3.15.0</version>
84+
<configuration>
85+
<source>${project.build.source}</source>
86+
<target>${project.build.target}</target>
87+
<encoding>${project.build.sourceEncoding}</encoding>
88+
</configuration>
89+
</plugin>
90+
91+
<plugin>
92+
<groupId>org.codehaus.mojo</groupId>
93+
<artifactId>appassembler-maven-plugin</artifactId>
94+
<version>2.1.0</version>
95+
<executions>
96+
<execution>
97+
<id>appassembler</id>
98+
<phase>package</phase>
99+
<goals>
100+
<goal>assemble</goal>
101+
</goals>
102+
<configuration>
103+
<repositoryLayout>flat</repositoryLayout>
104+
<repositoryName>lib</repositoryName>
105+
<binFileExtensions>
106+
<unix>.sh</unix>
107+
</binFileExtensions>
108+
<assembleDirectory>${project.build.directory}${file.separator}${project.artifactId}-${project.version}-dir</assembleDirectory>
109+
<programs>
110+
<program>
111+
<id>benchmark</id>
112+
<mainClass>org.rocksdb.benchmark.DbBenchmark</mainClass>
113+
</program>
114+
</programs>
115+
</configuration>
116+
</execution>
117+
</executions>
118+
</plugin>
88119
</plugins>
89120
</build>
90121

‎java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java‎

Lines changed: 136 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -247,15 +247,15 @@ public WriteTask(
247247
}
248248
} else {
249249
for (long i = 0; i < numEntries_; i += entriesPerBatch_) {
250-
WriteBatch batch = new WriteBatch();
251-
for (long j = 0; j < entriesPerBatch_; j++) {
252-
getKey(key, i + j, keyRange_);
253-
DbBenchmark.this.gen_.generate(value);
254-
batch.put(key, value);
255-
stats_.finishedSingleOp(keySize_ + valueSize_);
250+
try (final WriteBatch batch = new WriteBatch()) {
251+
for (long j = 0; j < entriesPerBatch_; j++) {
252+
getKey(key, i + j, keyRange_);
253+
DbBenchmark.this.gen_.generate(value);
254+
batch.put(key, value);
255+
stats_.finishedSingleOp(keySize_ + valueSize_);
256+
}
257+
db_.write(writeOpt_, batch);
256258
}
257-
db_.write(writeOpt_, batch);
258-
batch.dispose();
259259
writeRateControl(i);
260260
if (isFinished()) {
261261
return;
@@ -415,19 +415,18 @@ public ReadSequentialTask(
415415
super(tid, randSeed, numEntries, keyRange);
416416
}
417417
@Override public void runTask() throws RocksDBException {
418-
RocksIterator iter = db_.newIterator();
419-
long i;
420-
for (iter.seekToFirst(), i = 0;
421-
iter.isValid() && i < numEntries_;
422-
iter.next(), ++i) {
423-
stats_.found_++;
424-
stats_.finishedSingleOp(iter.key().length + iter.value().length);
425-
if (isFinished()) {
426-
iter.dispose();
427-
return;
418+
try (final RocksIterator iter = db_.newIterator()) {
419+
long i;
420+
for (iter.seekToFirst(),i=0;
421+
iter.isValid() &&i < numEntries_;
422+
iter.next(), ++i){
423+
stats_.found_++;
424+
stats_.finishedSingleOp(iter.key().length + iter.value().length);
425+
if (isFinished()) {
426+
return;
427+
}
428428
}
429429
}
430-
iter.dispose();
431430
}
432431
}
433432

@@ -476,17 +475,19 @@ public DbBenchmark(Map<Flag, Object> flags) throws Exception {
476475
gen_ = new RandomGenerator(randSeed_, compressionRatio_);
477476
}
478477

479-
private void prepareReadOptions(ReadOptions options) {
478+
private ReadOptions prepareReadOptions(final ReadOptions options) {
480479
options.setVerifyChecksums((Boolean)flags_.get(Flag.verify_checksum));
481480
options.setTailing((Boolean)flags_.get(Flag.use_tailing_iterator));
481+
return options;
482482
}
483483

484-
private void prepareWriteOptions(WriteOptions options) {
484+
private WriteOptions prepareWriteOptions(final WriteOptions options) {
485485
options.setSync((Boolean)flags_.get(Flag.sync));
486486
options.setDisableWAL((Boolean)flags_.get(Flag.disable_wal));
487+
return options;
487488
}
488489

489-
private void prepareOptions(Options options) throws RocksDBException {
490+
private Options prepareOptions(final Options options) throws RocksDBException {
490491
if (!useExisting_) {
491492
options.setCreateIfMissing(true);
492493
} else {
@@ -615,127 +616,128 @@ private void prepareOptions(Options options) throws RocksDBException {
615616
// TODO(yhchiang): add RocksDB.openForReadOnly() to enable Flag.readonly
616617
// TODO(yhchiang): enable Flag.merge_operator by switch
617618
*/
619+
620+
return options;
618621
}
619622

620623
private void run() throws RocksDBException {
621624
if (!useExisting_) {
622625
destroyDb();
623626
}
624-
Options options = new Options();
625-
prepareOptions(options);
626-
open(options);
627-
628-
printHeader(options);
629-
630-
for (String benchmark : benchmarks_) {
631-
List<Callable<Stats>> tasks = new ArrayList<Callable<Stats>>();
632-
List<Callable<Stats>> bgTasks = new ArrayList<Callable<Stats>>();
633-
WriteOptions writeOpt = new WriteOptions();
634-
prepareWriteOptions(writeOpt);
635-
ReadOptions readOpt = new ReadOptions();
636-
prepareReadOptions(readOpt);
637-
int currentTaskId = 0;
638-
boolean known = true;
639-
640-
switch (benchmark) {
641-
case "fillseq":
642-
tasks.add(new WriteSequentialTask(
643-
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
644-
break;
645-
case "fillbatch":
646-
tasks.add(
647-
new WriteSequentialTask(currentTaskId++, randSeed_, num_, num_, writeOpt, 1000));
648-
break;
649-
case "fillrandom":
650-
tasks.add(new WriteRandomTask(
651-
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
652-
break;
653-
case "filluniquerandom":
654-
tasks.add(new WriteUniqueRandomTask(
655-
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
656-
break;
657-
case "fillsync":
658-
writeOpt.setSync(true);
659-
tasks.add(new WriteRandomTask(
660-
currentTaskId++, randSeed_, num_ / 1000, num_ / 1000,
661-
writeOpt, 1));
662-
break;
663-
case "readseq":
664-
for (int t = 0; t < threadNum_; ++t) {
665-
tasks.add(new ReadSequentialTask(
666-
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
667-
}
668-
break;
669-
case "readrandom":
670-
for (int t = 0; t < threadNum_; ++t) {
671-
tasks.add(new ReadRandomTask(
672-
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
673-
}
674-
break;
675-
case "readwhilewriting":
676-
WriteTask writeTask = new WriteRandomTask(
677-
-1, randSeed_, Long.MAX_VALUE, num_, writeOpt, 1, writesPerSeconds_);
678-
writeTask.stats_.setExcludeFromMerge();
679-
bgTasks.add(writeTask);
680-
for (int t = 0; t < threadNum_; ++t) {
681-
tasks.add(new ReadRandomTask(
682-
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
683-
}
684-
break;
685-
case "readhot":
686-
for (int t = 0; t < threadNum_; ++t) {
687-
tasks.add(new ReadRandomTask(
688-
currentTaskId++, randSeed_, reads_ / threadNum_, num_ / 100));
689-
}
690-
break;
691-
case "delete":
692-
destroyDb();
693-
open(options);
694-
break;
695-
default:
696-
known = false;
697-
System.err.println("Unknown benchmark: " + benchmark);
698-
break;
699-
}
700-
if (known) {
701-
ExecutorService executor = Executors.newCachedThreadPool();
702-
ExecutorService bgExecutor = Executors.newCachedThreadPool();
703-
try {
704-
// measure only the main executor time
705-
List<Future<Stats>> bgResults = new ArrayList<Future<Stats>>();
706-
for (Callable bgTask : bgTasks) {
707-
bgResults.add(bgExecutor.submit(bgTask));
708-
}
709-
start();
710-
List<Future<Stats>> results = executor.invokeAll(tasks);
711-
executor.shutdown();
712-
boolean finished = executor.awaitTermination(10, TimeUnit.SECONDS);
713-
if (!finished) {
714-
System.out.format(
715-
"Benchmark %s was not finished before timeout.",
716-
benchmark);
717-
executor.shutdownNow();
718-
}
719-
setFinished(true);
720-
bgExecutor.shutdown();
721-
finished = bgExecutor.awaitTermination(10, TimeUnit.SECONDS);
722-
if (!finished) {
723-
System.out.format(
724-
"Benchmark %s was not finished before timeout.",
725-
benchmark);
726-
bgExecutor.shutdownNow();
627+
628+
try (final Options options = prepareOptions(new Options());
629+
final RocksDB db = open(options)) {
630+
631+
this.db_ = db;
632+
633+
printHeader(options);
634+
635+
for (String benchmark : benchmarks_) {
636+
List<Callable<Stats>> tasks = new ArrayList<>();
637+
List<Callable<Stats>> bgTasks = new ArrayList<>();
638+
639+
try (final WriteOptions writeOpt = prepareWriteOptions(new WriteOptions());
640+
final ReadOptions readOpt = prepareReadOptions(new ReadOptions())) {
641+
int currentTaskId = 0;
642+
boolean known = true;
643+
644+
switch (benchmark) {
645+
case "fillseq":
646+
tasks.add(new WriteSequentialTask(
647+
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
648+
break;
649+
case "fillbatch":
650+
tasks.add(
651+
new WriteSequentialTask(currentTaskId++, randSeed_, num_, num_, writeOpt, 1000));
652+
break;
653+
case "fillrandom":
654+
tasks.add(new WriteRandomTask(
655+
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
656+
break;
657+
case "filluniquerandom":
658+
tasks.add(new WriteUniqueRandomTask(
659+
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
660+
break;
661+
case "fillsync":
662+
writeOpt.setSync(true);
663+
tasks.add(new WriteRandomTask(
664+
currentTaskId++, randSeed_, num_ / 1000, num_ / 1000,
665+
writeOpt, 1));
666+
break;
667+
case "readseq":
668+
for (int t = 0; t < threadNum_; ++t) {
669+
tasks.add(new ReadSequentialTask(
670+
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
671+
}
672+
break;
673+
case "readrandom":
674+
for (int t = 0; t < threadNum_; ++t) {
675+
tasks.add(new ReadRandomTask(
676+
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
677+
}
678+
break;
679+
case "readwhilewriting":
680+
WriteTask writeTask = new WriteRandomTask(
681+
-1, randSeed_, Long.MAX_VALUE, num_, writeOpt, 1, writesPerSeconds_);
682+
writeTask.stats_.setExcludeFromMerge();
683+
bgTasks.add(writeTask);
684+
for (int t = 0; t < threadNum_; ++t) {
685+
tasks.add(new ReadRandomTask(
686+
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
687+
}
688+
break;
689+
case "readhot":
690+
for (int t = 0; t < threadNum_; ++t) {
691+
tasks.add(new ReadRandomTask(
692+
currentTaskId++, randSeed_, reads_ / threadNum_, num_ / 100));
693+
}
694+
break;
695+
case "delete":
696+
destroyDb();
697+
open(options);
698+
break;
699+
default:
700+
known = false;
701+
System.err.println("Unknown benchmark: " + benchmark);
702+
break;
727703
}
704+
if (known) {
705+
ExecutorService executor = Executors.newCachedThreadPool();
706+
ExecutorService bgExecutor = Executors.newCachedThreadPool();
707+
try {
708+
// measure only the main executor time
709+
List<Future<Stats>> bgResults = new ArrayList<Future<Stats>>();
710+
for (Callable bgTask : bgTasks) {
711+
bgResults.add(bgExecutor.submit(bgTask));
712+
}
713+
start();
714+
List<Future<Stats>> results = executor.invokeAll(tasks);
715+
executor.shutdown();
716+
boolean finished = executor.awaitTermination(10, TimeUnit.SECONDS);
717+
if (!finished) {
718+
System.out.format(
719+
"Benchmark %s was not finished before timeout.",
720+
benchmark);
721+
executor.shutdownNow();
722+
}
723+
setFinished(true);
724+
bgExecutor.shutdown();
725+
finished = bgExecutor.awaitTermination(10, TimeUnit.SECONDS);
726+
if (!finished) {
727+
System.out.format(
728+
"Benchmark %s was not finished before timeout.",
729+
benchmark);
730+
bgExecutor.shutdownNow();
731+
}
728732

729-
stop(benchmark, results, currentTaskId);
730-
} catch (InterruptedException e) {
731-
System.err.println(e);
733+
stop(benchmark, results, currentTaskId);
734+
} catch (InterruptedException e) {
735+
System.err.println(e);
736+
}
737+
}
732738
}
733739
}
734-
writeOpt.dispose();
735-
readOpt.dispose();
736740
}
737-
options.dispose();
738-
db_.close();
739741
}
740742

741743
private void printHeader(Options options) {
@@ -765,9 +767,9 @@ void printWarnings() {
765767
}
766768
}
767769

768-
private void open(Options options) throws RocksDBException {
770+
private RocksDB open(Options options) throws RocksDBException {
769771
System.out.println("Using database directory: " + databaseDir_);
770-
db_ = RocksDB.open(options, databaseDir_);
772+
return RocksDB.open(options, databaseDir_);
771773
}
772774

773775
private void start() {

0 commit comments

Comments
 (0)