bench: fix and expand benchmark coverage

Author: georgewfraser

TODOS.md

@@ -25,6 +25,7 @@
 - Compilation is very slow in the presence of lots of errors
 - Use package graph to limit search for find-usages/goto-def
 - Package separately for windows, mac, linux (see https://github.com/rust-lang/rust-analyzer/blob/master/.github/workflows/release.yaml)
+- Add more comprehensive benchmarks covering completion, navigation, indexing, and incremental compilation
 
 # Features
 - Autocomplete new method name based on "no such method" errors.
@@ -56,4 +57,4 @@
     - Implement single abstract method
     - Generate cases for enum
 - Code lens
-    - Inherited methods
+    - Inherited methods

scripts/benchmark.sh

@@ -1,14 +1,27 @@
 
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 # Compile the benchmark
 mvn test-compile
 
 # Emit the dependencies classpath
 mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=scripts/classpath.txt
 
-# Run the benchmark
-java -cp $(cat scripts/classpath.txt):target/classes:target/test-classes --illegal-access=warn org.openjdk.jmh.Main BenchmarkPruner
+java_args=(
+    --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+    --add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
+    --add-exports jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
+    --add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
+    --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+    --add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED
+    --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+    --add-opens jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+)
+
+# Run the benchmarks
+java "${java_args[@]}" -cp "$(cat scripts/classpath.txt):target/classes:target/test-classes" org.openjdk.jmh.Main BenchmarkPruner BenchmarkParser
 
 # Clean up
-rm scripts/classpath.txt
+rm scripts/classpath.txt

src/test/java/org/javacs/BenchmarkParser.java

@@ -1,9 +1,14 @@
 package org.javacs;
 
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Instant;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
 import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
 
 @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
 @Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
@@ -12,18 +17,45 @@ public class BenchmarkParser {
 
     @State(Scope.Benchmark)
     public static class CompilerState {
-        public Path file = Paths.get("src/main/java/org/javacs/JavaLanguageServer.java").normalize();
+        public Path file;
+        public String contents;
+        private long version;
+
+        @Setup(org.openjdk.jmh.annotations.Level.Trial)
+        public void setup() throws IOException {
+            FileStore.reset();
+            quietBenchmarkLogging();
+
+            file = Paths.get("src/main/java/org/javacs/JavaLanguageServer.java").normalize();
+            contents = Files.readString(file);
+        }
+
+        public SourceFileObject source() {
+            return new SourceFileObject(file, contents, Instant.ofEpochMilli(++version));
+        }
+
+        private static void quietBenchmarkLogging() {
+            Main.setRootFormat();
+            Logger.getLogger("").setLevel(java.util.logging.Level.WARNING);
+            Logger.getLogger("main").setLevel(java.util.logging.Level.WARNING);
+        }
     }
 
     @Benchmark
-    public void parse(CompilerState state) {
-        Parser.parseFile(state.file);
+    public void parse(CompilerState state, Blackhole blackhole) {
+        var parse = Parser.parseJavaFileObject(state.source());
+        blackhole.consume(parse.root);
     }
 
     public static void main(String[] args) {
         var state = new CompilerState();
+        try {
+            state.setup();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
         while (true) {
-            Parser.parseFile(state.file);
+            Parser.parseJavaFileObject(state.source());
         }
     }
 }

src/test/java/org/javacs/BenchmarkPruner.java

@@ -1,5 +1,7 @@
 package org.javacs;
 
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.time.Instant;
 import java.util.Collections;
@@ -9,6 +11,7 @@
 import java.util.logging.Logger;
 import org.javacs.completion.PruneMethodBodies;
 import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
 
 @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
 @Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS)
@@ -17,19 +20,35 @@ public class BenchmarkPruner {
 
     @State(Scope.Benchmark)
     public static class CompilerState {
-        public SourceFileObject file = file(false);
-        public SourceFileObject pruned = file(true);
-        public JavaCompilerService compiler = createCompiler();
+        public java.nio.file.Path file;
+        public String plainContents;
+        public String prunedContents;
+        public JavaCompilerService compiler;
+        private long version;
 
-        private SourceFileObject file(boolean prune) {
-            var file = Paths.get("src/main/java/org/javacs/InferConfig.java").normalize();
-            if (prune) {
-                var task = compiler.parse(file);
-                var contents = new PruneMethodBodies(task.task).scan(task.root, 11222L).toString();
-                return new SourceFileObject(file, contents, Instant.now());
-            } else {
-                return new SourceFileObject(file);
-            }
+        @Setup(org.openjdk.jmh.annotations.Level.Trial)
+        public void setup() throws IOException {
+            FileStore.reset();
+            quietBenchmarkLogging();
+
+            file = Paths.get("src/main/java/org/javacs/InferConfig.java").normalize();
+            plainContents = Files.readString(file);
+            compiler = createCompiler();
+
+            var task = compiler.parse(source(plainContents));
+            prunedContents = new PruneMethodBodies(task.task).scan(task.root, 11222L).toString();
+        }
+
+        public SourceFileObject plainSource() {
+            return source(plainContents);
+        }
+
+        public SourceFileObject prunedSource() {
+            return source(prunedContents);
+        }
+
+        private SourceFileObject source(String contents) {
+            return new SourceFileObject(file, contents, Instant.ofEpochMilli(++version));
         }
 
         private static JavaCompilerService createCompiler() {
@@ -40,21 +59,32 @@ private static JavaCompilerService createCompiler() {
             var classPath = new InferConfig(workspaceRoot).classPath();
             return new JavaCompilerService(classPath, Collections.emptySet(), Collections.emptySet());
         }
+
+        private static void quietBenchmarkLogging() {
+            Main.setRootFormat();
+            Logger.getLogger("").setLevel(java.util.logging.Level.WARNING);
+            Logger.getLogger("main").setLevel(java.util.logging.Level.WARNING);
+        }
     }
 
     @Benchmark
-    public void parsePlain(CompilerState state) {
-        Parser.parseJavaFileObject(state.file);
+    public void parsePlain(CompilerState state, Blackhole blackhole) {
+        var parse = Parser.parseJavaFileObject(state.plainSource());
+        blackhole.consume(parse.root);
     }
 
     @Benchmark
-    public void compilePruned(CompilerState state) {
-        state.compiler.compile(List.of(state.pruned)).close();
+    public void compilePruned(CompilerState state, Blackhole blackhole) {
+        try (var compile = state.compiler.compile(List.of(state.prunedSource()))) {
+            blackhole.consume(compile.root());
+        }
     }
 
     @Benchmark
-    public void compilePlain(CompilerState state) {
-        state.compiler.compile(List.of(state.file)).close();
+    public void compilePlain(CompilerState state, Blackhole blackhole) {
+        try (var compile = state.compiler.compile(List.of(state.plainSource()))) {
+            blackhole.consume(compile.root());
+        }
     }
 
     private static final Logger LOG = Logger.getLogger("main");