issue #424 - refactor parser benchmark and add valueset benchmarks

1. introduced run options under FHIRBenchmarkRunner for setting JMH
params instead of passing via system properties

2. refactored FHIRParserBenchmark to use JMH params

3. introduced LargeValueSetCreator under fhir-examples-generator and
specific examples for json and xml

4. added FHIRValueSetBenchmarks for measuring ValueSet parse times

5. added LargeValueSetCreator variants for creating HashSet and text
file equivalents, and added Benchmarks for measuring parse times for
those as well

Signed-off-by: Lee Surprenant <lmsurpre@us.ibm.com>

Author: lmsurpre

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRGeneratorBenchmark.java

@@ -6,13 +6,12 @@
 
 package com.ibm.fhir.benchmark;
 
-import static com.ibm.fhir.benchmark.runner.FHIRBenchmarkRunner.PROPERTY_EXAMPLE_NAME;
-
 import java.io.StringReader;
 import java.io.Writer;
 
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
@@ -30,9 +29,6 @@ public class FHIRGeneratorBenchmark {
     @State(Scope.Benchmark)
     public static class FHIRGeneratorState {
         public static final Writer NOP_WRITER = BenchmarkUtil.createNOPWriter();
-        public static final String SPEC_EXAMPLE_NAME = System.getProperty(PROPERTY_EXAMPLE_NAME);
-        public static final String JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, SPEC_EXAMPLE_NAME);
-        public static final String XML_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.XML, SPEC_EXAMPLE_NAME);
 
         public FhirContext context;
         public FHIRGenerator jsonGenerator;
@@ -48,6 +44,12 @@ public void setUp() throws Exception {
             resource = FHIRParser.parser(Format.JSON).parse(new StringReader(JSON_SPEC_EXAMPLE));
             baseResource = context.newJsonParser().parseResource(new StringReader(JSON_SPEC_EXAMPLE));
         }
+        
+        @Param({"test"})
+        public String JSON_SPEC_EXAMPLE;
+        
+        @Param({"test"})
+        public String XML_SPEC_EXAMPLE;
     }
 
     @Benchmark
@@ -72,7 +74,6 @@ public void benchmarkHAPIXMLGenerator(FHIRGeneratorState state) throws Exception
 
     public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRGeneratorBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
-                .run();
+                .run(BenchmarkUtil.getRandomSpecExampleName());
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRParserBenchmark.java

@@ -6,12 +6,11 @@
 
 package com.ibm.fhir.benchmark;
 
-import static com.ibm.fhir.benchmark.runner.FHIRBenchmarkRunner.PROPERTY_EXAMPLE_NAME;
-
+import java.io.IOException;
 import java.io.StringReader;
-import java.io.Writer;
 
 import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
@@ -20,54 +19,65 @@
 import com.ibm.fhir.benchmark.util.BenchmarkUtil;
 import com.ibm.fhir.model.format.Format;
 import com.ibm.fhir.model.parser.FHIRParser;
+import com.ibm.fhir.model.resource.Resource;
 
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.parser.StrictErrorHandler;
 
+
 public class FHIRParserBenchmark {
+    @State(Scope.Thread)
+    public static class FHIRParsers {
+        public FHIRParser jsonParser = FHIRParser.parser(Format.JSON);
+        public FHIRParser xmlParser = FHIRParser.parser(Format.XML);
+    }
+    
     @State(Scope.Benchmark)
     public static class FHIRParserState {
-        public static final Writer NOP_WRITER = BenchmarkUtil.createNOPWriter();
-        public static final String SPEC_EXAMPLE_NAME = System.getProperty(PROPERTY_EXAMPLE_NAME);
-        public static final String JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, SPEC_EXAMPLE_NAME);
-        public static final String XML_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.XML, SPEC_EXAMPLE_NAME);
-        
         public FhirContext context;
-        public FHIRParser jsonParser;
-        public FHIRParser xmlParser;
+        public String JSON_SPEC_EXAMPLE;
+        public String XML_SPEC_EXAMPLE;
+        
+        // JMH will inject the value into the annotated field before any Setup method is called.
+        @Param({"valuesets"})
+        public String exampleName;
 
         @Setup
-        public void setUp() {            
+        public void setUp() throws IOException {
+            if (exampleName == null) {
+                System.err.println("exampleName is null; if you're in Eclipse then make sure annotation processing is on and you've ran 'mvn clean package'.");
+                System.exit(1);
+            }
+            System.out.println("Setting up for example " + exampleName);
             context = FhirContext.forR4();
             context.setParserErrorHandler(new StrictErrorHandler());
-            jsonParser = FHIRParser.parser(Format.JSON);
-            xmlParser = FHIRParser.parser(Format.XML);
+            JSON_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.JSON, exampleName);
+            XML_SPEC_EXAMPLE = BenchmarkUtil.getSpecExample(Format.XML, exampleName);
         }
     }
 
     @Benchmark
-    public void benchmarkJsonParser(FHIRParserState state) throws Exception {
-        state.jsonParser.parse(new StringReader(FHIRParserState.JSON_SPEC_EXAMPLE));
+    public Resource benchmarkJsonParser(FHIRParsers parsers, FHIRParserState state) throws Exception {
+        return parsers.jsonParser.parse(new StringReader(state.JSON_SPEC_EXAMPLE));
     }
 
     @Benchmark
-    public void benchmarkXMLParser(FHIRParserState state) throws Exception {
-        state.xmlParser.parse(new StringReader(FHIRParserState.XML_SPEC_EXAMPLE));
+    public Resource benchmarkXMLParser(FHIRParsers parsers, FHIRParserState state) throws Exception {
+        return parsers.xmlParser.parse(new StringReader(state.XML_SPEC_EXAMPLE));
     }
 
     @Benchmark
     public void benchmarkHAPIJsonParser(FHIRParserState state) throws Exception {
-        state.context.newJsonParser().parseResource(new StringReader(FHIRParserState.JSON_SPEC_EXAMPLE));
+        state.context.newJsonParser().parseResource(new StringReader(state.JSON_SPEC_EXAMPLE));
     }
 
     @Benchmark
     public void benchmarkHAPIXMLParser(FHIRParserState state) throws Exception {
-        state.context.newXmlParser().parseResource(new StringReader(FHIRParserState.XML_SPEC_EXAMPLE));
+        state.context.newXmlParser().parseResource(new StringReader(state.XML_SPEC_EXAMPLE));
     }
 
     public static void main(String[] args) throws Exception {
-        new FHIRBenchmarkRunner(FHIRParserBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
-                .run();
+//        new FHIRBenchmarkRunner(FHIRParserBenchmark.class).runAll();
+        new FHIRBenchmarkRunner(FHIRParserBenchmark.class).run();
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRParserValidatorGeneratorBenchmark.java

@@ -91,7 +91,6 @@ public void benchmarkHAPIXMLParserValidatorGenerator(FHIRParserValidatorGenerato
 
     public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRParserValidatorGeneratorBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
                 .run();
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRPathEvaluatorBenchmark.java

@@ -91,6 +91,6 @@ public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRPathEvaluatorBenchmark.class)
                 .property(PROPERTY_EXAMPLE_NAME, EXAMPLE_NAME)
                 .property(PROPERTY_EXPRESSION, EXPRESSION)
-                .run();
+                .run(BenchmarkUtil.getRandomSpecExampleName());
     }
 }

fhir-benchmark/src/main/java/com/ibm/fhir/benchmark/FHIRValidatorBenchmark.java

@@ -63,7 +63,6 @@ public void benchmarkHAPIValidator(FHIRValidatorState state) throws Exception {
 
     public static void main(String[] args) throws Exception {
         new FHIRBenchmarkRunner(FHIRValidatorBenchmark.class)
-                .property(PROPERTY_EXAMPLE_NAME, BenchmarkUtil.getRandomSpecExampleName())
-                .run();
+                .run(BenchmarkUtil.getRandomSpecExampleName());
     }
 }