cleanup

* add case without Score-P
* disable tracing
* reduce calls to built-in functions

Author: AndreasGocht

benchmark/benchmark.py

@@ -11,19 +11,26 @@
 results = {}
 
 for test in tests:
-    results[test] = {"profile": {}, "trace": {}, "dummy": {}}
-    for elem in results[test]:
-        scorep_settings = ["--instrumenter-type={}".format(elem)]
+    results[test] = {"profile": {}, "trace": {}, "dummy": {}, "None": {}}
+    for instrumenter in results[test]:
+        if instrumenter is "None":
+            enable_scorep = False
+            scorep_settings = []
+        else:
+            enable_scorep = True
+            scorep_settings = ["--instrumenter-type={}".format(instrumenter)]
+
         print("#########")
         print("{}: {}".format(test, scorep_settings))
         print("#########")
         for reps in ["1000", "10000", "100000", "1000000", "10000000"]:
             times = bench.call(
                 test,
                 [reps],
+                enable_scorep,
                 scorep_settings=scorep_settings)
-            results[test][elem][reps] = times
+            results[test][instrumenter][reps] = times
             print("{:<8}: {}".format(reps, times))
 
 with open("results.pkl", "wb") as f:
-    pickle.dump(results,f)
+    pickle.dump(results, f)

benchmark/benchmark_helper.py

@@ -3,13 +3,14 @@
 import shutil
 import sys
 import time
+from IPython.core.magic_arguments import argument
 
 
 class BenchmarkEnv():
     def __init__(self, repetitions=10):
         self.env = os.environ.copy()
         self.env["SCOREP_ENABLE_PROFILING"] = "false"
-        self.env["SCOREP_ENABLE_TRACING"] = "true"
+        self.env["SCOREP_ENABLE_TRACING"] = "false"
         self.env["SCOREP_PROFILING_MAX_CALLPATH_DEPTH"] = "98"
         self.env["SCOREP_TOTAL_MEMORY"] = "3G"
         self.exp_dir = "benchmark_dir"
@@ -26,16 +27,14 @@ def __del__(self):
 #             self.exp_dir,
 #             ignore_errors=True)
 
-    def call(self, script="", ops=[], scorep_settings=[]):
-        """
-        return a triple with (returncode, stdout, stderr) from the call to subprocess
-        """
-
+    def call(self, script="", ops=[], enable_scorep=True, scorep_settings=[]):
         self.env["SCOREP_EXPERIMENT_DIRECTORY"] = self.exp_dir + \
             "/{}-{}-{}".format(script, ops, scorep_settings)
 
-        arguments = [sys.executable, "-m", "scorep"]
-        arguments.extend(scorep_settings)
+        arguments = [sys.executable]
+        if enable_scorep:
+            arguments.extend(["-m", "scorep"])
+            arguments.extend(scorep_settings)
         arguments.append(script)
         arguments.extend(ops)
 

benchmark/test_1.py

@@ -1,6 +1,10 @@
 import sys
 
 result = 0
-for i in range(int(sys.argv[1])):
+iterations = int(sys.argv[1])
+iteration_list = list(range(iterations))
+
+for i in iteration_list:
     result += 1
-print(result)
+
+assert(result == iterations)

benchmark/test_2.py

@@ -11,6 +11,10 @@ def add(val):
 
 
 result = 0
-for i in range(int(sys.argv[1])):
+iterations = int(sys.argv[1])
+iteration_list = list(range(iterations))
+
+for i in iteration_list:
     result = add(result)
-print(result)
+
+assert(result == iterations)