add benchmarks

Author: AndreasGocht

benchmark/benchmark.py

@@ -0,0 +1,29 @@
+'''
+Created on 04.10.2019
+
+@author: gocht
+'''
+import benchmark_helper
+import pickle
+
+bench = benchmark_helper.BenchmarkEnv()
+tests = ["test_1.py", "test_2.py"]
+resutls = {}
+
+for test in tests:
+    resutls[test] = {"profile": {}, "trace": {}, "dummy": {}}
+    for elem in resutls[test]:
+        scorep_settings = ["--instrumenter-type={}".format(elem)]
+        print("#########")
+        print("{}: {}".format(test, scorep_settings))
+        print("#########")
+        for reps in ["1000", "10000", "100000", "1000000", "10000000"]:
+            times = bench.call(
+                test,
+                [reps],
+                scorep_settings=scorep_settings)
+            resutls[test][elem][reps] = times
+            print("{:<8}: {}".format(reps, times))
+
+with open("resutls.pkl", "wb") as f:
+    pickle.dump(resutls,f)

benchmark/benchmark_helper.py

@@ -0,0 +1,56 @@
+import subprocess
+import os
+import shutil
+import sys
+import time
+
+
+class BenchmarkEnv():
+    def __init__(self, repetitons=10):
+        self.env = os.environ.copy()
+        self.env["SCOREP_ENABLE_PROFILING"] = "false"
+        self.env["SCOREP_ENABLE_TRACING"] = "true"
+        self.env["SCOREP_PROFILING_MAX_CALLPATH_DEPTH"] = "98"
+        self.env["SCOREP_TOTAL_MEMORY"] = "3G"
+        self.exp_dir = "benchmark_dir"
+        self.repetitons = repetitons
+
+        shutil.rmtree(
+            self.exp_dir,
+            ignore_errors=True)
+        os.mkdir(self.exp_dir)
+
+    def __del__(self):
+        pass
+#         shutil.rmtree(
+#             self.exp_dir,
+#             ignore_errors=True)
+
+    def call(self, script="", ops=[], scorep_settings=[]):
+        """
+        return a triple with (returncode, stdout, stderr) from the call to subprocess
+        """
+
+        self.env["SCOREP_EXPERIMENT_DIRECTORY"] = self.exp_dir + \
+            "/{}-{}-{}".format(script, ops, scorep_settings)
+
+        arguments = [sys.executable, "-m", "scorep"]
+        arguments.extend(scorep_settings)
+        arguments.append(script)
+        arguments.extend(ops)
+
+        runtimes = []
+        for i in range(self.repetitons):
+            begin = time.time()
+            out = subprocess.run(
+                arguments,
+                env=self.env,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            end = time.time()
+            assert(out.returncode == 0)
+
+            runtime = end - begin
+            runtimes.append(runtime)
+
+        return runtimes

benchmark/test_1.py

@@ -0,0 +1,6 @@
+import sys
+
+result = 0
+for i in range(int(sys.argv[1])):
+    result += 1
+print(result)

benchmark/test_2.py

@@ -0,0 +1,16 @@
+'''
+Created on 04.10.2019
+
+@author: gocht
+'''
+import sys
+
+
+def add(val):
+    return val + 1
+
+
+result = 0
+for i in range(int(sys.argv[1])):
+    result = add(result)
+print(result)