Merge pull request #51 from score-p/benchmark

Add some performance benchmarks.

Author: AndreasGocht

benchmark/benchmark.py

@@ -0,0 +1,36 @@
+'''
+Created on 04.10.2019
+
+@author: gocht
+'''
+import benchmark_helper
+import pickle
+
+bench = benchmark_helper.BenchmarkEnv(repetitions=11)
+tests = ["test_1.py", "test_2.py"]
+results = {}
+
+for test in tests:
+    results[test] = {"profile": {}, "trace": {}, "dummy": {}, "None": {}}
+    for instrumenter in results[test]:
+        if instrumenter is "None":
+            enable_scorep = False
+            scorep_settings = []
+        else:
+            enable_scorep = True
+            scorep_settings = ["--instrumenter-type={}".format(instrumenter)]
+
+        print("#########")
+        print("{}: {}".format(test, scorep_settings))
+        print("#########")
+        for reps in ["1000", "10000", "100000", "1000000", "10000000"]:
+            times = bench.call(
+                test,
+                [reps],
+                enable_scorep,
+                scorep_settings=scorep_settings)
+            results[test][instrumenter][reps] = times
+            print("{:<8}: {}".format(reps, times))
+
+with open("results.pkl", "wb") as f:
+    pickle.dump(results, f)

benchmark/benchmark_helper.py

@@ -0,0 +1,54 @@
+import subprocess
+import os
+import shutil
+import sys
+import time
+
+
+class BenchmarkEnv():
+    def __init__(self, repetitions=10):
+        self.env = os.environ.copy()
+        self.env["SCOREP_ENABLE_PROFILING"] = "false"
+        self.env["SCOREP_ENABLE_TRACING"] = "false"
+        self.env["SCOREP_PROFILING_MAX_CALLPATH_DEPTH"] = "98"
+        self.env["SCOREP_TOTAL_MEMORY"] = "3G"
+        self.exp_dir = "benchmark_dir"
+        self.repetitions = repetitions
+
+        shutil.rmtree(
+            self.exp_dir,
+            ignore_errors=True)
+        os.mkdir(self.exp_dir)
+
+    def __del__(self):
+        pass
+#         shutil.rmtree(
+#             self.exp_dir,
+#             ignore_errors=True)
+
+    def call(self, script="", ops=[], enable_scorep=True, scorep_settings=[]):
+        self.env["SCOREP_EXPERIMENT_DIRECTORY"] = self.exp_dir + \
+            "/{}-{}-{}".format(script, ops, scorep_settings)
+
+        arguments = [sys.executable]
+        if enable_scorep:
+            arguments.extend(["-m", "scorep"])
+            arguments.extend(scorep_settings)
+        arguments.append(script)
+        arguments.extend(ops)
+
+        runtimes = []
+        for i in range(self.repetitions):
+            begin = time.time()
+            out = subprocess.run(
+                arguments,
+                env=self.env,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            end = time.time()
+            assert(out.returncode == 0)
+
+            runtime = end - begin
+            runtimes.append(runtime)
+
+        return runtimes

benchmark/run.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --time=04:00:00
+#SBATCH --exclusive
+#SBATCH -p haswell
+#SBATCH -A p_readex
+#SBATCH -N 1
+#SBATCH --ntasks-per-node=1
+#SBATCH -c 24
+#SBATCH --comment=no_monitoring
+#SBATCH --job-name benchmark_python
+
+
+module load Python/3.6.6-foss-2019a
+
+module use /home/gocht/modules
+module load scorep/scorep-6.0-python3.6.6-foss-2019a
+# Score-P 6.0 build with
+# ../configure --prefix=/home/gocht/sw/scorep/scorep-6.0-python3.6.6-foss-2019a --enable-shared
+# scorep_binding_python git sha: 8d35a668b0a59480e4e7fd9f43ae50949d807241 
+# installed in virtual environment
+
+. ~/virtenv/Python-3.6.6-foss-2019a/bin/activate
+
+srun taskset -c 0 python benchmark.py

benchmark/test_1.py

@@ -0,0 +1,10 @@
+import sys
+
+result = 0
+iterations = int(sys.argv[1])
+iteration_list = list(range(iterations))
+
+for i in iteration_list:
+    result += 1
+
+assert(result == iterations)

benchmark/test_2.py

@@ -0,0 +1,20 @@
+'''
+Created on 04.10.2019
+
+@author: gocht
+'''
+import sys
+
+
+def add(val):
+    return val + 1
+
+
+result = 0
+iterations = int(sys.argv[1])
+iteration_list = list(range(iterations))
+
+for i in iteration_list:
+    result = add(result)
+
+assert(result == iterations)