Merge pull request #134 from AndreasGocht/master

Further update benchmarks and comparison.

Author: AndreasGocht

benchmark/bm_baseline.py

@@ -2,9 +2,8 @@
 
 result = 0
 iterations = int(sys.argv[1])
-iteration_list = list(range(iterations))
 
-for i in iteration_list:
+for i in range(iterations):
     result += 1
 
 assert(result == iterations)

benchmark/bm_simplefunc.py

@@ -1,8 +1,3 @@
-'''
-Created on 04.10.2019
-
-@author: gocht
-'''
 import sys
 
 
@@ -12,9 +7,8 @@ def add(val):
 
 result = 0
 iterations = int(sys.argv[1])
-iteration_list = list(range(iterations))
 
-for i in iteration_list:
+for i in range(iterations):
     result = add(result)
 
 assert(result == iterations)

benchmark/compare.py

@@ -1,7 +1,6 @@
 import argparse
 import pickle
 import numpy
-from scipy.stats import linregress
 
 parser = argparse.ArgumentParser(description='Compare two benchmarks.',
                                  formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -17,52 +16,52 @@
 with open(args.right, "rb") as f:
     right = pickle.load(f)
 
-if left.keys()!=right.keys():
+if left.keys() != right.keys():
     raise ValueError("Different Experiments")
 
 experiment = right.keys()
 
 for exp in experiment:
     print("\nExperiment: {}".format(exp))
-    if left[exp].keys()!=right[exp].keys():
+    if left[exp].keys() != right[exp].keys():
         raise ValueError("Different Instrumenters")
     instrumenters = left[exp].keys()
-    
+
     for inst in instrumenters:
         print("\n\tInstrumenter: {}".format(inst))
-        if left[exp][inst].keys()!=right[exp][inst].keys():
+        if left[exp][inst].keys() != right[exp][inst].keys():
             raise ValueError("Different Iterations")
         iterations = left[exp][inst].keys()
         Y_left = []
-        Y_right =  []
+        Y_right = []
         X = []
         for it in iterations:
             left_val = left[exp][inst][it]
             right_val = right[exp][inst][it]
-            if len(left_val)!=len(right_val):
+            if len(left_val) != len(right_val):
                 raise ValueError("Different Repetitons")
 
-            Y_left.append(left_val)
-            Y_right.append(right_val)
-            X.append(numpy.full([len(left_val)], it))
+            Y_left.append(numpy.mean(left_val))
+            Y_right.append(numpy.mean(right_val))
+            X.append(numpy.full([1], it))
 
             if args.s:
                 print("\t\tInterations {}".format(it))
                 print("\t\tMean:     {:>7.4f} s  {:>7.4f} s".format(numpy.mean(left_val), numpy.mean(right_val)))
-                print("\t\tMedian:   {:>7.4f} s  {:>7.4f} s".format(numpy.quantile(left_val, 0.50), numpy.quantile(right_val, 0.50)))
-                print("\t\t5%:       {:>7.4f} s  {:>7.4f} s".format(numpy.quantile(left_val, 0.05), numpy.quantile(right_val, 0.05)))
-                print("\t\t95%:      {:>7.4f} s  {:>7.4f} s".format(numpy.quantile(left_val, 0.95), numpy.quantile(right_val, 0.95)))
-        Y_left = numpy.asarray(Y_left,dtype=float).flatten()
-        Y_right = numpy.asarray(Y_right,dtype=float).flatten()
-        X = numpy.asarray(X,dtype=float).flatten()
+                print("\t\tMedian:   {:>7.4f} s  {:>7.4f} s".format(
+                    numpy.quantile(left_val, 0.50), numpy.quantile(right_val, 0.50)))
+                print("\t\t5%:       {:>7.4f} s  {:>7.4f} s".format(
+                    numpy.quantile(left_val, 0.05), numpy.quantile(right_val, 0.05)))
+                print("\t\t95%:      {:>7.4f} s  {:>7.4f} s".format(
+                    numpy.quantile(left_val, 0.95), numpy.quantile(right_val, 0.95)))
+        Y_left = numpy.asarray(Y_left, dtype=float).flatten()
+        Y_right = numpy.asarray(Y_right, dtype=float).flatten()
+        X = numpy.asarray(X, dtype=float).flatten()
 
-        cost_left = linregress(X, Y_left)
-        cost_right = linregress(X, Y_right)
+        cost_left = numpy.polyfit(X, Y_left, 1)
+        cost_right = numpy.polyfit(X, Y_right, 1)
 
         if args.s:
             print("")
-        print("\tSlope     {:>7.4f} ns {:>7.4f} ns".format(cost_left.slope * 1e6, cost_right.slope* 1e6))
-        print("\tIntercept {:>7.4f} s  {:>7.4f} s".format(cost_left.intercept, cost_right.intercept))
-            
-        
-    
+        print("\tSlope     {:>7.4f} us {:>7.4f} us".format(cost_left[0] * 1e6, cost_right[0] * 1e6))
+        print("\tIntercept {:>7.4f} s  {:>7.4f} s".format(cost_left[1], cost_right[1]))

benchmark/compare_commits.sh

@@ -12,7 +12,7 @@ function benchmark_branch {
 	head=`git rev-parse --short HEAD`
 	pip install .
 	cd $wd
-	taskset -c 0 python benchmark.py -o result-$1-$head.pkl
+	python benchmark.py -o result-$1-$head.pkl
 }
 
 sleep 5

benchmark/run.sh

@@ -9,12 +9,17 @@
 #SBATCH --comment=no_monitoring
 #SBATCH --job-name benchmark_python
 
-module load Python/3.6.4-intel-2018a
-module unload intel
-module load intel
+module load Python/3.8.6-GCCcore-10.2.0
+module load Score-P/7.0-gompic-2020b
 
-export PATH=/scratch/rschoene/scorep-6-inst/bin:$PATH
+env_dir=~/virtenv/p-3.8.6-GCCcore-10.2.0-scorep-7.0-gompic-2020b/
 
-. ~/scorep_binding_python/test/bin/activate
+if [ ! -d $env_dir ]
+then
+	echo "Please create virtual env under: $env_dir"
+	exit -1
+fi
 
-srun python benchmark.py
+source $env_dir/bin/activate
+
+srun compare_commits.sh master faster