Update gemini outputs (#19)

* update gemini outputs

Author: Dando18

analysis/all-metrics.sh

@@ -1,41 +1,45 @@
 #!/bin/sh
 
 # main metrics
-python metrics.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics.csv
-python metrics.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics.csv
-python metrics.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics.csv
-python metrics.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics.csv
-python metrics.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics.csv
-python metrics.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics.csv
-python metrics.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 --output ../results/a8724ee8/gpt-4_temp0.2/metrics.csv
-(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics.csv > all-metrics.csv && tail -n+2 -q */metrics.csv >> all-metrics.csv)
+#python metrics.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics.csv
+#python metrics.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics.csv
+#python metrics.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics.csv
+#python metrics.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics.csv
+#python metrics.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics.csv
+#python metrics.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics.csv
+#python metrics.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 --output ../results/a8724ee8/gpt-4_temp0.2/metrics.csv
+python metrics.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro --output ../results/015cff6f/gemini-pro_temp0.2/metrics.csv
+#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics.csv > all-metrics.csv && tail -n+2 -q */metrics.csv >> all-metrics.csv)
 
 # mpi scaling metrics
-python metrics-scaling.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-mpi.csv
-python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-mpi.csv
-(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv > all-metrics-scaling-mpi.csv && tail -n+2 -q */metrics-scaling-mpi.csv >> all-metrics-scaling-mpi.csv)
+#python metrics-scaling.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-mpi.csv
+#python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-mpi.csv
+python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 64 128 256 512 --execution-model mpi --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-mpi.csv
+#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-mpi.csv > all-metrics-scaling-mpi.csv && tail -n+2 -q */metrics-scaling-mpi.csv >> all-metrics-scaling-mpi.csv)
 
 # omp scaling metrics
-python metrics-scaling.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-omp.csv
-python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-omp.csv
-(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv > all-metrics-scaling-omp.csv && tail -n+2 -q */metrics-scaling-omp.csv >> all-metrics-scaling-omp.csv)
+#python metrics-scaling.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-omp.csv
+#python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-omp.csv
+python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model omp --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-omp.csv
+#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-omp.csv > all-metrics-scaling-omp.csv && tail -n+2 -q */metrics-scaling-omp.csv >> all-metrics-scaling-omp.csv)
 
 # kokkos scaling metrics
-python metrics-scaling.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-kokkos.csv
-python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-kokkos.csv
-(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv > all-metrics-scaling-kokkos.csv && tail -n+2 -q */metrics-scaling-kokkos.csv >> all-metrics-scaling-kokkos.csv)
+#python metrics-scaling.py ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-7B -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-13B -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/codellama-13b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/results.csv --model-name CodeLlama-34B -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/codellama-34b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/a8724ee8/starcoderbase_prompted_temp0.2/results.csv --model-name StarCoderBase -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/starcoderbase_prompted_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/a8724ee8/phind-v2_prompted_temp0.2/results.csv --model-name Phind-V2 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/phind-v2_prompted_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/a8724ee8/gpt-3.5_temp0.2/results.csv --model-name GPT-3.5 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-3.5_temp0.2/metrics-scaling-kokkos.csv
+#python metrics-scaling.py ../results/a8724ee8/gpt-4_temp0.2/results.csv --model-name GPT-4 -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/a8724ee8/gpt-4_temp0.2/metrics-scaling-kokkos.csv
+python metrics-scaling.py ../results/015cff6f/gemini-pro_temp0.2/results.csv --model-name Gemini-Pro -k 1 -n 1 2 4 8 16 32 --execution-model kokkos --output ../results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv
+#(cd ../results/a8724ee8 && head -n 1 codellama-7b-hf_prompted_temp0.2/metrics-scaling-kokkos.csv > all-metrics-scaling-kokkos.csv && tail -n+2 -q */metrics-scaling-kokkos.csv >> all-metrics-scaling-kokkos.csv)

results/015cff6f/gemini-pro_temp0.2/metrics-scaling-kokkos.csv

@@ -0,0 +1,12 @@
+model,execution model,problem type,speedup_1@1,efficiency_1@1,speedup_2@1,efficiency_2@1,speedup_4@1,efficiency_4@1,speedup_8@1,efficiency_8@1,speedup_16@1,efficiency_16@1,speedup_32@1,efficiency_32@1
+Gemini-Pro,kokkos,dense_la,0.3728804835517466,0.3728804835517466,0.6658593830857491,0.33292969154287455,1.2287940975556226,0.30719852438890566,2.372933381345086,0.29661667266813574,4.489830610758599,0.2806144131724124,7.124512834220095,0.22264102606937797
+Gemini-Pro,kokkos,geometry,0.28536601282263874,0.28536601282263874,0.36550935051280653,0.18275467525640327,0.5410999153591143,0.13527497883977857,0.8884109732728145,0.11105137165910181,1.4742201844353322,0.09213876152720826,2.629226346206082,0.08216332331894007
+Gemini-Pro,kokkos,graph,1.3484469025925678,1.3484469025925678,1.5611602206200945,0.7805801103100473,1.983377838620041,0.49584445965501023,2.843560777123364,0.3554450971404205,4.545805609267464,0.2841128505792165,7.230174529124,0.225942954035125
+Gemini-Pro,kokkos,histogram,0.5630339767561345,0.5630339767561345,0.3826904996936389,0.19134524984681944,0.055551579435236365,0.013887894858809091,0.07127170734050486,0.008908963417563108,0.11100117802527563,0.006937573626579727,0.06705429051180761,0.002095446578493988
+Gemini-Pro,kokkos,reduce,2.023557079572052,2.023557079572052,4.043177947053675,2.0215889735268373,8.052347535271778,2.0130868838179445,16.05980363737436,2.007475454671795,30.89521219390383,1.9309507621189894,55.59769295461553,1.7374279048317354
+Gemini-Pro,kokkos,scan,1.1348251929514055,1.1348251929514055,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+Gemini-Pro,kokkos,search,0.31131564163802083,0.31131564163802083,0.5582958337133762,0.2791479168566881,1.1110932704142025,0.2777733176035506,2.2165947146255056,0.2770743393281882,4.444992022900709,0.2778120014312943,8.110571392134153,0.2534553560041923
+Gemini-Pro,kokkos,sort,0.18626160776282524,0.18626160776282524,0.18487166027266105,0.09243583013633053,0.18406868222716527,0.04601717055679132,0.1839374944597275,0.02299218680746594,0.18539858984560303,0.01158741186535019,0.18720973009796554,0.005850304065561423
+Gemini-Pro,kokkos,sparse_la,0.3576307622714652,0.3576307622714652,0.3512088610432395,0.17560443052161975,0.3576079495449554,0.08940198738623885,0.34617864188019143,0.04327233023502393,0.364041347946141,0.022752584246633813,0.3714777862921434,0.011608680821629481
+Gemini-Pro,kokkos,stencil,0.34680647061417297,0.34680647061417297,0.5619193622034502,0.2809596811017251,0.9908049746480665,0.24770124366201662,1.8453378033469943,0.23066722541837428,3.548249585073459,0.22176559906709117,6.9704366918710505,0.21782614662097033
+Gemini-Pro,kokkos,transform,1.2817385146631302,1.2817385146631302,2.5510015712268443,1.2755007856134222,5.035318607836976,1.258829651959244,10.020423330811553,1.2525529163514442,20.032522236512836,1.2520326397820523,39.9433212353503,1.2482287886046968