Skip to content

Commit d4f17a2

Browse files
felsenhowerfelsenhower
committed
Update benchmark code
1 parent f10845e commit d4f17a2

5 files changed

Lines changed: 301 additions & 187 deletions

File tree

‎benchmark/benchmark.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env python3
2+
3+
import subprocess
4+
import sys
5+
import time
6+
from dataclasses import dataclass
7+
from pathlib import Path
8+
9+
GAUSS_SEIDEL_ARGS: tuple[str, ...] = ("1", "1", "100", "2", "2", "100")
10+
JACOBI_ARGS: tuple[str, ...] = ("1", "2", "100", "2", "2", "100")
11+
12+
NUM_REPETITIONS = 3
13+
14+
OUTPUT_FILE = Path("benchmark_results.csv").resolve()
15+
16+
@dataclass(frozen=True)
17+
class Variant:
18+
path: Path
19+
commandline: tuple[str, ...]
20+
do_warmup: bool
21+
22+
23+
def format_row(
24+
variant: str,
25+
i: int,
26+
method: str,
27+
runtime_internal: float,
28+
runtime_total: float,
29+
) -> str:
30+
return (
31+
f"{variant:<12}, {i}, {method:<11},"
32+
f" {runtime_internal:>16.4f}, {runtime_total:>13.4f}"
33+
)
34+
35+
36+
def run_and_measure(
37+
cmd: tuple[str, ...],
38+
args: tuple[str, ...],
39+
cwd: Path,
40+
) -> tuple[float, float]:
41+
start = time.perf_counter()
42+
proc = subprocess.run(
43+
(*cmd, *args),
44+
cwd=cwd,
45+
stdout=subprocess.PIPE,
46+
stderr=subprocess.STDOUT,
47+
text=True,
48+
check=True,
49+
)
50+
end = time.perf_counter()
51+
runtime_total = end - start
52+
runtime_internal = float("nan")
53+
for line in proc.stdout.splitlines():
54+
if "Calculation time" in line:
55+
runtime_internal = float(line.split()[2])
56+
break
57+
return runtime_internal, runtime_total
58+
59+
60+
def status_line(variant: str, method: str, i: int) -> None:
61+
print(
62+
f'\rRunning (variant="{variant}", method="{method}", i={i})...',
63+
end="",
64+
flush=True,
65+
)
66+
67+
68+
def clear_status_line() -> None:
69+
print("\r" + " " * 80 + "\r", end="", flush=True)
70+
71+
72+
def main() -> None:
73+
if len(sys.argv) < 2:
74+
print("Error: Must supply path to reference implementation!")
75+
sys.exit(1)
76+
reference = Path(sys.argv[1]).resolve()
77+
variants: dict[str, Variant] = {
78+
"reference": Variant(
79+
path=reference.parent,
80+
commandline=(str(reference),),
81+
do_warmup=False,
82+
),
83+
"simple": Variant(
84+
path=Path("../simple"),
85+
commandline=("uv", "run", "--python", "cpython3.13", "main.py"),
86+
do_warmup=False,
87+
),
88+
"nuitka": Variant(
89+
path=Path("../nuitka"),
90+
commandline=("./partdiff",),
91+
do_warmup=False,
92+
),
93+
"np_vectorize": Variant(
94+
path=Path("../np_vectorize"),
95+
commandline=("uv", "run", "--python", "cpython3.13", "main.py"),
96+
do_warmup=False,
97+
),
98+
"numba": Variant(
99+
path=Path("../numba"),
100+
commandline=("uv", "run", "--python", "cpython3.10", "main.py"),
101+
do_warmup=True,
102+
),
103+
"cython": Variant(
104+
path=Path("../cython"),
105+
commandline=("uv", "run", "--python", "cpython3.13", "main.py"),
106+
do_warmup=False,
107+
),
108+
}
109+
if "nuitka" in variants.keys():
110+
nuitka_bin = variants["nuitka"].path / "partdiff"
111+
if not nuitka_bin.is_file():
112+
raise RuntimeError(
113+
"Must build nuitka variant before running benchmark "
114+
"or disable nuitka benchmark."
115+
)
116+
print(
117+
f"{'variant':<12}, i, {'method':<11}, {'runtime_internal':<16}, {'runtime_total':<13}"
118+
)
119+
with OUTPUT_FILE.open("w", encoding="utf-8") as f:
120+
f.write(
121+
f"{'variant':<12}, i, {'method':<11}, {'runtime_internal':<16}, {'runtime_total':<13}\n"
122+
)
123+
for variant_name, variant in variants.items():
124+
cwd = variant.path
125+
start_i = 0 if variant.do_warmup else 1
126+
for i in range(start_i, NUM_REPETITIONS + 1):
127+
for method, args in (
128+
("Gauß-Seidel", GAUSS_SEIDEL_ARGS),
129+
("Jacobi", JACOBI_ARGS),
130+
):
131+
status_line(variant_name, method, i)
132+
runtime_internal, runtime_total = run_and_measure(
133+
variant.commandline,
134+
args,
135+
cwd,
136+
)
137+
clear_status_line()
138+
line = format_row(
139+
variant_name,
140+
i,
141+
method,
142+
runtime_internal,
143+
runtime_total,
144+
)
145+
print(line)
146+
f.write(line + "\n")
147+
148+
149+
if __name__ == "__main__":
150+
main()

‎benchmark/benchmark.sh‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎benchmark/benchmark_results.csv‎

Lines changed: 38 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,39 @@
11
variant , i, method , runtime_internal, runtime_total
2-
reference , 1, Gauß-Seidel, 0.55 , 0.55
3-
reference , 1, Jacobi , 0.48 , 0.48
4-
reference , 2, Gauß-Seidel, 0.59 , 0.60
5-
reference , 2, Jacobi , 0.48 , 0.48
6-
reference , 3, Gauß-Seidel, 0.55 , 0.55
7-
reference , 3, Jacobi , 0.51 , 0.52
8-
simple , 0, Gauß-Seidel, 52.43 , 52.72
9-
simple , 0, Jacobi , 52.42 , 52.72
10-
simple , 1, Gauß-Seidel, 52.12 , 52.41
11-
simple , 1, Jacobi , 52.67 , 52.97
12-
simple , 2, Gauß-Seidel, 51.59 , 51.88
13-
simple , 2, Jacobi , 51.71 , 51.00
14-
simple , 3, Gauß-Seidel, 51.74 , 52.03
15-
simple , 3, Jacobi , 52.48 , 52.78
16-
numba , 0, Gauß-Seidel, 0.69 , 1.12
17-
numba , 0, Jacobi , 0.41 , 0.84
18-
numba , 1, Gauß-Seidel, 0.69 , 1.14
19-
numba , 1, Jacobi , 0.42 , 0.86
20-
numba , 2, Gauß-Seidel, 0.69 , 1.14
21-
numba , 2, Jacobi , 0.41 , 0.85
22-
numba , 3, Gauß-Seidel, 0.73 , 1.17
23-
numba , 3, Jacobi , 0.42 , 0.87
24-
np_vectorize, 0, Gauß-Seidel, 51.49 , 51.77
25-
np_vectorize, 0, Jacobi , 0.21 , 0.50
26-
np_vectorize, 1, Gauß-Seidel, 55.31 , 55.60
27-
np_vectorize, 1, Jacobi , 0.21 , 0.49
28-
np_vectorize, 2, Gauß-Seidel, 55.39 , 55.68
29-
np_vectorize, 2, Jacobi , 0.21 , 0.49
30-
np_vectorize, 3, Gauß-Seidel, 54.83 , 55.12
31-
np_vectorize, 3, Jacobi , 0.22 , 0.51
2+
reference , 1, Gauß-Seidel, 0.5572, 0.5591
3+
reference , 1, Jacobi , 0.4808, 0.4826
4+
reference , 2, Gauß-Seidel, 0.5609, 0.5621
5+
reference , 2, Jacobi , 0.5448, 0.5466
6+
reference , 3, Gauß-Seidel, 0.5549, 0.5561
7+
reference , 3, Jacobi , 0.4832, 0.4850
8+
simple , 1, Gauß-Seidel, 55.5526, 55.8415
9+
simple , 1, Jacobi , 52.4647, 52.7504
10+
simple , 2, Gauß-Seidel, 52.7245, 53.0375
11+
simple , 2, Jacobi , 55.2271, 55.5206
12+
simple , 3, Gauß-Seidel, 53.0670, 53.3685
13+
simple , 3, Jacobi , 52.5683, 52.8698
14+
nuitka , 1, Gauß-Seidel, 50.0649, 50.3533
15+
nuitka , 1, Jacobi , 49.5409, 49.8102
16+
nuitka , 2, Gauß-Seidel, 49.6525, 49.9396
17+
nuitka , 2, Jacobi , 48.9229, 49.1859
18+
nuitka , 3, Gauß-Seidel, 50.6343, 50.8995
19+
nuitka , 3, Jacobi , 48.1602, 48.4233
20+
np_vectorize, 1, Gauß-Seidel, 51.6331, 51.9141
21+
np_vectorize, 1, Jacobi , 0.2080, 0.4897
22+
np_vectorize, 2, Gauß-Seidel, 51.7979, 52.0769
23+
np_vectorize, 2, Jacobi , 0.2162, 0.4970
24+
np_vectorize, 3, Gauß-Seidel, 52.4517, 52.7541
25+
np_vectorize, 3, Jacobi , 0.2364, 0.5373
26+
numba , 0, Gauß-Seidel, 0.6998, 1.1518
27+
numba , 0, Jacobi , 0.4169, 0.8607
28+
numba , 1, Gauß-Seidel, 0.6930, 1.1534
29+
numba , 1, Jacobi , 0.4122, 0.8647
30+
numba , 2, Gauß-Seidel, 0.6843, 1.1082
31+
numba , 2, Jacobi , 0.4065, 0.8361
32+
numba , 3, Gauß-Seidel, 0.6897, 1.1155
33+
numba , 3, Jacobi , 0.4019, 0.8296
34+
cython , 1, Gauß-Seidel, 0.2966, 0.5764
35+
cython , 1, Jacobi , 0.0612, 0.3399
36+
cython , 2, Gauß-Seidel, 0.2957, 0.5770
37+
cython , 2, Jacobi , 0.0600, 0.3432
38+
cython , 3, Gauß-Seidel, 0.2961, 0.5751
39+
cython , 3, Jacobi , 0.0582, 0.3375

‎benchmark/main.py‎

Lines changed: 0 additions & 95 deletions
This file was deleted.

0 commit comments

Comments
 (0)