diff --git a/scripts/engine-bench-report.py b/scripts/engine-bench-report.py index a38579c..698f54d 100755 --- a/scripts/engine-bench-report.py +++ b/scripts/engine-bench-report.py @@ -6,16 +6,16 @@ engine-bench-report.py [--baseline ] [--markdown ] Reads `combined_latency.csv` from `results_dir`, aggregates per-block data into -headline throughput and latency metrics, uses `summary.csv` existence only as -a benchmark-completion marker, resolves a report status +headline throughput and latency metrics, uses a non-empty `summary.csv` as a +benchmark-completion marker, resolves a report status (`normal`/`partial`/`no_data`/`error`), and prints a JSON analysis document to stdout. Callers (e.g., CI) parse the JSON directly. With `--baseline `, averages the headline metrics across prior-run complete prior runs under `//{summary,combined_latency}.csv`, -where `summary.csv` is only a completion marker, and emits a Δ table alongside -current values, flagging regressions against the trailing average. Incomplete -baseline runs are skipped. +where a non-empty `summary.csv` is the completion marker, and emits a Δ table +alongside current values, flagging regressions against the trailing average. +Incomplete baseline runs are skipped. With `--markdown `, also renders a markdown report at that path. @@ -180,8 +180,12 @@ def _load_combined_rows(path): def summary_marker_status(results_dir): path = os.path.join(results_dir, "summary.csv") - if not os.path.exists(path): + try: + size = os.path.getsize(path) + except FileNotFoundError: return SUMMARY_MARKER_MISSING + if size == 0: + return SUMMARY_MARKER_EMPTY return SUMMARY_MARKER_PRESENT diff --git a/scripts/test_engine_bench_report.py b/scripts/test_engine_bench_report.py new file mode 100644 index 0000000..d648ceb --- /dev/null +++ b/scripts/test_engine_bench_report.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Unit tests for engine-bench-report.py.""" + +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).with_name("engine-bench-report.py") +SPEC = importlib.util.spec_from_file_location("engine_bench_report", SCRIPT_PATH) +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) + + +class TestSummaryMarkerStatus(unittest.TestCase): + def test_missing_summary_is_incomplete(self): + with tempfile.TemporaryDirectory() as results_dir: + self.assertEqual( + REPORT.summary_marker_status(results_dir), + REPORT.SUMMARY_MARKER_MISSING, + ) + + def test_empty_summary_is_incomplete(self): + with tempfile.TemporaryDirectory() as results_dir: + Path(results_dir, "summary.csv").touch() + + self.assertEqual( + REPORT.summary_marker_status(results_dir), + REPORT.SUMMARY_MARKER_EMPTY, + ) + + def test_non_empty_summary_is_complete(self): + with tempfile.TemporaryDirectory() as results_dir: + Path(results_dir, "summary.csv").write_text( + "block_number,new_payload_ms\n", + encoding="utf-8", + ) + + self.assertEqual( + REPORT.summary_marker_status(results_dir), + REPORT.SUMMARY_MARKER_PRESENT, + ) + + def test_empty_summary_makes_report_partial(self): + status = REPORT.resolve_report_status( + REPORT.SUMMARY_MARKER_EMPTY, + {"samples": 1}, + ) + + self.assertEqual(status, REPORT.REPORT_STATUS_PARTIAL) + + +if __name__ == "__main__": + unittest.main()