SPMI: per-context tpdiff examples

src/coreclr/scripts/superpmi.py

@@ -2006,6 +2006,11 @@ def aggregate_diff_metrics(details_file):
     diffs_fields = ["Context", "Method full name", "Context size", "Base ActualCodeBytes", "Diff ActualCodeBytes", "Base PerfScore", "Diff PerfScore"]
     diffs = []
 
+    # Per-context throughput diffs (rows where PIN measured base != diff
+    # instruction count). Used by tpdiff to surface specific method examples.
+    tp_diffs_fields = ["Context", "MinOpts", "Base instructions", "Diff instructions"]
+    tp_diffs = []
+
     for row in read_csv(details_file):
         base_result = row["Base result"]
 
@@ -2044,6 +2049,9 @@ def aggregate_diff_metrics(details_file):
             base_dict["Diff executed instructions"] += base_insts
             diff_dict["Diff executed instructions"] += diff_insts
 
+            if base_insts > 0 and base_insts != diff_insts:
+                tp_diffs.append({f: row[f] for f in tp_diffs_fields})
+
             base_perfscore = float(row["Base PerfScore"])
             diff_perfscore = float(row["Diff PerfScore"])
             base_dict["Diffed PerfScore"] += base_perfscore
@@ -2083,7 +2091,8 @@ def aggregate_diff_metrics(details_file):
 
     return ({"Overall": base_overall, "MinOpts": base_minopts, "FullOpts": base_fullopts},
             {"Overall": diff_overall, "MinOpts": diff_minopts, "FullOpts": diff_fullopts},
-            diffs)
+            diffs,
+            tp_diffs)
 
 
 class SuperPMIReplayAsmDiffs:
@@ -2323,7 +2332,7 @@ def replay_with_asm_diffs(self):
 
                 print_superpmi_error_result(return_code, self.coreclr_args)
 
-                (base_metrics, diff_metrics, diffs) = aggregate_diff_metrics(details_info_file)
+                (base_metrics, diff_metrics, diffs, _) = aggregate_diff_metrics(details_info_file)
                 print_superpmi_success_result(return_code, base_metrics, diff_metrics)
 
                 artifacts_base_name = create_artifacts_base_name(self.coreclr_args, mch_file)
@@ -3193,7 +3202,7 @@ def replay_with_throughput_diff(self):
 
                 print_superpmi_error_result(return_code, self.coreclr_args)
 
-                (base_metrics, diff_metrics, _) = aggregate_diff_metrics(details_info_file)
+                (base_metrics, diff_metrics, _, tp_per_context) = aggregate_diff_metrics(details_info_file)
                 print_superpmi_success_result(return_code, base_metrics, diff_metrics)
 
                 if base_metrics is not None and diff_metrics is not None:
@@ -3205,7 +3214,7 @@ def replay_with_throughput_diff(self):
                     if base_instructions != 0 and diff_instructions != 0:
                         delta_instructions = diff_instructions - base_instructions
                         logging.info("Total instructions executed delta: {} ({:.2%} of base)".format(delta_instructions, delta_instructions / base_instructions))
-                        tp_diffs.append((os.path.basename(mch_file), base_metrics, diff_metrics))
+                        tp_diffs.append((os.path.basename(mch_file), base_metrics, diff_metrics, tp_per_context))
                     else:
                         logging.warning("One compilation failed to produce any results")
                 else:
@@ -3291,12 +3300,12 @@ def is_significant_pct(base, diff):
     def is_significant(row, base, diff):
         return is_significant_pct(base[row]["Diff executed instructions"], diff[row]["Diff executed instructions"])
 
-    if any(is_significant(row, base, diff) for row in ["Overall", "MinOpts", "FullOpts"] for (_, base, diff) in tp_diffs):
+    if any(is_significant(row, base, diff) for row in ["Overall", "MinOpts", "FullOpts"] for (_, base, diff, _) in tp_diffs):
         def write_pivot_section(row):
-            if not any(is_significant(row, base, diff) for (_, base, diff) in tp_diffs):
+            if not any(is_significant(row, base, diff) for (_, base, diff, _) in tp_diffs):
                 return
 
-            pcts = [compute_pct(base_metrics[row]["Diff executed instructions"], diff_metrics[row]["Diff executed instructions"]) for (_, base_metrics, diff_metrics) in tp_diffs]
+            pcts = [compute_pct(base_metrics[row]["Diff executed instructions"], diff_metrics[row]["Diff executed instructions"]) for (_, base_metrics, diff_metrics, _) in tp_diffs]
             min_pct_str = format_pct(min(pcts))
             max_pct_str = format_pct(max(pcts))
             if min_pct_str == max_pct_str:
@@ -3307,7 +3316,7 @@ def write_pivot_section(row):
             with DetailsSection(write_fh, tp_summary):
                 write_fh.write("|Collection|PDIFF|\n")
                 write_fh.write("|---|--:|\n")
-                for mch_file, base, diff in tp_diffs:
+                for mch_file, base, diff, _ in tp_diffs:
                     base_instructions = base[row]["Diff executed instructions"]
                     diff_instructions = diff[row]["Diff executed instructions"]
 
@@ -3320,6 +3329,8 @@ def write_pivot_section(row):
         write_pivot_section("Overall")
         write_pivot_section("MinOpts")
         write_pivot_section("FullOpts")
+        if include_details:
+            write_tpdiff_context_examples(write_fh, tp_diffs)
     elif include_details:
         write_top_context_section()
         write_fh.write("No significant throughput differences found\n")
@@ -3330,14 +3341,83 @@ def write_pivot_section(row):
                 write_fh.write("{} contexts:\n\n".format(disp))
                 write_fh.write("|Collection|Base # instructions|Diff # instructions|PDIFF|\n")
                 write_fh.write("|---|--:|--:|--:|\n")
-                for mch_file, base, diff in tp_diffs:
+                for mch_file, base, diff, _ in tp_diffs:
                     base_instructions = base[row]["Diff executed instructions"]
                     diff_instructions = diff[row]["Diff executed instructions"]
                     write_fh.write("|{}|{:,d}|{:,d}|{}|\n".format(
                         mch_file, base_instructions, diff_instructions,
                         compute_and_format_pct(base_instructions, diff_instructions)))
                 write_fh.write("\n")
 
+
+def write_tpdiff_context_examples(write_fh, tp_diffs):
+    """ Write top per-context throughput regression/improvement examples.
+
+    Args:
+        write_fh : file handle for file to output to
+        tp_diffs : list of (mch_file, base_metrics, diff_metrics, tp_per_context)
+                   where tp_per_context is a list of dicts with keys
+                   "Context", "MinOpts", "Base instructions", "Diff instructions".
+    """
+
+    # Flatten per-context rows; tag each with its originating collection.
+    flat = []
+    for (mch_file, _, _, tp_per_context) in tp_diffs:
+        for row in tp_per_context:
+            base_insts = int(row["Base instructions"])
+            diff_insts = int(row["Diff instructions"])
+            pct = (diff_insts - base_insts) / base_insts * 100
+            flat.append({
+                "Collection": mch_file,
+                "Context": row["Context"],
+                "MinOpts": row["MinOpts"] == "True",
+                "Base instructions": base_insts,
+                "Diff instructions": diff_insts,
+                "PDIFF pct": pct,
+            })
+
+    if not flat:
+        return
+
+    # Suppress tiny absolute deltas that show up as big percentages but are noise.
+    MIN_ABS_DELTA = 50
+    significant = [r for r in flat if abs(r["Diff instructions"] - r["Base instructions"]) >= MIN_ABS_DELTA]
+
+    if not significant:
+        return
+
+    def write_examples(title, rows):
+        if not rows:
+            return
+        with DetailsSection(write_fh, title):
+            write_fh.write("|Collection|Context|Base|Diff|PDIFF|\n")
+            write_fh.write("|---|--:|--:|--:|--:|\n")
+            for r in rows:
+                write_fh.write("|{}|{}|{:,d}|{:,d}|{}|\n".format(
+                    r["Collection"],
+                    r["Context"],
+                    r["Base instructions"],
+                    r["Diff instructions"],
+                    compute_and_format_pct(r["Base instructions"], r["Diff instructions"])))
+
+    TOP_N = 20
+
+    def split_and_emit(label, rows):
+        regressions = sorted([r for r in rows if r["PDIFF pct"] > 0],
+                             key=lambda r: r["PDIFF pct"], reverse=True)[:TOP_N]
+        improvements = sorted([r for r in rows if r["PDIFF pct"] < 0],
+                              key=lambda r: r["PDIFF pct"])[:TOP_N]
+        write_examples("Top method regressions ({}, by PDIFF %)".format(label), regressions)
+        write_examples("Top method improvements ({}, by PDIFF %)".format(label), improvements)
+
+    fullopts = [r for r in significant if not r["MinOpts"]]
+    minopts = [r for r in significant if r["MinOpts"]]
+
+    if fullopts:
+        split_and_emit("FullOpts", fullopts)
+    if minopts:
+        split_and_emit("MinOpts", minopts)
+
 ################################################################################
 # SuperPMI Metric Diff
 ################################################################################