Trading metrics + equity curve in backtest report (ABI v2)

CLAUDE.md

@@ -32,6 +32,28 @@ If `scripts/run_corpus.sh` reports any parity drift or failures, investigate the
 - Run all unit tests: `ctest --test-dir build --output-on-failure`
 - Run single test executable: `./build/bin/test_integration`
 
+## SOP: adding a runtime `PF_API` export (CI gate — recurring failure)
+
+CI runs `python3 scripts/check_c_abi_runtime.py` after build+test. It pins the
+exact set of `PF_API` symbols implemented in `src/c_abi.cpp` against the
+hardcoded `EXPECTED_RUNTIME` frozenset in that script. Adding (or removing) a
+runtime export WITHOUT updating that list fails ALL CI matrix jobs at the
+"C ABI runtime source check" step, even though build and ctest are green.
+
+Checklist when touching runtime exports — update ALL of these together:
+
+1. `src/c_abi.cpp` — the implementation (and its file-header symbol comment).
+2. `include/pineforge/pineforge.h` — the `PF_API` declaration (+ doxygen).
+3. `scripts/check_c_abi_runtime.py` — add the symbol to `EXPECTED_RUNTIME`.
+4. Python ctypes harnesses if consumers must call it
+   (`scripts/run_strategy.py`, `tutorial/run*.py`, `docker/run_json.py`,
+   `benchmarks/throughput/grid_search_repro.py`).
+5. README symbol table if it enumerates exports.
+
+Before pushing: `python3 scripts/check_c_abi_runtime.py` (must exit 0).
+Per-strategy symbols (strategy_create, run_backtest, …) are NOT in this list —
+they are codegen-emitted; the checker enforces exactly that split.
+
 ## Code Style & Invariants
 
 - Modern C++17. Use of `<cstdint>` fixed-width types.

CMakeLists.txt

@@ -78,6 +78,7 @@ add_library(pineforge STATIC
     src/c_abi.cpp
     src/engine_fills.cpp
     src/engine_lower_tf.cpp
+    src/engine_metrics.cpp
     src/engine_orders.cpp
     src/engine_path_resolve.cpp
     src/engine_report.cpp

README.md

@@ -299,7 +299,7 @@ int main(void) {
 
 ## Public C ABI (the stability surface)
 
-`<pineforge/pineforge.h>` is the **single canonical consumer header**. Every compiled PineForge strategy `.so` exports exactly the 10 symbols declared there:
+`<pineforge/pineforge.h>` is the **single canonical consumer header**. Every compiled PineForge strategy `.so` exports the symbols declared there:
 
 
 | Symbol                                   | Role                                         |
@@ -314,9 +314,10 @@ int main(void) {
 | `strategy_set_magnifier_volume_weighted` | Toggle volume-weighted magnifier             |
 | `strategy_set_trace_enabled`             | Toggle per-bar trace recording               |
 | `pf_version_get`                         | Runtime version                              |
+| `pf_abi_version`                         | Struct-layout version (`PF_ABI_VERSION`)     |
 
 
-POD types (`pf_bar_t`, `pf_trade_t`, `pf_report_t`, `pf_security_diag_t`, `pf_trace_entry_t`, `pf_version_t`) and the `pf_magnifier_distribution_t` enum complete the surface.
+POD types (`pf_bar_t`, `pf_trade_t`, `pf_report_t`, `pf_security_diag_t`, `pf_trace_entry_t`, `pf_version_t`, and — since ABI v2 — `pf_trade_stats_t`, `pf_equity_stats_t`, `pf_metrics_t`, `pf_equity_point_t`) and the `pf_magnifier_distribution_t` enum complete the surface. ABI v2 (`PF_ABI_VERSION == 2`, exported as `pf_abi_version()`) appends computed trading metrics and a per-script-bar equity curve to `pf_report_t`; callers must verify the version before running since the report struct is caller-allocated.
 
 **Stability guarantee:** within the same `PINEFORGE_VERSION_MAJOR`, struct layouts and `extern "C"` signatures are append-only. New fields may be appended; existing fields are never reordered, removed, or retyped. New functions may be added; existing functions are never removed or signature-changed. Compile-time `static_assert`s in `src/c_abi.cpp` pin the layouts against drift.
 

benchmarks/throughput/grid_search_repro.py

@@ -26,8 +26,53 @@ class pf_trade_t(ctypes.Structure):
         ("max_runup", ctypes.c_double),
         ("max_drawdown", ctypes.c_double),
         ("qty", ctypes.c_double),
+        ("commission", ctypes.c_double),
+        ("entry_bar_index", ctypes.c_int32),
+        ("exit_bar_index", ctypes.c_int32),
     ]
 
+class pf_trade_stats_t(ctypes.Structure):
+    _fields_ = [
+        ("num_trades", ctypes.c_int32), ("num_wins", ctypes.c_int32),
+        ("num_losses", ctypes.c_int32), ("num_even", ctypes.c_int32),
+        ("percent_profitable", ctypes.c_double),
+        ("net_profit", ctypes.c_double), ("net_profit_pct", ctypes.c_double),
+        ("gross_profit", ctypes.c_double), ("gross_profit_pct", ctypes.c_double),
+        ("gross_loss", ctypes.c_double), ("gross_loss_pct", ctypes.c_double),
+        ("profit_factor", ctypes.c_double),
+        ("avg_trade", ctypes.c_double), ("avg_trade_pct", ctypes.c_double),
+        ("avg_win", ctypes.c_double), ("avg_win_pct", ctypes.c_double),
+        ("avg_loss", ctypes.c_double), ("avg_loss_pct", ctypes.c_double),
+        ("ratio_avg_win_avg_loss", ctypes.c_double),
+        ("largest_win", ctypes.c_double), ("largest_win_pct", ctypes.c_double),
+        ("largest_loss", ctypes.c_double), ("largest_loss_pct", ctypes.c_double),
+        ("commission_paid", ctypes.c_double),
+        ("expectancy", ctypes.c_double),
+        ("max_consecutive_wins", ctypes.c_int32), ("max_consecutive_losses", ctypes.c_int32),
+        ("avg_bars_in_trade", ctypes.c_double), ("avg_bars_in_wins", ctypes.c_double),
+        ("avg_bars_in_losses", ctypes.c_double),
+    ]
+
+class pf_equity_stats_t(ctypes.Structure):
+    _fields_ = [
+        ("max_equity_drawdown", ctypes.c_double), ("max_equity_drawdown_pct", ctypes.c_double),
+        ("max_equity_runup", ctypes.c_double), ("max_equity_runup_pct", ctypes.c_double),
+        ("buy_hold_return", ctypes.c_double), ("buy_hold_return_pct", ctypes.c_double),
+        ("sharpe_tv", ctypes.c_double), ("sortino_tv", ctypes.c_double),
+        ("sharpe_bar", ctypes.c_double), ("sortino_bar", ctypes.c_double),
+        ("cagr", ctypes.c_double), ("calmar", ctypes.c_double),
+        ("recovery_factor", ctypes.c_double), ("time_in_market_pct", ctypes.c_double),
+        ("open_pl", ctypes.c_double),
+    ]
+
+class pf_metrics_t(ctypes.Structure):
+    _fields_ = [("all", pf_trade_stats_t), ("longs", pf_trade_stats_t),
+                ("shorts", pf_trade_stats_t), ("equity", pf_equity_stats_t)]
+
+class pf_equity_point_t(ctypes.Structure):
+    _fields_ = [("time_ms", ctypes.c_int64), ("equity", ctypes.c_double),
+                ("open_profit", ctypes.c_double)]
+
 class pf_security_diag_t(ctypes.Structure):
     _fields_ = [
         ("sec_id", ctypes.c_int),
@@ -74,8 +119,29 @@ class pf_report_t(ctypes.Structure):
         ("trace_len", ctypes.c_int),
         ("trace_names", ctypes.POINTER(ctypes.c_char_p)),
         ("trace_names_len", ctypes.c_int),
+
+        ("metrics", pf_metrics_t),
+        ("equity_curve", ctypes.POINTER(pf_equity_point_t)),
+        ("equity_curve_len", ctypes.c_int64),  # int64, NOT c_int
     ]
 
+# pf_report_t is caller-allocated; a layout mismatch means the runtime
+# writes past this script's report buffer. Verify the ABI before running.
+EXPECTED_PF_ABI = 2
+
+def check_abi(lib):
+    try:
+        lib.pf_abi_version.restype = ctypes.c_int
+        abi = lib.pf_abi_version()
+    except AttributeError:
+        raise RuntimeError(
+            "strategy library predates pf_abi_version (ABI v1); rebuild it "
+            "against the current pineforge runtime (pf_report_t grew).")
+    if abi != EXPECTED_PF_ABI:
+        raise RuntimeError(
+            f"pineforge ABI mismatch: library reports {abi}, harness expects "
+            f"{EXPECTED_PF_ABI}; rebuild.")
+
 def main():
     repo_root = Path(__file__).parent.parent.parent.resolve()
     dylib_path = repo_root / "benchmarks/assets/strategies/19-scalping-wunder-bots/strategy.dylib"
@@ -96,6 +162,7 @@ def main():
 
     # Load shared library
     lib = ctypes.CDLL(str(dylib_path))
+    check_abi(lib)
 
     # Declare functions
     lib.strategy_create.argtypes = [ctypes.c_char_p]

docker/run_json.py

@@ -81,9 +81,62 @@ class TradeC(ctypes.Structure):
         ("max_runup",    ctypes.c_double),
         ("max_drawdown", ctypes.c_double),
         ("qty",          ctypes.c_double),
+        ("commission",      ctypes.c_double),
+        ("entry_bar_index", ctypes.c_int32),
+        ("exit_bar_index",  ctypes.c_int32),
     ]
 
 
+class TradeStatsC(ctypes.Structure):
+    """Mirror of pf_trade_stats_t (ABI v2)."""
+    _fields_ = [
+        ("num_trades", ctypes.c_int32), ("num_wins", ctypes.c_int32),
+        ("num_losses", ctypes.c_int32), ("num_even", ctypes.c_int32),
+        ("percent_profitable", ctypes.c_double),
+        ("net_profit", ctypes.c_double), ("net_profit_pct", ctypes.c_double),
+        ("gross_profit", ctypes.c_double), ("gross_profit_pct", ctypes.c_double),
+        ("gross_loss", ctypes.c_double), ("gross_loss_pct", ctypes.c_double),
+        ("profit_factor", ctypes.c_double),
+        ("avg_trade", ctypes.c_double), ("avg_trade_pct", ctypes.c_double),
+        ("avg_win", ctypes.c_double), ("avg_win_pct", ctypes.c_double),
+        ("avg_loss", ctypes.c_double), ("avg_loss_pct", ctypes.c_double),
+        ("ratio_avg_win_avg_loss", ctypes.c_double),
+        ("largest_win", ctypes.c_double), ("largest_win_pct", ctypes.c_double),
+        ("largest_loss", ctypes.c_double), ("largest_loss_pct", ctypes.c_double),
+        ("commission_paid", ctypes.c_double),
+        ("expectancy", ctypes.c_double),
+        ("max_consecutive_wins", ctypes.c_int32), ("max_consecutive_losses", ctypes.c_int32),
+        ("avg_bars_in_trade", ctypes.c_double), ("avg_bars_in_wins", ctypes.c_double),
+        ("avg_bars_in_losses", ctypes.c_double),
+    ]
+
+
+class EquityStatsC(ctypes.Structure):
+    """Mirror of pf_equity_stats_t (ABI v2)."""
+    _fields_ = [
+        ("max_equity_drawdown", ctypes.c_double), ("max_equity_drawdown_pct", ctypes.c_double),
+        ("max_equity_runup", ctypes.c_double), ("max_equity_runup_pct", ctypes.c_double),
+        ("buy_hold_return", ctypes.c_double), ("buy_hold_return_pct", ctypes.c_double),
+        ("sharpe_tv", ctypes.c_double), ("sortino_tv", ctypes.c_double),
+        ("sharpe_bar", ctypes.c_double), ("sortino_bar", ctypes.c_double),
+        ("cagr", ctypes.c_double), ("calmar", ctypes.c_double),
+        ("recovery_factor", ctypes.c_double), ("time_in_market_pct", ctypes.c_double),
+        ("open_pl", ctypes.c_double),
+    ]
+
+
+class MetricsC(ctypes.Structure):
+    """Mirror of pf_metrics_t (ABI v2)."""
+    _fields_ = [("all", TradeStatsC), ("longs", TradeStatsC),
+                ("shorts", TradeStatsC), ("equity", EquityStatsC)]
+
+
+class EquityPointC(ctypes.Structure):
+    """Mirror of pf_equity_point_t (ABI v2)."""
+    _fields_ = [("time_ms", ctypes.c_int64), ("equity", ctypes.c_double),
+                ("open_profit", ctypes.c_double)]
+
+
 class SecurityDiagC(ctypes.Structure):
     _fields_ = [
         ("sec_id",              ctypes.c_int),
@@ -126,9 +179,31 @@ class ReportC(ctypes.Structure):
         ("trace_len",                    ctypes.c_int),
         ("trace_names",                  ctypes.POINTER(ctypes.c_char_p)),
         ("trace_names_len",              ctypes.c_int),
+        ("metrics",                      MetricsC),
+        ("equity_curve",                 ctypes.POINTER(EquityPointC)),
+        ("equity_curve_len",             ctypes.c_int64),  # int64, NOT c_int
     ]
 
 
+# pf_report_t is CALLER-allocated: a .so built against a different ABI
+# writes past (or short of) our ReportC buffer. Assert version up front.
+EXPECTED_PF_ABI = 2
+
+
+def check_abi(lib: ctypes.CDLL) -> None:
+    try:
+        lib.pf_abi_version.restype = ctypes.c_int
+        abi = lib.pf_abi_version()
+    except AttributeError:
+        raise RuntimeError(
+            "strategy .so predates pf_abi_version (ABI v1); rebuild it against "
+            "the current pineforge runtime (pf_report_t grew).")
+    if abi != EXPECTED_PF_ABI:
+        raise RuntimeError(
+            f"pineforge ABI mismatch: .so reports {abi}, harness expects "
+            f"{EXPECTED_PF_ABI}; rebuild.")
+
+
 # --- helpers ----------------------------------------------------------
 
 def load_bars(csv_path: Path) -> tuple[ctypes.Array, int]:
@@ -158,6 +233,7 @@ def load_bars(csv_path: Path) -> tuple[ctypes.Array, int]:
 
 def load_strategy(so_path: Path) -> ctypes.CDLL:
     lib = ctypes.CDLL(str(so_path))
+    check_abi(lib)
 
     lib.strategy_create.argtypes = [ctypes.c_char_p]
     lib.strategy_create.restype  = ctypes.c_void_p

docs/pages/examples-rust.md

@@ -56,6 +56,59 @@ struct PfTrade {
     is_long: c_int,
     max_runup: f64, max_drawdown: f64,
     qty: f64,
+    // ABI v2
+    commission: f64,
+    entry_bar_index: i32,
+    exit_bar_index: i32,
+}
+
+// ── ABI v2 metrics PODs ──────────────────────────────────────────────
+
+#[repr(C)]
+#[derive(Clone, Copy, Default)]
+struct PfTradeStats {
+    num_trades: i32, num_wins: i32, num_losses: i32, num_even: i32,
+    percent_profitable: f64,
+    net_profit: f64, net_profit_pct: f64,
+    gross_profit: f64, gross_profit_pct: f64,
+    gross_loss: f64, gross_loss_pct: f64,
+    profit_factor: f64,
+    avg_trade: f64, avg_trade_pct: f64,
+    avg_win: f64, avg_win_pct: f64,
+    avg_loss: f64, avg_loss_pct: f64,
+    ratio_avg_win_avg_loss: f64,
+    largest_win: f64, largest_win_pct: f64,
+    largest_loss: f64, largest_loss_pct: f64,
+    commission_paid: f64,
+    expectancy: f64,
+    max_consecutive_wins: i32, max_consecutive_losses: i32,
+    avg_bars_in_trade: f64, avg_bars_in_wins: f64, avg_bars_in_losses: f64,
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Default)]
+struct PfEquityStats {
+    max_equity_drawdown: f64, max_equity_drawdown_pct: f64,
+    max_equity_runup: f64, max_equity_runup_pct: f64,
+    buy_hold_return: f64, buy_hold_return_pct: f64,
+    sharpe_tv: f64, sortino_tv: f64,
+    sharpe_bar: f64, sortino_bar: f64,
+    cagr: f64, calmar: f64,
+    recovery_factor: f64, time_in_market_pct: f64,
+    open_pl: f64,
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Default)]
+struct PfMetrics {
+    all: PfTradeStats, longs: PfTradeStats, shorts: PfTradeStats,
+    equity: PfEquityStats,
+}
+
+#[repr(C)]
+#[derive(Clone, Copy, Default)]
+struct PfEquityPoint {
+    time_ms: i64, equity: f64, open_profit: f64,
 }
 
 #[repr(C)]
@@ -102,10 +155,19 @@ struct PfReport {
     trace_len: c_int,
     trace_names: *mut *const c_char,
     trace_names_len: c_int,
+
+    // ABI v2: computed metrics + per-script-bar equity curve
+    metrics: PfMetrics,
+    equity_curve: *mut PfEquityPoint,
+    equity_curve_len: i64,
 }
 
 const PF_MAGNIFIER_ENDPOINTS: c_int = 3;
 
+// PfReport is CALLER-allocated: before any run, resolve `pf_abi_version`
+// via libloading and assert it returns 2 (PF_ABI_VERSION) — an old .so
+// writing into this larger struct (or vice versa) corrupts memory silently.
+
 // ── Safe wrapper ──────────────────────────────────────────────────────
 
 struct StrategyLib {