Merge pull request #429 from coding-kitties/fix/large-report-performance

fix: optimize backtest report for 200+ strategies

Author: MDUYN

investing_algorithm_framework/app/reporting/backtest_report.py

@@ -10,7 +10,7 @@
 from jinja2 import Environment, FileSystemLoader
 
 from investing_algorithm_framework.domain import (
-    Backtest, OperationalException
+    Backtest, OperationalException, tqdm
 )
 
 logger = logging.getLogger("investing_algorithm_framework")
@@ -88,6 +88,70 @@ def _is_na(val):
     return False
 
 
+def _downsample(series, max_points=300):
+    """Downsample a time-series list to at most *max_points* entries.
+
+    Uses the Largest-Triangle-Three-Buckets (LTTB) algorithm to
+    preserve visual fidelity while drastically reducing data size
+    for the browser.  Always keeps the first and last points.
+    """
+    if not series or len(series) <= max_points:
+        return series
+
+    n = len(series)
+    # Each element is [value, date_str]
+    sampled = [series[0]]
+    bucket_size = (n - 2) / (max_points - 2)
+
+    a_idx = 0
+    for i in range(1, max_points - 1):
+        # Calculate bucket boundaries
+        avg_start = int((i) * bucket_size) + 1
+        avg_end = int((i + 1) * bucket_size) + 1
+        if avg_end > n - 1:
+            avg_end = n - 1
+
+        # Average point of next bucket
+        avg_x = 0
+        avg_y = 0
+        count = avg_end - avg_start
+        if count <= 0:
+            count = 1
+            avg_end = avg_start + 1
+        for j in range(avg_start, avg_end):
+            avg_x += j
+            avg_y += series[j][0]
+        avg_x /= count
+        avg_y /= count
+
+        # Current bucket boundaries
+        range_start = int((i - 1) * bucket_size) + 1
+        range_end = int(i * bucket_size) + 1
+        if range_end > n - 1:
+            range_end = n - 1
+
+        # Pick the point with the largest triangle area
+        a_x = a_idx
+        a_y = series[a_idx][0]
+        max_area = -1
+        best_idx = range_start
+
+        for j in range(range_start, range_end):
+            area = abs(
+                (a_x - avg_x) * (series[j][0] - a_y)
+                - (a_x - j) * (avg_y - a_y)
+            ) * 0.5
+            if area > max_area:
+                max_area = area
+                best_idx = j
+
+        sampled.append(series[best_idx])
+        a_idx = best_idx
+
+    sampled.append(series[-1])
+    return sampled
+
+
 @dataclass
 class BacktestReport:
     backtests: List[Backtest] = field(default_factory=list)
@@ -162,6 +226,7 @@ def _is_backtest(backtest_path):
     def open(
         backtests: List[Backtest] = None,
         directory_path: Union[str, List[str], None] = None,
+        show_progress: bool = False,
     ) -> "BacktestReport":
         loaded = []
         source_tags = []
@@ -178,11 +243,12 @@ def open(
         else:
             dir_paths = []
 
+        # Collect all backtest paths first for progress tracking
+        backtest_paths = []
         for dp in dir_paths:
             tag = os.path.basename(os.path.normpath(dp))
             if BacktestReport._is_backtest(dp):
-                loaded.append(Backtest.open(dp))
-                source_tags.append(tag)
+                backtest_paths.append((dp, tag))
             else:
                 for root, dirs, _ in os.walk(dp):
                     for dir_name in dirs:
@@ -192,10 +258,20 @@ def open(
                         if BacktestReport._is_backtest(
                             subdir
                         ):
-                            loaded.append(
-                                Backtest.open(subdir)
+                            backtest_paths.append(
+                                (subdir, tag)
                             )
-                            source_tags.append(tag)
+
+        iterator = backtest_paths
+        if show_progress:
+            iterator = tqdm(
+                backtest_paths,
+                desc="Loading backtests",
+            )
+
+        for path, tag in iterator:
+            loaded.append(Backtest.open(path))
+            source_tags.append(tag)
 
         for bt in backtests:
             if not isinstance(bt, Backtest):
@@ -339,10 +415,10 @@ def _build_strategies_data(self):
                     initial = ec[0][0] if ec else 1
                     if initial == 0:
                         initial = 1
-                    rep_eq = [
+                    rep_eq = _downsample([
                         [(v / initial - 1) * 100, _fmt_date(d)]
                         for v, d in ec
-                    ]
+                    ])
 
             # Run IDs / mappings / labels
             run_ids, run_name_map, run_labels_list = [], {}, []
@@ -680,6 +756,11 @@ def _build_run_data(self):
                             ),
                         })
 
+                # Downsample heavy time series for browser perf
+                eq = _downsample(eq)
+                dd = _downsample(dd)
+                rs = _downsample(rs)
+
                 run_data[rid] = {
                     'label': label,
                     'EQ': eq,

investing_algorithm_framework/app/reporting/templates/dashboard.js

@@ -63,6 +63,92 @@ let selectedRunView = 'summary';
   });
 })();
 
+// ===== LAZY STRATEGY PAGE RENDERING =====
+var _renderedStratPages = {};
+
+function ensureStratPage(stratIdx) {
+  if (_renderedStratPages[stratIdx]) return;
+  _renderedStratPages[stratIdx] = true;
+
+  var s = STRATEGIES[stratIdx];
+  if (!s) return;
+  var sid = s.id;
+  var i = stratIdx;
+  var rl = s.runLabels || [];
+
+  var h = '<div class="page" id="page-' + sid + '" style="display:none">';
+  h += '<h2 class="page-title">';
+  h += '<span class="sb-dot" style="background:' + s.color + ';width:12px;height:12px"></span> ';
+  h += escHtmlLazy(s.name);
+  if (s.tag) h += ' <span class="tag-badge">' + escHtmlLazy(s.tag) + '</span>';
+  if (rl.length > 1) {
+    h += ' <select class="view-select strat-window-select" data-strat="' + i + '" onchange="onStratWindowChange(' + i + ', this.value)">';
+    h += '<option value="summary">&Sigma; All Windows</option>';
+    rl.forEach(function(pair) {
+      var rn = pair[0], lab = pair[1];
+      var mapped = s.runNameMap[rn] || '';
+      h += '<option value="' + mapped + '">' + escHtmlLazy(lab) + '</option>';
+    });
+    h += '</select>';
+  }
+  h += '</h2>';
+
+  h += '<div id="' + sid + '-summary-content"></div>';
+  h += '<div id="' + sid + '-parameters"></div>';
+
+  if (rl.length > 1) {
+    h += '<div class="chart-card collapsed"><div class="chart-title">Backtest Runs</div>';
+    h += '<div class="table-wrap"><table class="comp-table" id="strat-runs-table-' + i + '">';
+    h += '<thead><tr><th>Run</th><th>Return</th><th>Sharpe</th><th>Max DD</th><th>Trades</th></tr></thead><tbody>';
+    rl.forEach(function(pair) {
+      var rn = pair[0], lab = pair[1];
+      var mapped = s.runNameMap[rn] || '';
+      h += '<tr data-run="' + mapped + '"><td>' + escHtmlLazy(lab) + '</td><td></td><td></td><td></td><td></td></tr>';
+    });
+    h += '</tbody></table></div></div>';
+  }
+
+  h += '<div class="nav-tabs strat-nav-tabs" id="' + sid + '-tabs">';
+  h += '<div class="tab active" onclick="switchStratTab(\'' + sid + '\',\'performance\')">Performance</div>';
+  h += '<div class="tab" onclick="switchStratTab(\'' + sid + '\',\'trading\')">Trading</div>';
+  h += '</div>';
+
+  h += '<div class="tab-panel active" id="' + sid + '-performance">';
+  h += '<div id="' + sid + '-portfolio-summary"></div>';
+  h += '<div id="' + sid + '-trading-activity"></div>';
+
+  h += '<div class="chart-card"><div class="chart-title" id="eq-title-' + sid + '">Equity Curves (All Runs)</div>';
+  h += '<div class="chart-wrap"><canvas id="c-' + sid + '-eq"></canvas><div class="tooltip" id="tt-' + sid + '-eq"></div></div></div>';
+
+  h += '<div class="chart-card"><div class="chart-title" id="rs-title-' + sid + '">Rolling Sharpe Ratio (All Runs)</div>';
+  h += '<div class="chart-wrap"><canvas id="c-' + sid + '-rsharpe"></canvas><div class="tooltip" id="tt-' + sid + '-rsharpe"></div></div></div>';
+
+  h += '<div class="chart-card"><div class="chart-title" id="dd-title-' + sid + '">Drawdown (All Runs)</div>';
+  h += '<div class="chart-wrap"><canvas id="c-' + sid + '-dd"></canvas><div class="tooltip" id="tt-' + sid + '-dd"></div></div></div>';
+
+  h += '<div id="' + sid + '-monthly-returns"></div>';
+  h += '<div id="' + sid + '-yearly-returns"></div>';
+  h += '<div id="' + sid + '-return-scenarios"></div>';
+  h += '</div>';
+
+  h += '<div class="tab-panel" id="' + sid + '-trading">';
+  h += '<div id="' + sid + '-timeline-scatter"></div>';
+  h += '<div id="' + sid + '-trades-table"></div>';
+  h += '<div id="' + sid + '-orders-table"></div>';
+  h += '<div id="' + sid + '-positions-table"></div>';
+  h += '</div>';
+
+  h += '</div>';
+
+  var container = document.getElementById('strategy-pages-container');
+  if (container) container.insertAdjacentHTML('beforeend', h);
+}
+
+function escHtmlLazy(str) {
+  if (!str) return '';
+  return String(str).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
+}
+
 function onOverviewWindowChange(value) {
   onRunViewChange(value);
 }
@@ -448,6 +534,10 @@ function syncSbChallenger() {
 }
 
 function showPage(pageId) {
+  // Lazy-render strategy pages on first visit
+  var stratMatch = pageId.match(/^strat-(\d+)$/);
+  if (stratMatch) ensureStratPage(parseInt(stratMatch[1]));
+
   document.querySelectorAll('.page').forEach(p => p.style.display = 'none');
   const target = document.getElementById('page-' + pageId);
   if (target) target.style.display = 'block';
@@ -4494,8 +4584,9 @@ function drawPageCharts(pageId) {
 
 // ===== POPULATE RUNS TAB CELLS =====
 function populateRunsTabs() {
+  // Only populate runs tables for already-rendered strategy pages
   STRATEGIES.forEach((s, idx) => {
-    populateStratRunsTable(idx);
+    if (_renderedStratPages[idx]) populateStratRunsTable(idx);
   });
 }
 
@@ -6468,6 +6559,17 @@ function updateStratTables(stratIdx) {
 }
 
 // ===== INIT =====
+// Auto-select top 10 by CAGR when there are more than 10 strategies
+if (!IS_SINGLE && STRATEGIES.length > 10) {
+  var _initIndices = STRATEGIES.map(function(_,i){return i;});
+  _initIndices.sort(function(a,b) {
+    var ca = STRATEGIES[a].summary.cagr, cb = STRATEGIES[b].summary.cagr;
+    if (ca == null) ca = -Infinity;
+    if (cb == null) cb = -Infinity;
+    return cb - ca;
+  });
+  _initIndices.slice(0, 10).forEach(function(i){ selectedForCompare.add(i); });
+}
 buildBenchmarkChips();
 rebuildWindowCoverage();
 rebuildOverviewKPIs();
@@ -6476,6 +6578,7 @@ rebuildOverviewTradingActivity();
 rebuildReturnScenarios();
 initCollapseButtons();
 syncModalCount();
+syncMainTableCheckboxes();
 populateRunsTabs();
 drawPageCharts('overview');
 renderNotesList();