Add backtest actual-vs-prediction points and render per-window charts

bryanwhiting · bryanwhiting · commit ef41b442c00a · 2026-02-23T13:27:42.000-08:00
diff --git a/site/src/layouts/ForecastPage.astro b/site/src/layouts/ForecastPage.astro
@@ -6,12 +6,14 @@ const { data } = Astro.props;
 const history = data.history ?? [];
 const forecast = data.forecast ?? [];
 const backtest = data.backtest ?? [];
+const backtestPoints = data.backtest_points ?? [];
 
 const runTitle = data?.meta?.slug || data.request?.run_name_root || (data.request?.series_names || []).join(', ') || 'series';
 
 const seriesIds = [...new Set([
   ...history.map((r) => r.unique_id),
   ...forecast.map((r) => r.unique_id),
+  ...backtestPoints.map((r) => r.unique_id),
 ])];
 
 const parseDs = (v) => {
@@ -21,13 +23,18 @@ const parseDs = (v) => {
   return Number.isNaN(d.getTime()) ? null : d;
 };
 
-const svgChartForSeries = (seriesId, valuesType = 'forecast') => {
-  const hRows = history.filter((r) => r.unique_id === seriesId).map((r) => ({ ...r, iso: parseDs(r.ds)?.toISOString() })).filter((r) => r.iso);
-  const fRows = forecast.filter((r) => r.unique_id === seriesId).map((r) => ({ ...r, iso: parseDs(r.ds)?.toISOString() })).filter((r) => r.iso);
-  const labels = [...new Set([...hRows.map((r) => r.iso), ...fRows.map((r) => r.iso)])].sort();
+const buildChartState = (actualRows, forecastRows, h = 320) => {
+  const aRows = actualRows
+    .map((r) => ({ ...r, iso: parseDs(r.ds)?.toISOString() }))
+    .filter((r) => r.iso);
+  const fRows = forecastRows
+    .map((r) => ({ ...r, iso: parseDs(r.ds)?.toISOString() }))
+    .filter((r) => r.iso);
+
+  const labels = [...new Set([...aRows.map((r) => r.iso), ...fRows.map((r) => r.iso)])].sort();
 
   const actualValues = labels.map((l) => {
-    const row = hRows.find((r) => r.iso === l);
+    const row = aRows.find((r) => r.iso === l);
     return row ? Number(row.y) : null;
   });
 
@@ -44,11 +51,12 @@ const svgChartForSeries = (seriesId, valuesType = 'forecast') => {
     ...actualValues.filter((v) => Number.isFinite(v)),
     ...modelSeries.flatMap((s) => s.values.filter((v) => Number.isFinite(v))),
   ];
+
   const yMin = allY.length ? Math.min(...allY) : 0;
   const yMax = allY.length ? Math.max(...allY) : 1;
 
   const W = 920;
-  const H = valuesType === 'backtest' ? 240 : 320;
+  const H = h;
   const PAD = { t: 18, r: 18, b: 40, l: 44 };
   const innerW = W - PAD.l - PAD.r;
   const innerH = H - PAD.t - PAD.b;
@@ -77,31 +85,32 @@ const svgChartForSeries = (seriesId, valuesType = 'forecast') => {
   return { W, H, PAD, innerW, innerH, labels, actualValues, actualPath, modelSeries, colors, xFor, yFor, yMin, yMax };
 };
 
-
-const buildSmapeSpark = (rows) => {
-  const W = 920, H = 200, PAD = { t: 16, r: 16, b: 32, l: 38 };
-  const vals = rows.map((r) => Number(r.smape || 0));
-  const min = vals.length ? Math.min(...vals) : 0;
-  const max = vals.length ? Math.max(...vals) : 1;
-  const range = max - min || 1;
-  const x = (i) => PAD.l + (rows.length <= 1 ? 0 : (i / (rows.length - 1)) * (W - PAD.l - PAD.r));
-  const y = (v) => PAD.t + (1 - ((v - min) / range)) * (H - PAD.t - PAD.b);
-  let path = '';
-  rows.forEach((r, i) => {
-    const xv = x(i), yv = y(Number(r.smape || 0));
-    path += path ? ` L ${xv} ${yv}` : `M ${xv} ${yv}`;
-  });
-  const points = rows.map((r, i) => ({ x: x(i), y: y(Number(r.smape || 0)) }));
-  return { W, H, PAD, path, points };
+const getMainChart = (sid) => {
+  const hRows = history.filter((r) => r.unique_id === sid);
+  const fRows = forecast.filter((r) => r.unique_id === sid);
+  return buildChartState(hRows, fRows, 320);
 };
 
 const backtestBySeries = seriesIds.map((sid) => {
-  const rows = backtest.filter((r) => r.unique_id === sid);
-  const byModel = [...new Set(rows.map((r) => r.model))].map((m) => ({
-    model: m,
-    rows: rows.filter((r) => r.model === m).sort((a, b) => Number(a.window) - Number(b.window)),
-  }));
-  return { sid, rows, byModel };
+  const scoreRows = backtest.filter((r) => r.unique_id === sid);
+  const windowIds = [...new Set(scoreRows.map((r) => Number(r.window)).filter(Number.isFinite))].sort((a, b) => a - b);
+
+  const windows = windowIds.map((w) => {
+    const pts = backtestPoints.filter((p) => p.unique_id === sid && Number(p.window) === w);
+    const actualMap = new Map();
+    pts.forEach((p) => {
+      const key = String(p.ds);
+      if (!actualMap.has(key)) actualMap.set(key, { ds: p.ds, y: p.y });
+    });
+    const actualRows = [...actualMap.values()];
+    const forecastRows = pts.map((p) => ({ ds: p.ds, model: p.model, yhat: p.yhat }));
+    const chart = buildChartState(actualRows, forecastRows, 260);
+
+    const scores = scoreRows.filter((r) => Number(r.window) === w).sort((a, b) => String(a.model).localeCompare(String(b.model)));
+    return { window: w, chart, scores };
+  });
+
+  return { sid, windows, scoreRows };
 });
 ---
 <MainLayout title={`Forecast ${runTitle}`}>
@@ -112,7 +121,7 @@ const backtestBySeries = seriesIds.map((sid) => {
 
   <section class="mt-4 space-y-4">
     {seriesIds.map((sid) => {
-      const chart = svgChartForSeries(sid);
+      const chart = getMainChart(sid);
       const back = backtestBySeries.find((x) => x.sid === sid);
       return (
         <article class="rounded-xl border border-slate-700 bg-slate-900/70 p-4">
@@ -175,35 +184,82 @@ const backtestBySeries = seriesIds.map((sid) => {
 
           <details class="mt-4 rounded-lg border border-slate-700 bg-slate-950/40 p-3">
             <summary class="cursor-pointer font-semibold text-sm">Backfill testing + accuracies</summary>
-            {back && back.rows.length > 0 ? (
-              <div class="mt-3 space-y-3">
-                {back.byModel.map((m, mi) => {
-                  const spark = buildSmapeSpark(m.rows);
-                  const color = ['#ef4444','#22c55e','#a855f7','#f97316'][mi%4];
-                  return (
-                    <div class="rounded border border-slate-700 p-2">
-                      <div class="text-xs text-slate-300 mb-1">{m.model} SMAPE by backtest window</div>
-                      <div class="overflow-x-auto">
-                        <svg viewBox={`0 0 ${spark.W} ${spark.H}`} class="min-w-[680px] w-full rounded bg-slate-950">
-                          <rect x={spark.PAD.l} y={spark.PAD.t} width={spark.W-spark.PAD.l-spark.PAD.r} height={spark.H-spark.PAD.t-spark.PAD.b} fill="none" stroke="#1e293b" />
-                          <path d={spark.path} fill="none" stroke={color} stroke-width="2" />
-                          {spark.points.map((pt) => <circle cx={pt.x} cy={pt.y} r="3" fill={color} />)}
-                        </svg>
-                      </div>
+            {back && back.windows.length > 0 ? (
+              <div class="mt-3 space-y-4">
+                {back.windows.map((wObj) => (
+                  <div class="rounded border border-slate-700 p-3">
+                    <div class="text-xs text-slate-300 mb-2">Backtest window {wObj.window} · actual vs predicted</div>
+                    <div class="overflow-x-auto">
+                      <svg viewBox={`0 0 ${wObj.chart.W} ${wObj.chart.H}`} class="min-w-[720px] w-full rounded bg-slate-950">
+                        <rect x={wObj.chart.PAD.l} y={wObj.chart.PAD.t} width={wObj.chart.innerW} height={wObj.chart.innerH} fill="none" stroke="#1e293b" />
+
+                        {Array.from({ length: 5 }).map((_, i) => {
+                          const y = wObj.chart.PAD.t + (i / 4) * wObj.chart.innerH;
+                          const val = (wObj.chart.yMax - ((i / 4) * (wObj.chart.yMax - wObj.chart.yMin))).toFixed(1);
+                          return (
+                            <g>
+                              <line x1={wObj.chart.PAD.l} y1={y} x2={wObj.chart.W - wObj.chart.PAD.r} y2={y} stroke="#0f172a" />
+                              <text x="6" y={y + 4} font-size="11" fill="#94a3b8">{val}</text>
+                            </g>
+                          );
+                        })}
+
+                        {wObj.chart.actualPath && <path d={wObj.chart.actualPath} fill="none" stroke="#38bdf8" stroke-width="2" />}
+                        {wObj.chart.actualValues.map((v, i) => {
+                          const y = wObj.chart.yFor(v);
+                          if (y === null) return null;
+                          return <circle cx={wObj.chart.xFor(i)} cy={y} r="2.7" fill="#38bdf8" />;
+                        })}
+
+                        {wObj.chart.modelSeries.map((s, idx) => {
+                          const d = (() => {
+                            let p = '';
+                            s.values.forEach((v, i) => {
+                              const y = wObj.chart.yFor(v);
+                              if (y === null) return;
+                              const x = wObj.chart.xFor(i);
+                              p += p ? ` L ${x} ${y}` : `M ${x} ${y}`;
+                            });
+                            return p;
+                          })();
+
+                          return (
+                            <>
+                              {d && <path d={d} fill="none" stroke={wObj.chart.colors[idx % wObj.chart.colors.length]} stroke-width="2" stroke-dasharray="6 4" />}
+                              {s.values.map((v, i) => {
+                                const y = wObj.chart.yFor(v);
+                                if (y === null) return null;
+                                return <circle cx={wObj.chart.xFor(i)} cy={y} r="2.4" fill={wObj.chart.colors[idx % wObj.chart.colors.length]} />;
+                              })}
+                            </>
+                          );
+                        })}
+                      </svg>
                     </div>
-                  );
-                })}
-
-                <div class="overflow-x-auto">
-                  <table class="min-w-[760px] text-sm w-full">
-                    <thead><tr class="text-slate-300"><th class="p-2 text-left">Window</th><th class="p-2 text-left">Model</th><th class="p-2 text-left">SMAPE</th><th class="p-2 text-left">Holdout</th></tr></thead>
-                    <tbody>
-                      {back.rows.sort((a,b)=>Number(a.window)-Number(b.window)).map((row) => (
-                        <tr><td class="p-2">{row.window}</td><td class="p-2">{row.model}</td><td class="p-2">{row.smape}</td><td class="p-2">{String(row.holdout_start)} → {String(row.holdout_end)}</td></tr>
+
+                    <div class="mt-2 flex flex-wrap gap-3 text-xs text-slate-300">
+                      <span class="inline-flex items-center gap-1"><span class="inline-block w-4 h-[2px] bg-sky-400"></span>Actual</span>
+                      {wObj.chart.modelSeries.map((s, idx) => (
+                        <span class="inline-flex items-center gap-1"><span class="inline-block w-4 h-[2px]" style={`background:${wObj.chart.colors[idx % wObj.chart.colors.length]}`}></span>Predicted ({s.model})</span>
                       ))}
-                    </tbody>
-                  </table>
-                </div>
+                    </div>
+
+                    <div class="mt-2 overflow-x-auto">
+                      <table class="min-w-[460px] text-xs w-full">
+                        <thead><tr class="text-slate-300"><th class="p-2 text-left">Model</th><th class="p-2 text-left">SMAPE</th><th class="p-2 text-left">Holdout Range</th></tr></thead>
+                        <tbody>
+                          {wObj.scores.map((row) => (
+                            <tr>
+                              <td class="p-2">{row.model}</td>
+                              <td class="p-2">{row.smape}</td>
+                              <td class="p-2">{String(row.holdout_start)} → {String(row.holdout_end)}</td>
+                            </tr>
+                          ))}
+                        </tbody>
+                      </table>
+                    </div>
+                  </div>
+                ))}
               </div>
             ) : (
               <p class="mt-2 text-sm text-slate-400">No backtest rows for this series yet (need more history or windows).</p>
diff --git a/site/src/pages/forecasts/[slug].astro b/site/src/pages/forecasts/[slug].astro
@@ -20,6 +20,7 @@ export async function getStaticPaths() {
     history: z.array(z.any()).default([]),
     forecast: z.array(z.any()).default([]),
     backtest: z.array(z.any()).default([]),
+    backtest_points: z.array(z.any()).default([]),
     meta: z.any().optional(),
   });
 
diff --git a/weatherman/cli.py b/weatherman/cli.py
@@ -29,6 +29,7 @@ def main() -> None:
         "history": result.history.to_dict(orient="records"),
         "forecast": result.forecast.to_dict(orient="records"),
         "backtest": result.backtest.to_dict(orient="records"),
+        "backtest_points": result.backtest_points.to_dict(orient="records"),
     }
     _save_payload(Path(args.output), payload)
 
diff --git a/weatherman/service.py b/weatherman/service.py
@@ -28,6 +28,7 @@ class ForecastResult:
     forecast: pd.DataFrame
     backend: str
     backtest: pd.DataFrame
+    backtest_points: pd.DataFrame
 
 
 def _smape(y_true: np.ndarray, y_pred: np.ndarray) -> float:
@@ -81,16 +82,18 @@ def _forecast_nixtla_compare(
     freq: str,
     do_backtest: bool,
     backtest_windows: int,
-) -> tuple[pd.DataFrame, pd.DataFrame]:
+) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
     models = [AutoARIMA(season_length=1), AutoETS(season_length=1)]
 
     backtest_df = pd.DataFrame(columns=["unique_id", "window", "model", "smape", "horizon", "holdout_start", "holdout_end"])
+    backtest_points_df = pd.DataFrame(columns=["unique_id", "window", "model", "ds", "y", "yhat"])
     if do_backtest:
         min_len = int(df.groupby("unique_id").size().min())
         max_possible_windows = max(0, (min_len // horizon) - 1)
         windows = max(1, min(backtest_windows, max_possible_windows)) if max_possible_windows > 0 else 0
 
         scores = []
+        backtest_points = []
         for w in range(windows):
             # Rolling holdout from older to newer windows
             offset = horizon * (windows - w)
@@ -124,14 +127,26 @@ def _forecast_nixtla_compare(
                                 "holdout_end": holdout_end,
                             }
                         )
+                        for _, r in uid_df.iterrows():
+                            backtest_points.append(
+                                {
+                                    "unique_id": uid,
+                                    "window": w + 1,
+                                    "model": model_name,
+                                    "ds": r["ds"],
+                                    "y": float(r["y"]),
+                                    "yhat": float(r[model_name]),
+                                }
+                            )
         backtest_df = pd.DataFrame(scores)
+        backtest_points_df = pd.DataFrame(backtest_points)
 
     sf = StatsForecast(models=models, freq=freq, n_jobs=1)
     fcst = sf.forecast(df=df, h=horizon)
 
     cols = [c for c in MODEL_NAMES if c in fcst.columns]
     long_fcst = fcst.melt(id_vars=["unique_id", "ds"], value_vars=cols, var_name="model", value_name="yhat")
-    return long_fcst, backtest_df
+    return long_fcst, backtest_df, backtest_points_df
 
 
 def _forecast_autogluon(df: pd.DataFrame, horizon: int) -> pd.DataFrame:
@@ -169,8 +184,9 @@ def forecast_from_request(req: ForecastRequest) -> ForecastResult:
     if backend == "autogluon":
         forecast = _forecast_autogluon(history, req.horizon)
         backtest = pd.DataFrame(columns=["model", "smape", "horizon"])
+        backtest_points = pd.DataFrame(columns=["unique_id", "window", "model", "ds", "y", "yhat"])
     else:
-        forecast, backtest = _forecast_nixtla_compare(
+        forecast, backtest, backtest_points = _forecast_nixtla_compare(
             history,
             req.horizon,
             freq,
@@ -179,4 +195,4 @@ def forecast_from_request(req: ForecastRequest) -> ForecastResult:
         )
         backend = "nixtla"
 
-    return ForecastResult(history=history, forecast=forecast, backend=backend, backtest=backtest)
+    return ForecastResult(history=history, forecast=forecast, backend=backend, backtest=backtest, backtest_points=backtest_points)

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ def main() -> None:`
`29`	`29`	`"history": result.history.to_dict(orient="records"),`
`30`	`30`	`"forecast": result.forecast.to_dict(orient="records"),`
`31`	`31`	`"backtest": result.backtest.to_dict(orient="records"),`
	`32`	`+ "backtest_points": result.backtest_points.to_dict(orient="records"),`
`32`	`33`	`}`
`33`	`34`	`_save_payload(Path(args.output), payload)`
`34`	`35`