Improve investing decision quality: constraints, EV neutrality, eval metrics

internal/backtest/harness.go

@@ -59,7 +59,7 @@ func (h *Harness) Run(ctx context.Context, sc Scenario) (Report, error) {
 	}
 
 	rep := Report{Scenario: sc.Name}
-	var reacts, correctReacts, shouldKills int
+	var reacts, correctReacts, shouldKills, noiseEvents, noiseReacts int
 
 	for _, ev := range sc.Events {
 		h.sim.t = ev.At
@@ -85,6 +85,11 @@ func (h *Harness) Run(ctx context.Context, sc Scenario) (Report, error) {
 
 		if ev.ShouldKill {
 			shouldKills++
+		} else {
+			noiseEvents++
+			if res.Material {
+				noiseReacts++
+			}
 		}
 		if res.Material {
 			reacts++
@@ -102,27 +107,67 @@ func (h *Harness) Run(ctx context.Context, sc Scenario) (Report, error) {
 	if shouldKills > 0 {
 		rep.KillRecall = float64(correctReacts) / float64(shouldKills)
 	}
-	rep.HypotheticalPnL = h.hypotheticalPnL(ctx, current, start, h.sim.t)
+	rep.NoiseRobustness = 1
+	if noiseEvents > 0 {
+		rep.NoiseRobustness = 1 - float64(noiseReacts)/float64(noiseEvents)
+	}
+	h.scoreCalibration(ctx, &rep, h.sim.t)
+	finalTop, _ := topMove(current)
+	rep.HypotheticalPnL, _ = h.tickerReturn(ctx, tickerFromTitle(finalTop), start, h.sim.t)
 	return rep, nil
 }
 
-// hypotheticalPnL is an illustrative buy-and-hold return of the final top thesis's
-// ticker over the scenario window. It is NOT a strategy return.
-func (h *Harness) hypotheticalPnL(ctx context.Context, current domain.PlanVersion, from, to time.Time) float64 {
-	top, _ := topMove(current)
-	ticker := strings.TrimPrefix(top, "Thesis: ")
-	if ticker == top || ticker == "" {
-		return 0
+// scoreCalibration fills per-decision forward returns and the report's Brier
+// score. The forward window runs from each decision to the scenario end; it is
+// computed after the replay, so no future data ever reaches a decision. The
+// outcome label is 1 when the top thesis's forward return is positive, falling
+// back to the analyst kill label when no return is resolvable.
+func (h *Harness) scoreCalibration(ctx context.Context, rep *Report, end time.Time) {
+	if len(rep.Decisions) == 0 {
+		return
+	}
+	var sum float64
+	for i := range rep.Decisions {
+		d := &rep.Decisions[i]
+		fr, ok := h.tickerReturn(ctx, tickerFromTitle(d.TopMove), d.At, end)
+		if ok {
+			d.ForwardReturn = fr
+		}
+		var label float64
+		if (ok && fr > 0) || (!ok && !d.ShouldKill) {
+			label = 1
+		}
+		diff := d.TopConfidence - label
+		sum += diff * diff
+	}
+	rep.BrierScore = sum / float64(len(rep.Decisions))
+}
+
+// tickerReturn is the buy-and-hold return of one ticker over [from, to] from
+// point-in-time quotes. It reports ok=false when either quote is unavailable.
+func (h *Harness) tickerReturn(ctx context.Context, ticker string, from, to time.Time) (float64, bool) {
+	if ticker == "" {
+		return 0, false
 	}
 	startQ, err := h.provider.Quote(ctx, ticker, from)
 	if err != nil || startQ.Price == 0 {
-		return 0
+		return 0, false
 	}
 	endQ, err := h.provider.Quote(ctx, ticker, to)
 	if err != nil {
-		return 0
+		return 0, false
+	}
+	return (endQ.Price - startQ.Price) / startQ.Price, true
+}
+
+// tickerFromTitle extracts the ticker from a finance-planner move title
+// ("Thesis: ACME" -> "ACME"); non-thesis titles yield "".
+func tickerFromTitle(title string) string {
+	ticker := strings.TrimPrefix(title, "Thesis: ")
+	if ticker == title {
+		return ""
 	}
-	return (endQ.Price - startQ.Price) / startQ.Price
+	return ticker
 }
 
 func topMove(v domain.PlanVersion) (string, float64) {

internal/backtest/harness_test.go

@@ -3,6 +3,7 @@ package backtest
 import (
 	"context"
 	"encoding/json"
+	"math"
 	"os"
 	"testing"
 
@@ -56,4 +57,23 @@ func TestHarnessRun(t *testing.T) {
 	if rep.HypotheticalPnL <= 0 {
 		t.Errorf("expected positive illustrative pnl from fixtures, got %v", rep.HypotheticalPnL)
 	}
+
+	// The valuation_change event is labeled non-kill and produces no material
+	// replan, so the run never reacts to noise.
+	if rep.NoiseRobustness != 1 {
+		t.Errorf("expected noise robustness 1, got %v", rep.NoiseRobustness)
+	}
+	// Calibration: confidence should beat a coin flip (Brier 0.25) on this
+	// scenario — ACME rallies while held, and the broken thesis ends at zero
+	// confidence with a zero forward return.
+	if rep.BrierScore <= 0 || rep.BrierScore >= 0.25 {
+		t.Errorf("expected brier score in (0, 0.25), got %v", rep.BrierScore)
+	}
+	first, last := rep.Decisions[0], rep.Decisions[len(rep.Decisions)-1]
+	if math.Abs(first.ForwardReturn-0.08) > 1e-9 {
+		t.Errorf("expected +8%% forward return on first decision from fixtures, got %v", first.ForwardReturn)
+	}
+	if last.ForwardReturn != 0 {
+		t.Errorf("final decision has no forward window, expected 0, got %v", last.ForwardReturn)
+	}
 }

internal/backtest/report.go

@@ -15,6 +15,9 @@ type Decision struct {
 	TopMove       string    `json:"top_move"`
 	TopConfidence float64   `json:"top_confidence"`
 	ShouldKill    bool      `json:"should_kill"`
+	// ForwardReturn is the top thesis ticker's return from decision time to the
+	// end of the scenario (evaluation-only attribution; never a planner input).
+	ForwardReturn float64 `json:"forward_return"`
 }
 
 // Report summarises a backtest run. The metrics describe decision/replanning
@@ -25,7 +28,15 @@ type Report struct {
 	VersionsCreated int        `json:"versions_created"`
 	KillPrecision   float64    `json:"kill_precision"` // correct reactions / total reactions
 	KillRecall      float64    `json:"kill_recall"`    // correct reactions / total should-kill events
-	HypotheticalPnL float64    `json:"hypothetical_pnl"`
+	// BrierScore is the mean squared error between each decision's top-move
+	// confidence and its realized outcome label (1 if the top thesis's forward
+	// return was positive, falling back to the analyst kill label). Lower is
+	// better; a coin-flip confidence of 0.5 scores 0.25.
+	BrierScore float64 `json:"brier_score"`
+	// NoiseRobustness is the share of non-kill events that did NOT trigger a
+	// material replan (1 == never reacted to noise).
+	NoiseRobustness float64 `json:"noise_robustness"`
+	HypotheticalPnL float64 `json:"hypothetical_pnl"`
 }
 
 // Render writes a human-readable report to w.
@@ -34,14 +45,16 @@ func (r Report) Render(w io.Writer) {
 	fmt.Fprintf(w, "  versions created : %d\n", r.VersionsCreated)
 	fmt.Fprintf(w, "  kill precision   : %.2f\n", r.KillPrecision)
 	fmt.Fprintf(w, "  kill recall      : %.2f\n", r.KillRecall)
+	fmt.Fprintf(w, "  brier score      : %.3f (lower is better; 0.25 == coin-flip confidence)\n", r.BrierScore)
+	fmt.Fprintf(w, "  noise robustness : %.2f (share of non-kill events not reacted to)\n", r.NoiseRobustness)
 	fmt.Fprintf(w, "  hypothetical pnl : %+.2f%% (illustrative, not a strategy return)\n", r.HypotheticalPnL*100)
 	fmt.Fprintln(w, "  decisions:")
 	for _, d := range r.Decisions {
 		mark := " "
 		if d.Material {
 			mark = "*"
 		}
-		fmt.Fprintf(w, "    %s %s [%s] material=%v should_kill=%v top=%q conf=%.2f — %s\n",
-			mark, d.At.Format("2006-01-02"), d.Kind, d.Material, d.ShouldKill, d.TopMove, d.TopConfidence, d.Reason)
+		fmt.Fprintf(w, "    %s %s [%s] material=%v should_kill=%v top=%q conf=%.2f fwd=%+.2f%% — %s\n",
+			mark, d.At.Format("2006-01-02"), d.Kind, d.Material, d.ShouldKill, d.TopMove, d.TopConfidence, d.ForwardReturn*100, d.Reason)
 	}
 }

internal/finance/constraints.go

@@ -0,0 +1,76 @@
+package finance
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/vingrad/dynamic-decision-engine/internal/domain"
+)
+
+var (
+	percentRe      = regexp.MustCompile(`(\d+(?:\.\d+)?)\s*%`)
+	conservativeRe = regexp.MustCompile(`\b(conservative|cautious|low)\b`)
+	aggressiveRe   = regexp.MustCompile(`\b(aggressive|high)\b`)
+)
+
+// EffectiveRiskBudget tightens a base risk budget with the goal's stated
+// drawdown_limit and risk_tolerance constraints, so per-goal risk preferences
+// actually bind position sizing. Constraints that don't parse leave the base
+// unchanged. The returned note names what bound, for the move rationale.
+func EffectiveRiskBudget(base RiskBudget, constraints []domain.Constraint) (RiskBudget, string) {
+	out := base
+	var notes []string
+	for _, c := range constraints {
+		switch strings.ToLower(strings.TrimSpace(c.Kind)) {
+		case "drawdown_limit":
+			dd, ok := parsePercent(c.Name)
+			if !ok {
+				dd, ok = parsePercent(c.Description)
+			}
+			if !ok || dd <= 0 || dd >= 1 {
+				continue
+			}
+			// A total loss of a single position must not exceed the stated
+			// drawdown limit, and a run of five stopped-out trades must stay
+			// within it.
+			if out.MaxPositionPct == 0 || dd < out.MaxPositionPct {
+				out.MaxPositionPct = dd
+			}
+			if perTrade := dd / 5; out.MaxPortfolioRiskPct == 0 || perTrade < out.MaxPortfolioRiskPct {
+				out.MaxPortfolioRiskPct = perTrade
+			}
+			notes = append(notes, fmt.Sprintf("drawdown_limit %.0f%%", dd*100))
+		case "risk_tolerance":
+			text := strings.ToLower(c.Name + " " + c.Description)
+			switch {
+			case conservativeRe.MatchString(text):
+				out.KellyFraction *= 0.5
+				out.MaxPositionPct *= 0.5
+				notes = append(notes, "risk_tolerance conservative")
+			case aggressiveRe.MatchString(text):
+				// Aggressive loosens the Kelly multiplier only; the hard caps
+				// (concentration, per-trade risk) remain guardrails.
+				out.KellyFraction *= 1.5
+				notes = append(notes, "risk_tolerance aggressive")
+			}
+			// Moderate or unrecognised wording: the base budget already
+			// encodes a moderate stance.
+		}
+	}
+	return out, strings.Join(notes, ", ")
+}
+
+// parsePercent extracts the first "N%" in s as a fraction (10% -> 0.10).
+func parsePercent(s string) (float64, bool) {
+	m := percentRe.FindStringSubmatch(s)
+	if m == nil {
+		return 0, false
+	}
+	n, err := strconv.ParseFloat(m[1], 64)
+	if err != nil {
+		return 0, false
+	}
+	return n / 100, true
+}

internal/finance/constraints_test.go

@@ -0,0 +1,136 @@
+package finance
+
+import (
+	"testing"
+
+	"github.com/vingrad/dynamic-decision-engine/internal/domain"
+)
+
+func baseBudget() RiskBudget {
+	return RiskBudget{MaxPortfolioRiskPct: 0.02, MaxPositionPct: 0.20, KellyFraction: 0.25}
+}
+
+func TestEffectiveRiskBudget(t *testing.T) {
+	cases := []struct {
+		name        string
+		constraints []domain.Constraint
+		want        RiskBudget
+		note        string
+	}{
+		{
+			name: "no constraints leaves base unchanged",
+			want: baseBudget(),
+		},
+		{
+			name: "unrelated kinds leave base unchanged",
+			constraints: []domain.Constraint{
+				{Name: "2 year horizon", Kind: "time_horizon"},
+				{Name: "no leverage", Kind: "mandate"},
+			},
+			want: baseBudget(),
+		},
+		{
+			name: "drawdown limit tightens position and per-trade caps",
+			constraints: []domain.Constraint{
+				{Name: "max 5% drawdown", Kind: "drawdown_limit"},
+			},
+			want: RiskBudget{MaxPortfolioRiskPct: 0.01, MaxPositionPct: 0.05, KellyFraction: 0.25},
+			note: "drawdown_limit 5%",
+		},
+		{
+			name: "drawdown limit looser than base does not loosen it",
+			constraints: []domain.Constraint{
+				{Name: "max 50% drawdown", Kind: "drawdown_limit"},
+			},
+			want: baseBudget(),
+			note: "drawdown_limit 50%",
+		},
+		{
+			name: "drawdown percent parsed from description",
+			constraints: []domain.Constraint{
+				{Name: "drawdown cap", Kind: "drawdown_limit", Description: "no more than 10% peak-to-trough"},
+			},
+			want: RiskBudget{MaxPortfolioRiskPct: 0.02, MaxPositionPct: 0.10, KellyFraction: 0.25},
+			note: "drawdown_limit 10%",
+		},
+		{
+			name: "unparseable drawdown is ignored",
+			constraints: []domain.Constraint{
+				{Name: "keep drawdowns small", Kind: "drawdown_limit"},
+			},
+			want: baseBudget(),
+		},
+		{
+			name: "conservative tolerance halves kelly and concentration",
+			constraints: []domain.Constraint{
+				{Name: "conservative risk tolerance", Kind: "risk_tolerance"},
+			},
+			want: RiskBudget{MaxPortfolioRiskPct: 0.02, MaxPositionPct: 0.10, KellyFraction: 0.125},
+			note: "risk_tolerance conservative",
+		},
+		{
+			name: "aggressive tolerance scales kelly only",
+			constraints: []domain.Constraint{
+				{Name: "aggressive", Kind: "risk_tolerance"},
+			},
+			want: RiskBudget{MaxPortfolioRiskPct: 0.02, MaxPositionPct: 0.20, KellyFraction: 0.375},
+			note: "risk_tolerance aggressive",
+		},
+		{
+			name: "moderate tolerance is a no-op",
+			constraints: []domain.Constraint{
+				{Name: "moderate risk tolerance", Kind: "risk_tolerance"},
+			},
+			want: baseBudget(),
+		},
+		{
+			name: "drawdown and tolerance combine",
+			constraints: []domain.Constraint{
+				{Name: "max 10% drawdown", Kind: "drawdown_limit"},
+				{Name: "low risk appetite", Kind: "risk_tolerance"},
+			},
+			want: RiskBudget{MaxPortfolioRiskPct: 0.02, MaxPositionPct: 0.05, KellyFraction: 0.125},
+			note: "drawdown_limit 10%, risk_tolerance conservative",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, note := EffectiveRiskBudget(baseBudget(), tc.constraints)
+			if got != tc.want {
+				t.Errorf("budget = %+v, want %+v", got, tc.want)
+			}
+			if note != tc.note {
+				t.Errorf("note = %q, want %q", note, tc.note)
+			}
+		})
+	}
+}
+
+func TestEffectiveRiskBudgetNoFalseKeywordMatch(t *testing.T) {
+	// "low" must match as a word, not inside e.g. "follow".
+	got, note := EffectiveRiskBudget(baseBudget(), []domain.Constraint{
+		{Name: "follow the mandate", Kind: "risk_tolerance"},
+	})
+	if got != baseBudget() || note != "" {
+		t.Errorf("substring should not trigger a tolerance match: %+v %q", got, note)
+	}
+}
+
+func TestParsePercent(t *testing.T) {
+	cases := []struct {
+		in   string
+		want float64
+		ok   bool
+	}{
+		{"max 10% drawdown", 0.10, true},
+		{"2.5 %", 0.025, true},
+		{"no percent here", 0, false},
+		{"", 0, false},
+	}
+	for _, tc := range cases {
+		got, ok := parsePercent(tc.in)
+		if got != tc.want || ok != tc.ok {
+			t.Errorf("parsePercent(%q) = (%v, %v), want (%v, %v)", tc.in, got, ok, tc.want, tc.ok)
+		}
+	}
+}