From 96b67614dd12cd42d4bf5729d7348912a8f90c42 Mon Sep 17 00:00:00 2001 From: Harun Baris Bulut Date: Sun, 11 Oct 2026 11:24:33 +0300 Subject: [PATCH] fix: Whoopsy! band re-learns after a lasting shift (v0.13.2) The band froze while values broke it, so a level shift that stayed never resolved: mean and bounds stayed flat and hits grew without end until someone reset by hand. The engine now keeps the latest outliers and, after WhoopsyRelearn (5) times the longer of window and consecutive results outside the band, takes them as the new window. The incident resolves and the band follows the new level. Co-Authored-By: Claude Sonnet 5.5 --- internal/engine/evaluate.go | 34 ++++++++++++++++++++++++-------- internal/engine/evaluate_test.go | 25 +++++++++++++++++++++++ internal/threshold/whoopsy.go | 1 + 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go index 07f0271..c9f6068 100644 --- a/internal/engine/evaluate.go +++ b/internal/engine/evaluate.go @@ -47,12 +47,15 @@ type RuleState struct { // metric, how many results in a row left the band, and the band the last // result was judged against. type WhoopsyState struct { - Metric string `json:"metric"` - Values []float64 `json:"values"` - Hits int `json:"hits"` - Last float64 `json:"last"` // the last value judged, inside the band or not - Level string `json:"level,omitempty"` - Band *threshold.Band `json:"band,omitempty"` + Metric string `json:"metric"` + Values []float64 `json:"values"` + Hits int `json:"hits"` + // Outliers are the latest values that broke the band, kept while the + // band is frozen so a lasting shift can become the new normal. + Outliers []float64 `json:"outliers,omitempty"` + Last float64 `json:"last"` // the last value judged, inside the band or not + Level string `json:"level,omitempty"` + Band *threshold.Band `json:"band,omitempty"` } // State is a check's state machine, stored in check_state. @@ -179,6 +182,7 @@ func Evaluate(cfg Config, prev State, in Input) Outcome { } else { c := *ws c.Values = slices.Clone(ws.Values) + c.Outliers = slices.Clone(ws.Outliers) ws = &c } v := metric(cfg.Metrics, in.Metrics, w.Metric) @@ -197,8 +201,22 @@ func Evaluate(cfg Config, prev State, in Input) Outcome { ws.Last = v // The band is frozen while values break it: an outlier never // enters the window, so a slowdown cannot widen the band and - // resolve itself. Resetting Whoopsy! accepts a new normal. - if !outside { + // resolve itself. Resetting Whoopsy! accepts a new normal, and so + // does a shift that lasts WhoopsyRelearn times the longer of the + // window and the consecutive count: the band is learned again + // from the latest values. + if outside { + ws.Outliers = append(ws.Outliers, v) + if extra := len(ws.Outliers) - w.Window; extra > 0 { + ws.Outliers = ws.Outliers[extra:] + } + if ws.Hits >= threshold.WhoopsyRelearn*max(w.Window, w.Consecutive) && len(ws.Outliers) >= w.Window { + ws.Values, ws.Outliers = ws.Outliers, nil + ws.Hits, ws.Band = 0, nil + outside = false + } + } else { + ws.Outliers = nil ws.Values = append(ws.Values, v) } if extra := len(ws.Values) - w.Window; extra > 0 { diff --git a/internal/engine/evaluate_test.go b/internal/engine/evaluate_test.go index a5040f9..b9d1384 100644 --- a/internal/engine/evaluate_test.go +++ b/internal/engine/evaluate_test.go @@ -254,6 +254,31 @@ func TestWhoopsy(t *testing.T) { } } +// A shift that lasts is learned as the new normal instead of freezing the +// band for good: the incident resolves and the band follows the new level. +func TestWhoopsyRelearn(t *testing.T) { + w := &threshold.Whoopsy{Metric: "total_ms", Window: 7, Deviations: 1, Consecutive: 3, Direction: "above", Severity: "critical"} + s := newSim(t, Config{FailureCount: 3, RecoveryCount: 1, Metrics: []string{"total_ms"}, Whoopsy: w}) + for _, v := range []float64{400, 500, 600, 400, 500, 600, 500} { + s.step(plugin.OK, v) + } + limit := threshold.WhoopsyRelearn * 7 + for range limit - 1 { + s.step(plugin.OK, 1000) + } + if s.count(ActionOpen) != 1 || s.count(ActionResolve) != 0 || s.state.Whoopsy.Band.Upper > 600 { + t.Fatalf("band frozen before the limit: %+v %v", s.state.Whoopsy, s.acts) + } + s.step(plugin.OK, 1000) + if s.count(ActionResolve) != 1 || s.state.Whoopsy.Hits != 0 || len(s.state.Whoopsy.Values) != 7 { + t.Fatalf("the shift was not accepted: %+v %v", s.state.Whoopsy, s.acts) + } + s.step(plugin.OK, 1000) + if b := s.state.Whoopsy.Band; b == nil || b.Mean != 1000 || s.state.Whoopsy.Hits != 0 { + t.Errorf("band did not follow 1000: %+v", s.state.Whoopsy) + } +} + func TestWhoopsyBand(t *testing.T) { w := threshold.Whoopsy{Deviations: 2, MinDelta: 50, Direction: "both"} b := w.BandOf([]float64{100, 100, 100}) diff --git a/internal/threshold/whoopsy.go b/internal/threshold/whoopsy.go index 003b2bb..b14905d 100644 --- a/internal/threshold/whoopsy.go +++ b/internal/threshold/whoopsy.go @@ -27,6 +27,7 @@ const ( WhoopsyWindow = 7 WhoopsyDeviations = 1 WhoopsyConsecutive = 3 + WhoopsyRelearn = 5 // a shift lasting this many windows becomes the new normal WhoopsyRuleID = "whoopsy" WhoopsyName = "Whoopsy!" )