fix: Whoopsy! band re-learns after a lasting shift (v0.13.2)

internal/engine/evaluate.go

@@ -47,12 +47,15 @@ type RuleState struct {
 // metric, how many results in a row left the band, and the band the last
 // result was judged against.
 type WhoopsyState struct {
-	Metric string          `json:"metric"`
-	Values []float64       `json:"values"`
-	Hits   int             `json:"hits"`
-	Last   float64         `json:"last"` // the last value judged, inside the band or not
-	Level  string          `json:"level,omitempty"`
-	Band   *threshold.Band `json:"band,omitempty"`
+	Metric string    `json:"metric"`
+	Values []float64 `json:"values"`
+	Hits   int       `json:"hits"`
+	// Outliers are the latest values that broke the band, kept while the
+	// band is frozen so a lasting shift can become the new normal.
+	Outliers []float64       `json:"outliers,omitempty"`
+	Last     float64         `json:"last"` // the last value judged, inside the band or not
+	Level    string          `json:"level,omitempty"`
+	Band     *threshold.Band `json:"band,omitempty"`
 }
 
 // State is a check's state machine, stored in check_state.
@@ -179,6 +182,7 @@ func Evaluate(cfg Config, prev State, in Input) Outcome {
 		} else {
 			c := *ws
 			c.Values = slices.Clone(ws.Values)
+			c.Outliers = slices.Clone(ws.Outliers)
 			ws = &c
 		}
 		v := metric(cfg.Metrics, in.Metrics, w.Metric)
@@ -197,8 +201,22 @@ func Evaluate(cfg Config, prev State, in Input) Outcome {
 			ws.Last = v
 			// The band is frozen while values break it: an outlier never
 			// enters the window, so a slowdown cannot widen the band and
-			// resolve itself. Resetting Whoopsy! accepts a new normal.
-			if !outside {
+			// resolve itself. Resetting Whoopsy! accepts a new normal, and so
+			// does a shift that lasts WhoopsyRelearn times the longer of the
+			// window and the consecutive count: the band is learned again
+			// from the latest values.
+			if outside {
+				ws.Outliers = append(ws.Outliers, v)
+				if extra := len(ws.Outliers) - w.Window; extra > 0 {
+					ws.Outliers = ws.Outliers[extra:]
+				}
+				if ws.Hits >= threshold.WhoopsyRelearn*max(w.Window, w.Consecutive) && len(ws.Outliers) >= w.Window {
+					ws.Values, ws.Outliers = ws.Outliers, nil
+					ws.Hits, ws.Band = 0, nil
+					outside = false
+				}
+			} else {
+				ws.Outliers = nil
 				ws.Values = append(ws.Values, v)
 			}
 			if extra := len(ws.Values) - w.Window; extra > 0 {

internal/engine/evaluate_test.go

@@ -254,6 +254,31 @@ func TestWhoopsy(t *testing.T) {
 	}
 }
 
+// A shift that lasts is learned as the new normal instead of freezing the
+// band for good: the incident resolves and the band follows the new level.
+func TestWhoopsyRelearn(t *testing.T) {
+	w := &threshold.Whoopsy{Metric: "total_ms", Window: 7, Deviations: 1, Consecutive: 3, Direction: "above", Severity: "critical"}
+	s := newSim(t, Config{FailureCount: 3, RecoveryCount: 1, Metrics: []string{"total_ms"}, Whoopsy: w})
+	for _, v := range []float64{400, 500, 600, 400, 500, 600, 500} {
+		s.step(plugin.OK, v)
+	}
+	limit := threshold.WhoopsyRelearn * 7
+	for range limit - 1 {
+		s.step(plugin.OK, 1000)
+	}
+	if s.count(ActionOpen) != 1 || s.count(ActionResolve) != 0 || s.state.Whoopsy.Band.Upper > 600 {
+		t.Fatalf("band frozen before the limit: %+v %v", s.state.Whoopsy, s.acts)
+	}
+	s.step(plugin.OK, 1000)
+	if s.count(ActionResolve) != 1 || s.state.Whoopsy.Hits != 0 || len(s.state.Whoopsy.Values) != 7 {
+		t.Fatalf("the shift was not accepted: %+v %v", s.state.Whoopsy, s.acts)
+	}
+	s.step(plugin.OK, 1000)
+	if b := s.state.Whoopsy.Band; b == nil || b.Mean != 1000 || s.state.Whoopsy.Hits != 0 {
+		t.Errorf("band did not follow 1000: %+v", s.state.Whoopsy)
+	}
+}
+
 func TestWhoopsyBand(t *testing.T) {
 	w := threshold.Whoopsy{Deviations: 2, MinDelta: 50, Direction: "both"}
 	b := w.BandOf([]float64{100, 100, 100})

internal/threshold/whoopsy.go

@@ -27,6 +27,7 @@ const (
 	WhoopsyWindow      = 7
 	WhoopsyDeviations  = 1
 	WhoopsyConsecutive = 3
+	WhoopsyRelearn     = 5 // a shift lasting this many windows becomes the new normal
 	WhoopsyRuleID      = "whoopsy"
 	WhoopsyName        = "Whoopsy!"
 )