diff --git a/internal/engine/evaluate.go b/internal/engine/evaluate.go index 07f0271..c9f6068 100644 --- a/internal/engine/evaluate.go +++ b/internal/engine/evaluate.go @@ -47,12 +47,15 @@ type RuleState struct { // metric, how many results in a row left the band, and the band the last // result was judged against. type WhoopsyState struct { - Metric string `json:"metric"` - Values []float64 `json:"values"` - Hits int `json:"hits"` - Last float64 `json:"last"` // the last value judged, inside the band or not - Level string `json:"level,omitempty"` - Band *threshold.Band `json:"band,omitempty"` + Metric string `json:"metric"` + Values []float64 `json:"values"` + Hits int `json:"hits"` + // Outliers are the latest values that broke the band, kept while the + // band is frozen so a lasting shift can become the new normal. + Outliers []float64 `json:"outliers,omitempty"` + Last float64 `json:"last"` // the last value judged, inside the band or not + Level string `json:"level,omitempty"` + Band *threshold.Band `json:"band,omitempty"` } // State is a check's state machine, stored in check_state. @@ -179,6 +182,7 @@ func Evaluate(cfg Config, prev State, in Input) Outcome { } else { c := *ws c.Values = slices.Clone(ws.Values) + c.Outliers = slices.Clone(ws.Outliers) ws = &c } v := metric(cfg.Metrics, in.Metrics, w.Metric) @@ -197,8 +201,22 @@ func Evaluate(cfg Config, prev State, in Input) Outcome { ws.Last = v // The band is frozen while values break it: an outlier never // enters the window, so a slowdown cannot widen the band and - // resolve itself. Resetting Whoopsy! accepts a new normal. - if !outside { + // resolve itself. Resetting Whoopsy! accepts a new normal, and so + // does a shift that lasts WhoopsyRelearn times the longer of the + // window and the consecutive count: the band is learned again + // from the latest values. + if outside { + ws.Outliers = append(ws.Outliers, v) + if extra := len(ws.Outliers) - w.Window; extra > 0 { + ws.Outliers = ws.Outliers[extra:] + } + if ws.Hits >= threshold.WhoopsyRelearn*max(w.Window, w.Consecutive) && len(ws.Outliers) >= w.Window { + ws.Values, ws.Outliers = ws.Outliers, nil + ws.Hits, ws.Band = 0, nil + outside = false + } + } else { + ws.Outliers = nil ws.Values = append(ws.Values, v) } if extra := len(ws.Values) - w.Window; extra > 0 { diff --git a/internal/engine/evaluate_test.go b/internal/engine/evaluate_test.go index a5040f9..b9d1384 100644 --- a/internal/engine/evaluate_test.go +++ b/internal/engine/evaluate_test.go @@ -254,6 +254,31 @@ func TestWhoopsy(t *testing.T) { } } +// A shift that lasts is learned as the new normal instead of freezing the +// band for good: the incident resolves and the band follows the new level. +func TestWhoopsyRelearn(t *testing.T) { + w := &threshold.Whoopsy{Metric: "total_ms", Window: 7, Deviations: 1, Consecutive: 3, Direction: "above", Severity: "critical"} + s := newSim(t, Config{FailureCount: 3, RecoveryCount: 1, Metrics: []string{"total_ms"}, Whoopsy: w}) + for _, v := range []float64{400, 500, 600, 400, 500, 600, 500} { + s.step(plugin.OK, v) + } + limit := threshold.WhoopsyRelearn * 7 + for range limit - 1 { + s.step(plugin.OK, 1000) + } + if s.count(ActionOpen) != 1 || s.count(ActionResolve) != 0 || s.state.Whoopsy.Band.Upper > 600 { + t.Fatalf("band frozen before the limit: %+v %v", s.state.Whoopsy, s.acts) + } + s.step(plugin.OK, 1000) + if s.count(ActionResolve) != 1 || s.state.Whoopsy.Hits != 0 || len(s.state.Whoopsy.Values) != 7 { + t.Fatalf("the shift was not accepted: %+v %v", s.state.Whoopsy, s.acts) + } + s.step(plugin.OK, 1000) + if b := s.state.Whoopsy.Band; b == nil || b.Mean != 1000 || s.state.Whoopsy.Hits != 0 { + t.Errorf("band did not follow 1000: %+v", s.state.Whoopsy) + } +} + func TestWhoopsyBand(t *testing.T) { w := threshold.Whoopsy{Deviations: 2, MinDelta: 50, Direction: "both"} b := w.BandOf([]float64{100, 100, 100}) diff --git a/internal/threshold/whoopsy.go b/internal/threshold/whoopsy.go index 003b2bb..b14905d 100644 --- a/internal/threshold/whoopsy.go +++ b/internal/threshold/whoopsy.go @@ -27,6 +27,7 @@ const ( WhoopsyWindow = 7 WhoopsyDeviations = 1 WhoopsyConsecutive = 3 + WhoopsyRelearn = 5 // a shift lasting this many windows becomes the new normal WhoopsyRuleID = "whoopsy" WhoopsyName = "Whoopsy!" )