Set -1 when Applies=Unknown

Author: hongkailiu

install/0000_90_cluster-version-operator_02_prometheusrule-TechPreviewNoUpgrade.yaml

@@ -3,24 +3,24 @@ kind: PrometheusRule
 metadata:
   labels:
     k8s-app: cluster-version-operator
-  name: cluster-version-operator-tech-preview
+  name: cluster-version-operator-accept-risks
   namespace: openshift-cluster-version
   annotations:
-    kubernetes.io/description: Alerting rules for when cluster-version operator metrics call for administrator attention.
+    kubernetes.io/description: Alerting rules for the feature gate ClusterUpdateAcceptRisks.
     exclude.release.openshift.io/internal-openshift-hosted: "true"
     include.release.openshift.io/self-managed-high-availability: "true"
-    release.openshift.io/feature-set: TechPreviewNoUpgrade
+    release.openshift.io/feature-gate: "ClusterUpdateAcceptRisks"
 spec:
   groups:
   - name: cluster-version-tech-preview
     rules:
-    - alert: OpenShiftUpdateRiskApplies
+    - alert: OpenShiftUpdateRiskMightApply
       annotations:
-        summary: The cluster has been exposed to the conditional update risk for 10 minutes.
-        description: The conditional update risk {{ "{{ $labels.risk }}" }} applies to the cluster, and the cluster update to a version exposed to the risk is not recommended. For more information refer to 'oc adm upgrade'.
-        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/OpenShiftUpdateRiskApplies.md
+        summary: The cluster might have been exposed to the conditional update risk for 10 minutes.
+        description: The conditional update risk {{ "{{ $labels.risk }}" }} might apply to the cluster because of {{ "{{ $labels.reason }}" }}, and the cluster update to a version exposed to the risk is not recommended. For more information refer to 'oc adm upgrade'.
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-monitoring-operator/OpenShiftUpdateRiskMightApply.md
       expr: |
-        max by (namespace, risk) (last_over_time(cluster_version_risk_conditions{job="cluster-version-operator", condition="Applies"}[5m]) == 1)
+        max by (namespace, risk, reason) (abs(last_over_time(cluster_version_risk_conditions{job="cluster-version-operator", condition="Applies"}[5m])) == 1)
       for: 10m
       labels:
         severity: warning

pkg/cvo/metrics.go

@@ -108,8 +108,8 @@ penultimate completed version for 'completed'.
 		}, []string{"name", "condition", "reason"}),
 		clusterVersionRiskConditions: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "cluster_version_risk_conditions",
-			Help: "Report the risk conditions for the cluster version. 0 is False and 1 is True.",
-		}, []string{"condition", "risk"}),
+			Help: "Report the risk conditions for the cluster version. -1 is Unknown, 0 is False and 1 is True.",
+		}, []string{"condition", "risk", "reason"}),
 		clusterOperatorConditionTransitions: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "cluster_operator_condition_transitions",
 			Help: "Reports the number of times that a condition on a cluster operator changes status",
@@ -495,7 +495,7 @@ func (m *operatorMetrics) Describe(ch chan<- *prometheus.Desc) {
 	ch <- m.capability.WithLabelValues("").Desc()
 	ch <- m.clusterOperatorUp.WithLabelValues("", "", "").Desc()
 	ch <- m.clusterOperatorConditions.WithLabelValues("", "", "").Desc()
-	ch <- m.clusterVersionRiskConditions.WithLabelValues("", "").Desc()
+	ch <- m.clusterVersionRiskConditions.WithLabelValues("", "", "").Desc()
 	ch <- m.clusterOperatorConditionTransitions.WithLabelValues("", "").Desc()
 	ch <- m.clusterInstaller.WithLabelValues("", "", "").Desc()
 	ch <- m.clusterVersionOperatorUpdateRetrievalTimestampSeconds.WithLabelValues("").Desc()
@@ -524,9 +524,12 @@ func (m *operatorMetrics) collectConditionalUpdateRisks(ch chan<- prometheus.Met
 				continue
 			}
 
-			g := m.clusterVersionRiskConditions.WithLabelValues(condition.Type, risk.Name)
-			if condition.Status == metav1.ConditionTrue {
+			g := m.clusterVersionRiskConditions.WithLabelValues(condition.Type, risk.Name, condition.Reason)
+			switch condition.Status {
+			case metav1.ConditionTrue:
 				g.Set(1)
+			case metav1.ConditionUnknown:
+				g.Set(-1)
 			}
 			// We do not need to do g.Set(0) as it is done when g is initialized
 			ch <- g

pkg/cvo/metrics_test.go

@@ -1019,7 +1019,7 @@ func Test_collectConditionalUpdateRisks(t *testing.T) {
 				},
 			},
 			expected: []valueWithLabels{{
-				labels: map[string]string{"condition": "Applies", "risk": "RiskX"},
+				labels: map[string]string{"condition": "Applies", "risk": "RiskX", "reason": "ReasonA"},
 			}},
 		},
 		{
@@ -1037,7 +1037,7 @@ func Test_collectConditionalUpdateRisks(t *testing.T) {
 			},
 			expected: []valueWithLabels{{
 				value:  1,
-				labels: map[string]string{"condition": "Applies", "risk": "RiskX"},
+				labels: map[string]string{"condition": "Applies", "risk": "RiskX", "reason": "ReasonA"},
 			}},
 		},
 		{
@@ -1054,7 +1054,8 @@ func Test_collectConditionalUpdateRisks(t *testing.T) {
 				},
 			},
 			expected: []valueWithLabels{{
-				labels: map[string]string{"condition": "Applies", "risk": "RiskX"},
+				value:  -1,
+				labels: map[string]string{"condition": "Applies", "risk": "RiskX", "reason": "ReasonA"},
 			}},
 		},
 	}