Merge pull request #2781 from dgoodwin/master

Allow JobTier=candidate in featuregate-test-analyzer with warning

Author: openshift-merge-bot[bot]

tools/codegen/cmd/featuregate-test-analyzer.go

@@ -217,7 +217,7 @@ func (o *FeatureGateTestAnalyzerOptions) Run(ctx context.Context) error {
 			md.Textf("* Tests must be be run on every TechPreview platform (ask for an exception if your feature doesn't support a variant)")
 			md.Textf("* All tests must run at least 14 times on every platform")
 			md.Textf("* All tests must pass at least 95%% of the time")
-			md.Textf("* JobTier must be one of: standard, informing, blocking\n")
+			md.Textf("* JobTier must be one of: standard, informing, blocking, candidate (candidate is allowed but produces a warning as it is not covered by Component Readiness)\n")
 			md.Text("")
 
 			if len(warnings) > 0 {
@@ -375,6 +375,18 @@ func checkIfTestingIsSufficient(featureGate string, testingResults map[JobVarian
 		// Optional variants (like RHEL 10 in 4.22) have non-blocking warnings
 		isOptional := jobVariant.Optional
 
+		// If candidate-tier queries returned results for this variant, emit a warning.
+		// Candidate tier jobs are not covered by the Component Readiness main view and
+		// do not have our standard regression protection mechanisms. The results are still
+		// included in the pass/fail calculation alongside other tiers.
+		if testedVariant.HasCandidateTierResults {
+			results = append(results, ValidationResult{
+				Error: fmt.Errorf("warning: variant %v includes test data from candidate-tier jobs which are not covered by Component Readiness and lack standard regression protection",
+					jobVariant),
+				IsWarning: true,
+			})
+		}
+
 		if len(testedVariant.TestResults) < requiredNumberOfTests {
 			results = append(results, ValidationResult{
 				Error: fmt.Errorf("error: only %d tests found, need at least %d for %q on %v",
@@ -639,7 +651,8 @@ func (a OrderedJobVariants) Less(i, j int) bool {
 type TestingResults struct {
 	JobVariant JobVariant
 
-	TestResults []TestResults
+	TestResults              []TestResults
+	HasCandidateTierResults  bool // true if candidate-tier queries returned any test data
 }
 
 type TestResults struct {
@@ -674,6 +687,7 @@ func validateJobTiers(jobVariant JobVariant) error {
 		"standard":  true,
 		"informing": true,
 		"blocking":  true,
+		"candidate": true,
 	}
 
 	hasValidTier := false
@@ -682,7 +696,7 @@ func validateJobTiers(jobVariant JobVariant) error {
 		if tier != "" {
 			hasValidTier = true
 			if !validTiers[tier] {
-				return fmt.Errorf("invalid JobTier %q in variant %+v - must be one of: standard, informing, blocking", tier, jobVariant)
+				return fmt.Errorf("invalid JobTier %q in variant %+v - must be one of: standard, informing, blocking, candidate", tier, jobVariant)
 			}
 		}
 	}
@@ -838,6 +852,7 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi
 	}
 
 	testNameToResults := map[string]*TestResults{}
+	hasCandidateTierResults := false
 	queries := sippy.QueriesFor(jobVariant.Cloud, jobVariant.Architecture, jobVariant.Topology, jobVariant.NetworkStack, jobVariant.OS, jobVariant.JobTiers, testPattern)
 	release, err := getRelease()
 	if err != nil {
@@ -884,6 +899,10 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi
 			return nil, err
 		}
 
+		if currQuery.TierName == "candidate" && len(testInfos) > 0 {
+			hasCandidateTierResults = true
+		}
+
 		for _, currTest := range testInfos {
 			testResults, ok := testNameToResults[currTest.Name]
 			if !ok {
@@ -912,8 +931,9 @@ func listTestResultForVariant(featureGate string, jobVariant JobVariant) (*Testi
 	}
 
 	jobVariantResults := &TestingResults{
-		JobVariant:  jobVariant,
-		TestResults: nil,
+		JobVariant:              jobVariant,
+		TestResults:             nil,
+		HasCandidateTierResults: hasCandidateTierResults,
 	}
 	testNames := sets.StringKeySet(testNameToResults)
 	for _, testName := range testNames.List() {

tools/codegen/cmd/featuregate-test-analyzer_test.go

@@ -79,6 +79,195 @@ func Test_listTestResultFor(t *testing.T) {
 	}
 }
 
+func Test_validateJobTiers_candidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		variant JobVariant
+		wantErr bool
+	}{
+		{
+			name:    "candidate is valid",
+			variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "candidate"},
+			wantErr: false,
+		},
+		{
+			name:    "candidate with standard is valid",
+			variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "standard,candidate"},
+			wantErr: false,
+		},
+		{
+			name:    "invalid tier still rejected",
+			variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "bogus"},
+			wantErr: true,
+		},
+		{
+			name:    "candidate with invalid tier rejected",
+			variant: JobVariant{Cloud: "aws", Architecture: "amd64", Topology: "ha", JobTiers: "candidate,bogus"},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateJobTiers(tt.variant)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("validateJobTiers() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_checkIfTestingIsSufficient_CandidateVariants(t *testing.T) {
+	sufficientTests := []TestResults{
+		{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
+		{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
+		{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
+		{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
+		{TestName: "test5", TotalRuns: 15, SuccessfulRuns: 15},
+	}
+	insufficientTests := []TestResults{
+		{TestName: "test1", TotalRuns: 15, SuccessfulRuns: 15},
+		{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
+		// Only 2 tests, need 5
+	}
+
+	tests := []struct {
+		name               string
+		featureGate        string
+		testingResults     map[JobVariant]*TestingResults
+		wantBlockingErrors int
+		wantWarnings       int
+	}{
+		{
+			name:        "candidate tier returned results with sufficient tests - warning about component readiness",
+			featureGate: "TestFeature",
+			testingResults: map[JobVariant]*TestingResults{
+				{
+					Cloud:        "aws",
+					Architecture: "amd64",
+					Topology:     "ha",
+				}: {
+					TestResults:             sufficientTests,
+					HasCandidateTierResults: true,
+				},
+			},
+			wantBlockingErrors: 0,
+			wantWarnings:       1, // component readiness warning
+		},
+		{
+			name:        "candidate tier returned results with insufficient tests - blocking error plus warning",
+			featureGate: "TestFeature",
+			testingResults: map[JobVariant]*TestingResults{
+				{
+					Cloud:        "aws",
+					Architecture: "amd64",
+					Topology:     "ha",
+				}: {
+					TestResults:             insufficientTests,
+					HasCandidateTierResults: true,
+				},
+			},
+			wantBlockingErrors: 1, // insufficient tests is still blocking
+			wantWarnings:       1, // component readiness warning
+		},
+		{
+			name:        "no candidate tier results - no warning",
+			featureGate: "TestFeature",
+			testingResults: map[JobVariant]*TestingResults{
+				{
+					Cloud:        "aws",
+					Architecture: "amd64",
+					Topology:     "ha",
+				}: {
+					TestResults:             sufficientTests,
+					HasCandidateTierResults: false,
+				},
+			},
+			wantBlockingErrors: 0,
+			wantWarnings:       0,
+		},
+		{
+			name:        "candidate tier returned results with low pass rate - blocking error plus warning",
+			featureGate: "TestFeature",
+			testingResults: map[JobVariant]*TestingResults{
+				{
+					Cloud:        "aws",
+					Architecture: "amd64",
+					Topology:     "ha",
+				}: {
+					TestResults: []TestResults{
+						{TestName: "test1", TotalRuns: 20, SuccessfulRuns: 18}, // 90%
+						{TestName: "test2", TotalRuns: 15, SuccessfulRuns: 15},
+						{TestName: "test3", TotalRuns: 15, SuccessfulRuns: 15},
+						{TestName: "test4", TotalRuns: 15, SuccessfulRuns: 15},
+						{TestName: "test5", TotalRuns: 15, SuccessfulRuns: 15},
+					},
+					HasCandidateTierResults: true,
+				},
+			},
+			wantBlockingErrors: 1, // low pass rate is still blocking
+			wantWarnings:       1, // component readiness warning
+		},
+		{
+			name:        "mix of variants - one with candidate results one without",
+			featureGate: "TestFeature",
+			testingResults: map[JobVariant]*TestingResults{
+				{
+					Cloud:        "aws",
+					Architecture: "amd64",
+					Topology:     "ha",
+				}: {
+					TestResults:             sufficientTests,
+					HasCandidateTierResults: false,
+				},
+				{
+					Cloud:        "gcp",
+					Architecture: "amd64",
+					Topology:     "ha",
+				}: {
+					TestResults:             sufficientTests,
+					HasCandidateTierResults: true,
+				},
+			},
+			wantBlockingErrors: 0,
+			wantWarnings:       1, // only gcp variant has candidate warning
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			results := checkIfTestingIsSufficient(tt.featureGate, tt.testingResults)
+
+			blockingErrors := 0
+			warnings := 0
+			for _, result := range results {
+				if result.IsWarning {
+					warnings++
+				} else {
+					blockingErrors++
+				}
+			}
+
+			if blockingErrors != tt.wantBlockingErrors {
+				t.Errorf("got %d blocking errors, want %d", blockingErrors, tt.wantBlockingErrors)
+				for _, result := range results {
+					if !result.IsWarning {
+						t.Logf("  Blocking error: %v", result.Error)
+					}
+				}
+			}
+			if warnings != tt.wantWarnings {
+				t.Errorf("got %d warnings, want %d", warnings, tt.wantWarnings)
+				for _, result := range results {
+					if result.IsWarning {
+						t.Logf("  Warning: %v", result.Error)
+					}
+				}
+			}
+		})
+	}
+}
+
 func Test_checkIfTestingIsSufficient_OptionalVariants(t *testing.T) {
 	tests := []struct {
 		name               string

tools/codegen/pkg/sippy/json_types.go

@@ -12,6 +12,7 @@ import (
 type SippyQueryStruct struct {
 	Items        []SippyQueryItem `json:"items"`
 	LinkOperator string           `json:"linkOperator"`
+	TierName     string           `json:"-"` // not serialized, used to track which tier this query is for
 }
 
 type SippyQueryItem struct {
@@ -138,6 +139,7 @@ func QueriesFor(cloud, architecture, topology, networkStack, os, jobTiers, testP
 		queries = append(queries, &SippyQueryStruct{
 			Items:        items,
 			LinkOperator: "and",
+			TierName:     jobTier,
 		})
 	}