Merge pull request #359 from lunarwhite/expand-variants

CM-159, CM-165: Support e2es to be run in Proxy and Manual OIDC envs

Author: openshift-merge-bot[bot]

Makefile

@@ -104,7 +104,7 @@ COMMIT ?= $(shell git rev-parse HEAD)
 SHORTCOMMIT ?= $(shell git rev-parse --short HEAD)
 GOBUILD_VERSION_ARGS = -ldflags "-X $(PACKAGE)/pkg/version.SHORTCOMMIT=$(SHORTCOMMIT) -X $(PACKAGE)/pkg/version.COMMIT=$(COMMIT)"
 
-E2E_TIMEOUT ?= 1h
+E2E_TIMEOUT ?= 2h
 # E2E_GINKGO_LABEL_FILTER is ginkgo label query for selecting tests. See
 # https://onsi.github.io/ginkgo/#spec-labels. The default is to run tests on the AWS platform.
 E2E_GINKGO_LABEL_FILTER ?= "Platform: isSubsetOf {AWS} && CredentialsMode: isSubsetOf {Mint}"

test/e2e/config_template.go

@@ -27,6 +27,9 @@ type IstioCSRGRPCurlJobConfig struct {
 	IstioCSRStatus            v1alpha1.IstioCSRStatus
 	ClusterID                 string
 	JobName                   string
+	HTTPProxy                 string
+	HTTPSProxy                string
+	NoProxy                   string
 }
 
 // ServiceMonitorConfig customizes fields in the ServiceMonitor spec

test/e2e/issuer_acme_dns01_test.go

@@ -493,9 +493,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() {
 
 			By("calculating parent domain from base domain")
 			parts := strings.Split(baseDomain, ".")
-			if len(parts) <= 1 {
-				Skip("Cannot derive parent domain from base domain: " + baseDomain)
-			}
+			Expect(len(parts)).To(BeNumerically(">", 1), "cannot derive parent domain from base domain")
 			parentDomain := strings.Join(parts[1:], ".")
 
 			By("getting Route53 hosted zone ID from DNS object")
@@ -598,7 +596,16 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() {
 			isSTS, err := isSTSCluster(ctx, oseOperatorClient, configClient)
 			Expect(err).NotTo(HaveOccurred())
 			if !isSTS {
-				Skip("Tests requires AWS Security Token Service enabled")
+				Skip("Test requires AWS Security Token Service enabled")
+			}
+
+			By("setting up AWS authentication environment variable from credentials file")
+			if os.Getenv("OPENSHIFT_CI") == "true" {
+				clusterProfileDir := os.Getenv("CLUSTER_PROFILE_DIR")
+				Expect(clusterProfileDir).NotTo(BeEmpty(), "CLUSTER_PROFILE_DIR should exist when running in OpenShift CI")
+				os.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(clusterProfileDir, ".awscred"))
+			} else {
+				Expect(os.Getenv("AWS_SHARED_CREDENTIALS_FILE")).NotTo(BeEmpty(), "AWS_SHARED_CREDENTIALS_FILE must be set when running locally")
 			}
 
 			// Get AWS region and determinate partition
@@ -948,7 +955,16 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() {
 			isSTS, err := isSTSCluster(ctx, oseOperatorClient, configClient)
 			Expect(err).NotTo(HaveOccurred())
 			if !isSTS {
-				Skip("Cluster is not workload identity enabled, skipping test")
+				Skip("Test requires GCP Workload Identity enabled")
+			}
+
+			By("setting up GCP authentication environment variable from credentials file")
+			if os.Getenv("OPENSHIFT_CI") == "true" {
+				clusterProfileDir := os.Getenv("CLUSTER_PROFILE_DIR")
+				Expect(clusterProfileDir).NotTo(BeEmpty(), "CLUSTER_PROFILE_DIR should exist when running in OpenShift CI")
+				os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", filepath.Join(clusterProfileDir, "gce.json"))
+			} else {
+				Expect(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")).NotTo(BeEmpty(), "GOOGLE_APPLICATION_CREDENTIALS must be set when running locally")
 			}
 
 			By("creating GCP IAM and CloudResourceManager clients")
@@ -1131,7 +1147,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() {
 					Fail("cisCRN is required for IBM Cloud platform")
 				}
 			} else {
-				Skip("skipping as the cluster does not use IBM Cloud CIS")
+				Skip("Test requires IBM Cloud CIS enabled")
 			}
 
 			By("creating ClusterIssuer with IBM Cloud CIS webhook solver")

test/e2e/istio_csr_test.go

@@ -26,9 +26,6 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-// backOffLimit is the max retries for the Job
-const backOffLimit int32 = 10
-
 // istioCSRProtoURL links to proto for istio-csr API spec
 const istioCSRProtoURL = "https://raw.githubusercontent.com/istio/api/v1.24.1/security/v1alpha1/ca.proto"
 
@@ -44,6 +41,7 @@ type IstioCSRConfig struct {
 var _ = Describe("Istio-CSR", Ordered, Label("Feature:IstioCSR"), func() {
 	ctx := context.TODO()
 	var clientset *kubernetes.Clientset
+	var httpProxy, httpsProxy, noProxy string
 
 	generateCSR := func() string {
 		csrTemplate := &x509.CertificateRequest{
@@ -87,6 +85,10 @@ var _ = Describe("Istio-CSR", Ordered, Label("Feature:IstioCSR"), func() {
 			"OPERATOR_LOG_LEVEL": "5",
 		})
 		Expect(err).NotTo(HaveOccurred())
+
+		By("getting cluster proxy configuration")
+		httpProxy, httpsProxy, noProxy, err = getClusterProxyConfig(ctx, configClient)
+		Expect(err).Should(BeNil(), "failed to get cluster proxy config")
 	})
 
 	var ns *corev1.Namespace
@@ -186,6 +188,9 @@ var _ = Describe("Istio-CSR", Ordered, Label("Feature:IstioCSR"), func() {
 				IstioCSRGRPCurlJobConfig{
 					CertificateSigningRequest: csr,
 					IstioCSRStatus:            istioCSRStatus,
+					HTTPProxy:                 httpProxy,
+					HTTPSProxy:                httpsProxy,
+					NoProxy:                   noProxy,
 				},
 			), filepath.Join("testdata", "istio", "grpcurl_job.yaml"), ns.Name)
 			DeferCleanup(func() {
@@ -263,6 +268,9 @@ var _ = Describe("Istio-CSR", Ordered, Label("Feature:IstioCSR"), func() {
 					IstioCSRStatus:            istioCSRStatus,
 					ClusterID:                 clusterName, // matches the IstioCSR resource
 					JobName:                   grpcAppName,
+					HTTPProxy:                 httpProxy,
+					HTTPSProxy:                httpsProxy,
+					NoProxy:                   noProxy,
 				},
 			), filepath.Join("testdata", "istio", "grpcurl_job_with_cluster_id.yaml"), ns.Name)
 			DeferCleanup(func() {
@@ -333,6 +341,9 @@ var _ = Describe("Istio-CSR", Ordered, Label("Feature:IstioCSR"), func() {
 					IstioCSRStatus:            istioCSRStatus,
 					ClusterID:                 "wrong-cluster-id", // doesn't match the IstioCSR resource
 					JobName:                   grpcAppName,
+					HTTPProxy:                 httpProxy,
+					HTTPSProxy:                httpsProxy,
+					NoProxy:                   noProxy,
 				},
 			), filepath.Join("testdata", "istio", "grpcurl_job_with_cluster_id.yaml"), ns.Name)
 			DeferCleanup(func() {

test/e2e/suite_test.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -93,6 +94,11 @@ func TestAll(t *testing.T) {
 
 	suiteConfig, reportConfig := GinkgoConfiguration()
 
+	suiteConfig.Timeout = 120 * time.Minute // Set Ginkgo suite-level timeout
+	suiteConfig.FailFast = false            // Continue after first failure to see all issues
+	suiteConfig.FlakeAttempts = 0           // Retry on flaky tests (helpful when deflaking tests)
+	suiteConfig.MustPassRepeatedly = 1      // Must pass repeatedly times (helpful when deflaking tests)
+
 	testDir := getTestDir()
 	reportConfig.JSONReport = filepath.Join(testDir, "report.json")
 	reportConfig.JUnitReport = filepath.Join(testDir, "junit.xml")
@@ -146,6 +152,6 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 
 	By("setting defaultNetworkPolicy to true")
-	err = resetCertManagerNetworkPolicyState(context.TODO(), certmanageroperatorclient, loader)
+	err = resetCertManagerNetworkPolicyState(context.TODO(), certmanageroperatorclient)
 	Expect(err).NotTo(HaveOccurred())
 })

test/e2e/testdata/istio/grpcurl_job.yaml

@@ -34,6 +34,12 @@ spec:
               value: /tmp/go-cache
             - name: GOPATH
               value: /tmp/go
+            - name: HTTP_PROXY
+              value: {{.HTTPProxy}}
+            - name: HTTPS_PROXY
+              value: {{.HTTPSProxy}}
+            - name: NO_PROXY
+              value: {{.NoProxy}}
           image: registry.redhat.io/rhel9/go-toolset
           name: grpcurl
           volumeMounts:

test/e2e/testdata/istio/grpcurl_job_with_cluster_id.yaml

@@ -35,6 +35,12 @@ spec:
               value: /tmp/go-cache
             - name: GOPATH
               value: /tmp/go
+            - name: HTTP_PROXY
+              value: {{.HTTPProxy}}
+            - name: HTTPS_PROXY
+              value: {{.HTTPSProxy}}
+            - name: NO_PROXY
+              value: {{.NoProxy}}
           image: registry.redhat.io/rhel9/go-toolset
           name: grpcurl
           volumeMounts:

test/e2e/utils_test.go

@@ -850,7 +850,7 @@ func verifyCertificateRenewed(ctx context.Context, secretName, namespace string,
 		}
 
 		// checks if expiry time was updated
-		if *initExpiryTime == cert.NotAfter {
+		if (*initExpiryTime).Equal(cert.NotAfter) {
 			return false, nil
 		}
 
@@ -1124,7 +1124,7 @@ func VerifyContainerResources(pod corev1.Pod, containerName string, expectedReso
 }
 
 // resetCertManagerNetworkPolicyState resets the CertManager to have defaultNetworkPolicy="true"
-func resetCertManagerNetworkPolicyState(ctx context.Context, client *certmanoperatorclient.Clientset, loader library.DynamicResourceLoader) error {
+func resetCertManagerNetworkPolicyState(ctx context.Context, client *certmanoperatorclient.Clientset) error {
 	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		var operatorState *v1alpha1.CertManager
 		err := wait.PollUntilContextTimeout(context.TODO(), slowPollInterval, highTimeout, true, func(context.Context) (bool, error) {
@@ -1770,3 +1770,14 @@ func getSAToken(ctx context.Context, saName, namespace string) (string, error) {
 
 	return result.Status.Token, nil
 }
+
+// getClusterProxyConfig retrieves the cluster-wide proxy configuration
+// Returns httpProxy, httpsProxy, noProxy values from the cluster Proxy resource
+func getClusterProxyConfig(ctx context.Context, client configv1.ConfigV1Interface) (httpProxy, httpsProxy, noProxy string, err error) {
+	proxy, err := client.Proxies().Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return "", "", "", fmt.Errorf("failed to get cluster proxy config: %w", err)
+	}
+
+	return proxy.Status.HTTPProxy, proxy.Status.HTTPSProxy, proxy.Status.NoProxy, nil
+}