Skip to content

Commit ed35bb6

Browse files
jgwestjgwest
authored
chore: fix intermittent E2E test fail in '1-040_validate_quoted_RBAC_group_names_test' (#1063)
* chore: fix intermittent fail in 1-040 Signed-off-by: Jonathan West <jgwest@gmail.com> * chore: increase timeout on waiting for nginx deployment on 1-027 Signed-off-by: Jonathan West <jgwest@gmail.com> * chore: add debug hooks to additional tests Signed-off-by: Jonathan West <jgwest@gmail.com> * chore: add debug on fail outputs to additional tests Signed-off-by: Jonathan West <jgwest@gmail.com> * chore: add logging of applications to debug hook Signed-off-by: Jonathan West <jgwest@gmail.com> * chore: add additional debug hooks to tests Signed-off-by: Jonathan West <jgwest@gmail.com> --------- Signed-off-by: Jonathan West <jgwest@gmail.com>
1 parent eaaf361 commit ed35bb6

14 files changed

Lines changed: 287 additions & 118 deletions

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ e2e-tests-sequential-ginkgo: ginkgo ## Runs kuttl e2e sequential tests
174174
.PHONY: e2e-tests-parallel-ginkgo ## Runs kuttl e2e parallel tests, (Defaults to 5 runs at a time)
175175
e2e-tests-parallel-ginkgo: ginkgo
176176
@echo "Running GitOps Operator parallel Ginkgo E2E tests..."
177-
$(GINKGO_CLI) -p -v -procs=5 --trace --timeout 30m -r ./test/openshift/e2e/ginkgo/parallel
177+
$(GINKGO_CLI) -p -v -procs=5 --trace --timeout 60m -r ./test/openshift/e2e/ginkgo/parallel
178178

179179
.PHONY: e2e-tests-sequential
180180
e2e-tests-sequential:
@@ -285,7 +285,7 @@ kustomize: ## Download kustomize locally if necessary.
285285
GINKGO_CLI = $(shell pwd)/bin/ginkgo
286286
.PHONY: ginkgo
287287
ginkgo: ## Download ginkgo locally if necessary.
288-
$(call go-get-tool,$(GINKGO_CLI),github.com/onsi/ginkgo/v2/ginkgo@v2.22.2)
288+
$(call go-get-tool,$(GINKGO_CLI),github.com/onsi/ginkgo/v2/ginkgo@v2.25.3)
289289

290290

291291
# go-get-tool will 'go install' any package $2 and install it to $1.

‎test/openshift/e2e/ginkgo/fixture/argocd/fixture.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,11 @@ func RunArgoCDCLI(args ...string) (string, error) {
261261

262262
cmdArgs := append([]string{"argocd"}, args...)
263263

264-
GinkgoWriter.Println("executing command", cmdArgs)
264+
if args[0] == "login" {
265+
GinkgoWriter.Println("executing command: argocd login (...)")
266+
} else {
267+
GinkgoWriter.Println("executing command", cmdArgs)
268+
}
265269

266270
// #nosec G204
267271
cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)

‎test/openshift/e2e/ginkgo/fixture/fixture.go‎

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,8 @@ func CreateNamespace(name string) *corev1.Namespace {
282282
Labels: NamespaceLabels,
283283
}}
284284

285+
By("creating namespace '" + ns.Name + "'")
286+
285287
err := k8sClient.Create(context.Background(), ns)
286288
Expect(err).ToNot(HaveOccurred())
287289

@@ -902,6 +904,9 @@ func OutputDebugOnFail(namespaceParams ...any) {
902904
continue
903905
}
904906

907+
// Uncomment this to output the pod logs from app controller and repo server in ns
908+
// outputAppControllerAndRepoLogsInNamespace(namespace)
909+
905910
GinkgoWriter.Println("")
906911
GinkgoWriter.Println("----------------------------------------------------------------")
907912
GinkgoWriter.Println("'kubectl get deployments -n " + namespace + " -o yaml")
@@ -932,6 +937,17 @@ func OutputDebugOnFail(namespaceParams ...any) {
932937
GinkgoWriter.Println("----------------------------------------------------------------")
933938
}
934939

940+
kubectlOutput, err = osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "get", "applications", "-A", "-o", "yaml")
941+
if err != nil {
942+
GinkgoWriter.Println("unable to output all argo cd statuses", err, kubectlOutput)
943+
} else {
944+
GinkgoWriter.Println("")
945+
GinkgoWriter.Println("----------------------------------------------------------------")
946+
GinkgoWriter.Println("'kubectl get applications -A -o yaml':")
947+
GinkgoWriter.Println(kubectlOutput)
948+
GinkgoWriter.Println("----------------------------------------------------------------")
949+
}
950+
935951
GinkgoWriter.Println("You can skip this debug output by setting 'SKIP_DEBUG_OUTPUT=true'")
936952

937953
}
@@ -981,7 +997,7 @@ func outputPodLog(podSubstring string) {
981997
return
982998
}
983999

984-
// Look specifically for operator pod
1000+
// Look specifically for pod with name
9851001
matchingPods := []corev1.Pod{}
9861002
for idx := range podList.Items {
9871003
pod := podList.Items[idx]
@@ -992,19 +1008,19 @@ func outputPodLog(podSubstring string) {
9921008

9931009
if len(matchingPods) == 0 {
9941010
// This can happen when the operator is not running on the cluster
995-
GinkgoWriter.Println("DebugOutputOperatorLogs was called, but no pods were found.")
1011+
GinkgoWriter.Println("outputPodLog was called looking for substring '" + podSubstring + "', but no pods were found.")
9961012
return
9971013
}
9981014

9991015
if len(matchingPods) != 1 {
1000-
GinkgoWriter.Println("unexpected number of operator pods", matchingPods)
1016+
GinkgoWriter.Println("unexpected number of pods", matchingPods)
10011017
return
10021018
}
10031019

10041020
// Extract operator logs
10051021
kubectlLogOutput, err := osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "logs", "pod/"+matchingPods[0].Name, "manager", "-n", matchingPods[0].Namespace)
10061022
if err != nil {
1007-
GinkgoWriter.Println("unable to extract operator logs", err)
1023+
GinkgoWriter.Println("unable to extract logs for", matchingPods[0].Name, err)
10081024
return
10091025
}
10101026

@@ -1015,14 +1031,49 @@ func outputPodLog(podSubstring string) {
10151031

10161032
GinkgoWriter.Println("")
10171033
GinkgoWriter.Println("----------------------------------------------------------------")
1018-
GinkgoWriter.Println("Log output from operator pod:")
1034+
GinkgoWriter.Println("Log output from pod '" + matchingPods[0].Name + "' in " + matchingPods[0].Namespace + ":")
10191035
for _, line := range lines[startIndex:] {
10201036
GinkgoWriter.Println(">", line)
10211037
}
10221038
GinkgoWriter.Println("----------------------------------------------------------------")
10231039

10241040
}
10251041

1042+
//nolint:unused
1043+
func outputAppControllerAndRepoLogsInNamespace(namespace string) {
1044+
1045+
var podList corev1.PodList
1046+
k8sClient, _ := utils.GetE2ETestKubeClient()
1047+
err := k8sClient.List(context.Background(), &podList, client.InNamespace(namespace))
1048+
Expect(err).ToNot(HaveOccurred())
1049+
for _, pod := range podList.Items {
1050+
1051+
want := strings.Contains(pod.Name, "repo-server") || strings.Contains(pod.Name, "application-controller")
1052+
if !want {
1053+
continue
1054+
}
1055+
1056+
kubectlLogOutput, err := osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "logs", "pod/"+pod.Name, "-n", pod.Namespace)
1057+
if err != nil {
1058+
GinkgoWriter.Println("unable to extract logs for", pod.Name, err)
1059+
return
1060+
}
1061+
1062+
// Output only the last 500 lines
1063+
lines := strings.Split(kubectlLogOutput, "\n")
1064+
1065+
startIndex := max(len(lines)-500, 0)
1066+
1067+
GinkgoWriter.Println("")
1068+
GinkgoWriter.Println("----------------------------------------------------------------")
1069+
GinkgoWriter.Println("Log output from pod '" + pod.Name + "' in " + pod.Namespace + ":")
1070+
for _, line := range lines[startIndex:] {
1071+
GinkgoWriter.Println(">", line)
1072+
}
1073+
GinkgoWriter.Println("----------------------------------------------------------------")
1074+
}
1075+
}
1076+
10261077
func IsUpstreamOperatorTests() bool {
10271078
return false // This function should return true if running from argocd-operator repo, false if running from gitops-operator repo. This is to distinguish between tests in upstream argocd-operator and downstream gitops-operator repos.
10281079
}

‎test/openshift/e2e/ginkgo/parallel/1-025-validate-managed-by-change_test.go‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,14 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
4242
Context("1-025-validate-managed-by-change", func() {
4343

4444
var (
45-
ctx context.Context
46-
k8sClient client.Client
45+
ctx context.Context
46+
k8sClient client.Client
47+
test_1_25_argo1 *corev1.Namespace
48+
test_1_25_argo2 *corev1.Namespace
49+
test_1_25_target *corev1.Namespace
50+
cleanup1 func()
51+
cleanup2 func()
52+
cleanup3 func()
4753
)
4854

4955
BeforeEach(func() {
@@ -52,27 +58,39 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
5258
ctx = context.Background()
5359
})
5460

61+
AfterEach(func() {
62+
63+
fixture.OutputDebugOnFail("test-1-25-argo1", "test-1-25-argo2", "test-1-25-target")
64+
65+
if cleanup3 != nil {
66+
cleanup3()
67+
}
68+
if cleanup2 != nil {
69+
cleanup2()
70+
}
71+
if cleanup1 != nil {
72+
cleanup1()
73+
}
74+
})
75+
5576
It("ensuring that managed-by label can transition between two different Argo CD instances", func() {
5677

5778
By("creating 3 namespaces: 2 contain argo cd instances, and one will be managed by one of those namespaces")
5879

59-
test_1_25_argo1NS, cleanup1 := fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo1")
60-
defer cleanup1()
80+
test_1_25_argo1, cleanup1 = fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo1")
6181

62-
test_1_25_argo2NS, cleanup2 := fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo2")
63-
defer cleanup2()
82+
test_1_25_argo2, cleanup2 = fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo2")
6483

65-
test_1_25_targetNS, cleanup3 := fixture.CreateManagedNamespaceWithCleanupFunc("test-1-25-target", test_1_25_argo1NS.Name)
66-
defer cleanup3()
84+
test_1_25_target, cleanup3 = fixture.CreateManagedNamespaceWithCleanupFunc("test-1-25-target", test_1_25_argo1.Name)
6785

6886
argoCDtest_1_25_argo1 := &argov1beta1api.ArgoCD{
69-
ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo1NS.Name},
87+
ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo1.Name},
7088
Spec: argov1beta1api.ArgoCDSpec{},
7189
}
7290
Expect(k8sClient.Create(ctx, argoCDtest_1_25_argo1)).To(Succeed())
7391

7492
argoCDtest_1_25_argo2 := &argov1beta1api.ArgoCD{
75-
ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo2NS.Name},
93+
ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo2.Name},
7694
Spec: argov1beta1api.ArgoCDSpec{},
7795
}
7896
Expect(k8sClient.Create(ctx, argoCDtest_1_25_argo2)).To(Succeed())
@@ -91,7 +109,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
91109
TargetRevision: "HEAD",
92110
},
93111
Destination: argocdv1alpha1.ApplicationDestination{
94-
Namespace: test_1_25_targetNS.Name,
112+
Namespace: test_1_25_target.Name,
95113
Server: "https://kubernetes.default.svc",
96114
},
97115
Project: "default",
@@ -104,20 +122,20 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
104122
Expect(k8sClient.Create(ctx, app)).To(Succeed())
105123

106124
By("waiting for all pods to be ready in both Argo CD namespaces")
107-
fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo1NS.Name, k8sClient)
108-
fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo2NS.Name, k8sClient)
125+
fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo1.Name, k8sClient)
126+
fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo2.Name, k8sClient)
109127

110128
By("verifying Argo CD Application deployed as expected and is healthy and synced")
111129
Eventually(app, "3m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
112130
Eventually(app, "60s", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
113131

114132
By("update 'test_1_25_target' NS to be managed by the second Argo CD instance, rather than the first")
115-
namespaceFixture.Update(test_1_25_targetNS, func(n *corev1.Namespace) {
133+
namespaceFixture.Update(test_1_25_target, func(n *corev1.Namespace) {
116134
n.Labels["argocd.argoproj.io/managed-by"] = "test-1-25-argo2"
117135
})
118136

119137
By("verifying that RoleBinding in 'test_1_25_target' is updated to the second namespace")
120-
roleBindingIntest_1_25_targetNS := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: test_1_25_targetNS.Name}}
138+
roleBindingIntest_1_25_targetNS := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: test_1_25_target.Name}}
121139
Eventually(roleBindingIntest_1_25_targetNS).Should(rolebindingFixture.HaveSubject(rbacv1.Subject{
122140
Kind: "ServiceAccount",
123141
Name: "argocd-argocd-application-controller",
@@ -142,7 +160,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
142160
TargetRevision: "HEAD",
143161
},
144162
Destination: argocdv1alpha1.ApplicationDestination{
145-
Namespace: test_1_25_targetNS.Name,
163+
Namespace: test_1_25_target.Name,
146164
Server: "https://kubernetes.default.svc",
147165
},
148166
Project: "default",
@@ -155,12 +173,12 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
155173
Expect(k8sClient.Create(ctx, app_argo2)).To(Succeed())
156174

157175
By("First Argo CD instance Application should be unhealthy, because it is no longer managing the namespace")
158-
Eventually(app, "4m", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusMissing))
159-
Eventually(app, "4m", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeUnknown))
176+
Eventually(app, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusMissing))
177+
Eventually(app, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeUnknown))
160178

161179
By("Second Argo CD instance Application should be healthy, because it is now managing the namespace")
162-
Eventually(app_argo2, "60s", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
163-
Eventually(app_argo2, "60s", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
180+
Eventually(app_argo2, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
181+
Eventually(app_argo2, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
164182
})
165183

166184
})

‎test/openshift/e2e/ginkgo/parallel/1-034_validate_webhook_notifications_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ UVwpFuaKz5vTCD36Gmmy/u8y
380380
Eventually(notifConfigMap).Should(k8sFixture.ExistByName())
381381
Eventually(notifConfigMap).Should(configmap.HaveStringDataKeyValueContainsSubstring("template.test-app-created", `{"created":"{{.app.metadata.name}}","type":"{{(call .repo.GetAppDetails).Type}}"}`))
382382

383-
By("creating an Argo CD Application that contains a notificatio annotation, which will trigger the notifications controller")
383+
By("creating an Argo CD Application that contains a notification annotation, which will trigger the notifications controller")
384384
app := &argocdv1alpha1.Application{
385385
ObjectMeta: metav1.ObjectMeta{
386386
Name: "my-app-3",
@@ -419,7 +419,7 @@ UVwpFuaKz5vTCD36Gmmy/u8y
419419

420420
return strings.Contains(out, `{"created":"my-app-3","type":"Directory"}`)
421421

422-
}).Should(BeTrue())
422+
}, "4m", "5s").Should(BeTrue())
423423

424424
})
425425

‎test/openshift/e2e/ginkgo/parallel/1-116_annotation-based-tracking_test.go‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,16 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
4141
Context("1-116_annotation-based-tracking_test", func() {
4242

4343
var (
44-
k8sClient client.Client
45-
ctx context.Context
44+
k8sClient client.Client
45+
ctx context.Context
46+
nsTestDemo1 *corev1.Namespace
47+
nsTestDemo2 *corev1.Namespace
48+
nsAppNS1 *corev1.Namespace
49+
nsAppNS2 *corev1.Namespace
50+
cleanupTestDemo1 func()
51+
cleanupTestDemo2 func()
52+
cleanupAppNS1 func()
53+
cleanupAppNS2 func()
4654
)
4755

4856
BeforeEach(func() {
@@ -53,16 +61,32 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
5361

5462
})
5563

64+
AfterEach(func() {
65+
66+
fixture.OutputDebugOnFail(nsTestDemo1, nsTestDemo2, nsAppNS1, nsAppNS2)
67+
68+
if cleanupAppNS2 != nil {
69+
cleanupAppNS2()
70+
}
71+
if cleanupAppNS1 != nil {
72+
cleanupAppNS1()
73+
}
74+
if cleanupTestDemo2 != nil {
75+
cleanupTestDemo2()
76+
}
77+
if cleanupTestDemo1 != nil {
78+
cleanupTestDemo1()
79+
}
80+
})
81+
5682
It("verifies that when annotation tracking is enabled that Argo CD instance starts and has the annotation tracking value specified in ConfigMap", func() {
5783

5884
By("creating Argo CD instances with annotation+label in two different namespaces")
59-
nsTestDemo1, cleanupFunc := fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-1")
60-
defer cleanupFunc()
85+
nsTestDemo1, cleanupTestDemo1 = fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-1")
6186
Eventually(nsTestDemo1).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
6287

63-
nsTestDemo2, cleanupFunc := fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-2")
88+
nsTestDemo2, cleanupTestDemo2 = fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-2")
6489
Eventually(nsTestDemo2).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
65-
defer cleanupFunc()
6690

6791
argoCDTestDemo1 := &argov1beta1api.ArgoCD{
6892
ObjectMeta: metav1.ObjectMeta{Name: "argocd-instance-demo-1", Namespace: nsTestDemo1.Name},
@@ -87,12 +111,10 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
87111

88112
By("creating 2 more namespaces, each managed by one of the above Argo CD instances")
89113

90-
nsAppNS1, cleanupFunc := fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-1", "argocd-test-demo-1")
91-
defer cleanupFunc()
114+
nsAppNS1, cleanupAppNS1 = fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-1", "argocd-test-demo-1")
92115
Eventually(nsAppNS1).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
93116

94-
nsAppNS2, cleanupFunc := fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-2", "argocd-test-demo-2")
95-
defer cleanupFunc()
117+
nsAppNS2, cleanupAppNS2 = fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-2", "argocd-test-demo-2")
96118
Eventually(nsAppNS2).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
97119

98120
By("creating an Application in each Argo CD instance, targeting one of the namespaces and verifying the deploy succeeds")

0 commit comments

Comments
 (0)