fix some test flakes 1-009 1-012 1-030 & 1-067

test/openshift/e2e/ginkgo/fixture/statefulset/fixture.go

@@ -61,6 +61,23 @@ func HaveReadyReplicas(readyReplicas int) matcher.GomegaMatcher {
 	})
 }
 
+// ready replicas stay green mid-rollout; this waits for the new revision
+func HaveCompletedRollout(replicas int) matcher.GomegaMatcher {
+	return fetchStatefulSet(func(ss *appsv1.StatefulSet) bool {
+		GinkgoWriter.Println("StatefulSet HaveCompletedRollout:",
+			"ready:", ss.Status.ReadyReplicas,
+			"updated:", ss.Status.UpdatedReplicas,
+			"currentRev:", ss.Status.CurrentRevision,
+			"updateRev:", ss.Status.UpdateRevision,
+			"gen:", ss.Generation, "observed:", ss.Status.ObservedGeneration)
+		return int(ss.Status.ReadyReplicas) == replicas &&
+			int(ss.Status.UpdatedReplicas) == replicas &&
+			ss.Status.CurrentRevision != "" &&
+			ss.Status.CurrentRevision == ss.Status.UpdateRevision &&
+			ss.Generation == ss.Status.ObservedGeneration
+	})
+}
+
 func GetTemplateSpecInitContainerByName(name string, depl appsv1.StatefulSet) *corev1.Container {
 
 	for idx := range depl.Spec.Template.Spec.InitContainers {

test/openshift/e2e/ginkgo/parallel/1-009_validate-manage-other-namespace_test.go

@@ -87,6 +87,21 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Eventually(defaultClusterConfigSecret).Should(
 				secretFixture.HaveStringDataKeyValue("namespaces", argoCDInRandomNS.Namespace+","+nsTest_1_9_custom.Name))
 
+			By("wait until dest ns has controller rbac before we create the app")
+			Eventually(&rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: nsTest_1_9_custom.Name}}).Should(k8sFixture.ExistByName())
+			Eventually(func() bool {
+				rb := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: nsTest_1_9_custom.Name}}
+				if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(rb), rb); err != nil {
+					return false
+				}
+				for _, subject := range rb.Subjects {
+					if subject.Kind == "ServiceAccount" && subject.Name == "argocd-argocd-application-controller" && subject.Namespace == argoCDInRandomNS.Namespace {
+						return true
+					}
+				}
+				return false
+			}).Should(BeTrue(), "controller rolebinding never pointed at our sa")
+
 			By("creating Argo CD Application targeting the other namespace")
 			app := &argocdv1alpha1.Application{
 				ObjectMeta: metav1.ObjectMeta{Name: "test-1-9-custom", Namespace: argoCDInRandomNS.Namespace},

test/openshift/e2e/ginkgo/parallel/1-030_validate_reencrypt_test.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 
 	argov1beta1api "github.com/argoproj-labs/argocd-operator/api/v1beta1"
 	. "github.com/onsi/ginkgo/v2"
@@ -31,6 +32,7 @@ import (
 	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
 	routeFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/route"
+	secretFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/secret"
 	fixtureUtils "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/utils"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -100,31 +102,11 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Expect(r).Should(routeFixture.HaveTo(routev1.RouteTargetReference{Kind: "Service", Name: "argocd-server", Weight: new(int32(100))}))
 
 			By("verifying the Route was successfully admitted, and ths TLS Secret exists")
-			Eventually(func() bool {
-				if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(r), r); err != nil {
-					GinkgoWriter.Println(err)
-					return false
-				}
-
-				ingressSlice := r.Status.Ingress
-				if ingressSlice == nil || len(ingressSlice) != 1 {
-					return false
-				}
-
-				ingress := ingressSlice[0]
-
-				if ingress.Conditions == nil || len(ingress.Conditions) != 1 {
-					return false
-				}
-
-				condition := ingress.Conditions[0]
-
-				return condition.Status == "True" && condition.Type == routev1.RouteAdmitted
-
-			}).Should(BeTrue(), ".status.ingress.conditions[0] should have status:true and type:admitted")
+			Eventually(r, "3m", "5s").Should(routeFixture.HaveAdmittedIngress())
 
 			secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "argocd-server-tls", Namespace: test_1_30_argo1.Name}}
 			Eventually(secret).Should(k8sFixture.ExistByName())
+			Eventually(secret, "2m", "5s").Should(secretFixture.HaveNonEmptyKeyValue("tls.crt"))
 
 			fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_30_argo1.Name, k8sClient)
 
@@ -141,39 +123,38 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 
 				host := r.Status.Ingress[0].Host
 
-				// Create a custom HTTP transport
 				tr := &http.Transport{
 					TLSClientConfig: &tls.Config{
-						InsecureSkipVerify: true, // Disable TLS certificate verification
+						InsecureSkipVerify: true,
 					},
 				}
 
-				// Create an HTTP client with the custom transport
-				client := &http.Client{Transport: tr}
+				httpClient := &http.Client{Transport: tr, Timeout: 10 * time.Second}
 
-				// Make a GET request
-				resp, err := client.Get("https://" + host)
+				resp, err := httpClient.Get("https://" + host)
 				if err != nil {
 					GinkgoWriter.Println("Error:", err)
 					return false
 				}
 				defer resp.Body.Close()
 
-				// Read the response body
 				body, err := io.ReadAll(resp.Body)
 				if err != nil {
 					GinkgoWriter.Println("Error reading body:", err)
 					return false
 				}
 
-				// Print the response body
-				GinkgoWriter.Println(string(body))
-
+				bodyStr := string(body)
+				GinkgoWriter.Println(bodyStr)
 				GinkgoWriter.Println(r.Status.Ingress, r.Spec.Host)
 
-				return strings.Contains(string(body), "Your browser does not support JavaScript.")
+				if strings.Contains(bodyStr, "Application is not available") {
+					return false
+				}
+
+				return strings.Contains(bodyStr, "Your browser does not support JavaScript.")
 
-			}, "90s", "5s").Should(BeTrue())
+			}, "3m", "5s").Should(BeTrue())
 
 		})
 

test/openshift/e2e/ginkgo/sequential/1-012_validate-managed-by-chain_test.go

@@ -105,6 +105,18 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				Eventually(&rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-server", Namespace: managedNS}}).Should(k8sFixture.ExistByName())
 
 				Eventually(&rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: managedNS}}).Should(k8sFixture.ExistByName())
+				Eventually(func() bool {
+					controllerRB := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: managedNS}}
+					if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(controllerRB), controllerRB); err != nil {
+						return false
+					}
+					for _, subject := range controllerRB.Subjects {
+						if subject.Kind == "ServiceAccount" && subject.Name == "argocd-argocd-application-controller" && subject.Namespace == argoCDRandomNS.Namespace {
+							return true
+						}
+					}
+					return false
+				}).Should(BeTrue(), "controller rolebinding never pointed at our sa")
 
 				rb := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-server", Namespace: managedNS}}
 				Eventually(rb).Should(k8sFixture.ExistByName())
@@ -164,6 +176,14 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			By("validating role/rolebindings are valid for second managed namespace")
 			expectRoleAndRoleBindingAreValidForManagedNamespace(nsTest_1_12_custom2.Name)
 
+			By("wait until the cluster secret lists custom2 before we create the app")
+			Eventually(func() string {
+				if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterSecret), clusterSecret); err != nil {
+					return ""
+				}
+				return string(clusterSecret.Data["namespaces"])
+			}).Should(ContainSubstring(nsTest_1_12_custom2.Name))
+
 			By("validating Argo CD is able to deploy to second managed namespace")
 			app2 := &argocdv1alpha1.Application{
 				ObjectMeta: metav1.ObjectMeta{Name: "test-1-12-custom2", Namespace: argoCDRandomNS.Namespace},
@@ -185,8 +205,8 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			}
 			Expect(k8sClient.Create(ctx, app2)).To(Succeed())
 
-			Eventually(app2, "4m", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
-			Eventually(app2, "4m", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
+			Eventually(app2, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
+			Eventually(app2, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
 
 			By("deleting all Argo CD applications and first managed namespace")
 
@@ -257,8 +277,8 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			Eventually(app, "1m", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusMissing))
 			Eventually(app, "1m", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeUnknown))
 
-			Eventually(app2, "4m", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
-			Eventually(app2, "4m", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
+			Eventually(app2, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
+			Eventually(app2, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
 
 		})
 

test/openshift/e2e/ginkgo/sequential/1-067_validate_redis_secure_comm_no_autotls_ha_test.go

@@ -29,6 +29,7 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture"
 	argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd"
+	configmapFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/configmap"
 	deplFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/deployment"
 	k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s"
 	nodeFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/node"
@@ -113,7 +114,8 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 					statefulSet := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: ss, Namespace: ns.Name}}
 					Eventually(statefulSet, "2m", "5s").Should(k8sFixture.ExistByName(), "StatefulSet "+ss+" did not exist within timeout")
 					Eventually(statefulSet, "3m", "5s").Should(statefulsetFixture.HaveReplicas(replicas), "StatefulSet "+ss+" did not have correct replicas within timeout")
-					Eventually(statefulSet, "6m", "10s").Should(statefulsetFixture.HaveReadyReplicas(replicas), "StatefulSet "+ss+" did not have ready replicas within timeout")
+					//ready replicas can still be the old pods while a rollout is in flight
+					Eventually(statefulSet, "6m", "10s").Should(statefulsetFixture.HaveCompletedRollout(replicas), "ss "+ss+" never finished rolling out")
 				}
 
 			}
@@ -157,11 +159,55 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 			expectComponentsAreRunning()
 
 			By("adding argo cd label to argocd-operator-redis-tls secret")
+			redisHAStatefulSet := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "argocd-redis-ha-server", Namespace: ns.Name}}
+			Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(redisHAStatefulSet), redisHAStatefulSet)).To(Succeed())
+			ssUIDBeforeTLS := redisHAStatefulSet.UID
+
+			oldServer0 := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "argocd-redis-ha-server-0", Namespace: ns.Name}}
+			Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(oldServer0), oldServer0)).To(Succeed())
+			server0UIDBeforeTLS := oldServer0.UID
+
 			_, err = osFixture.ExecCommand("kubectl", "annotate", "secret", "argocd-operator-redis-tls", "argocds.argoproj.io/name=argocd", "-n", ns.Name)
 			Expect(err).ToNot(HaveOccurred())
 
+			//operator recreates the ha configmap with tls, then deletes the ss (a rollout would mix tls and non-tls pods)
+			By("wait until redis ha configmap has tls")
+			redisHAConfigMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "argocd-redis-ha-configmap", Namespace: ns.Name}}
+			Eventually(redisHAConfigMap, "2m", "5s").Should(configmapFixture.HaveStringDataKeyValueContainsSubstring("redis.conf", "tls-port 6379"))
+
+			By("wait until redis ha statefulset is recreated")
+			Eventually(func() bool {
+				if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(redisHAStatefulSet), redisHAStatefulSet); err != nil {
+					return false
+				}
+				return redisHAStatefulSet.UID != ssUIDBeforeTLS
+			}, "2m", "5s").Should(BeTrue(), "redis ha ss was never recreated after tls")
+
 			expectComponentsAreRunning()
 
+			By("wait until server-0 is the new pod")
+			Eventually(func() bool {
+				if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(redisHAStatefulSet), redisHAStatefulSet); err != nil {
+					return false
+				}
+				pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "argocd-redis-ha-server-0", Namespace: ns.Name}}
+				if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod); err != nil {
+					return false
+				}
+				if pod.UID == server0UIDBeforeTLS || pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning {
+					return false
+				}
+				if redisHAStatefulSet.Status.UpdateRevision != "" && pod.Labels["controller-revision-hash"] != redisHAStatefulSet.Status.UpdateRevision {
+					return false
+				}
+				for _, cs := range pod.Status.ContainerStatuses {
+					if cs.Name == "redis" && cs.Ready {
+						return true
+					}
+				}
+				return false
+			}, "2m", "5s").Should(BeTrue(), "server-0 still isn't the post-tls pod")
+
 			By("extracting the contents of /data/conf/redis.conf and checking it contains expected values")
 			expectedRedisConfig := []string{
 				"port 0",
@@ -191,9 +237,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				return nil
 			}
 
-			// First, wait for redis.conf to eventually contain the expected values, then
-			// verify it consistently contains them.
-			Eventually(redisConfHasExpectedValues, "10m", "5s").Should(Succeed())
+			Eventually(redisConfHasExpectedValues, "2m", "5s").Should(Succeed())
 			Consistently(redisConfHasExpectedValues, "30s", "5s").Should(Succeed())
 
 			By("extracting the contents of /data/conf/sentinel.conf and checking it contains expected values")
@@ -234,8 +278,6 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() {
 				return nil
 			}
 
-			// First, wait for sentinel.conf to eventually contain the expected values, then
-			// verify it consistently contains them.
 			Eventually(sentinelConfHasExpectedValues, "2m", "5s").Should(Succeed())
 			Consistently(sentinelConfHasExpectedValues, "30s", "5s").Should(Succeed())