Deps fix

.circleci/config.yml

@@ -3,9 +3,9 @@ version: 2.1
 # Add additional CircleCI Orbs dependencies
 orbs:
   # https://circleci.com/orbs/registry/orb/circleci/kubernetes
-  kubernetes: circleci/kubernetes@1.3.1
+  kubernetes: circleci/kubernetes@2.0.2
   # https://circleci.com/orbs/registry/orb/circleci/helm
-  helm: circleci/helm@3.0.0
+  helm: circleci/helm@3.2.0
   # https://circleci.com/orbs/registry/orb/ccpgames/minikube
   minikube: ccpgames/minikube@0.0.1
 
@@ -27,28 +27,171 @@ jobs:
       - kubernetes/install
       - minikube/minikube-install:
           # https://github.com/kubernetes/minikube/releases
-          version: v1.31.2
+          version: v1.38.1
       - run:
           name: Install Helm v3
           command: curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
       - run:
           name: Create new K8s cluster
           command: minikube start --vm-driver=docker --memory 8192 --cpus 4 --kubernetes-version=<< parameters.kubernetes-version >>
+      - run:
+          name: Add mirror repos for aliased dependencies
+          command: |
+            helm repo add bitnami-mirror https://charts.bitnami.com/bitnami
+            helm repo add valkey-mirror https://valkey.io/valkey-helm/
+            helm repo add mongodb https://mongodb.github.io/helm-charts
       - run:
           name: Update stackstorm-ha chart dependencies
           command: helm dependency update
+      - run:
+          name: Install CRDs first (MongoDB, RabbitMQ, cert-manager)
+          # Install every CRD up front, before any operator. The RabbitMQ operator
+          # manifest also ships cert-manager Certificate/Issuer resources for its
+          # serving+metrics TLS, so cert-manager's CRDs must exist too.
+          command: |
+            helm install community-operator-crds mongodb/community-operator-crds --wait
+            kubectl apply -f https://raw.githubusercontent.com/rabbitmq/cluster-operator/main/config/crd/bases/rabbitmq.com_rabbitmqclusters.yaml
+            kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml
+      - run:
+          name: Wait for CRDs to be established
+          command: |
+            kubectl wait --for=condition=established --timeout=120s \
+              crd/mongodbcommunity.mongodbcommunity.mongodb.com \
+              crd/rabbitmqclusters.rabbitmq.com \
+              crd/certificates.cert-manager.io \
+              crd/issuers.cert-manager.io
+      - run:
+          name: Wait for cert-manager to be ready
+          # cert-manager controllers must be running to issue the RabbitMQ operator's
+          # serving cert before we install that operator.
+          command: |
+            kubectl -n cert-manager rollout status deploy/cert-manager --timeout=180s
+            kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s
+            kubectl -n cert-manager rollout status deploy/cert-manager-cainjector --timeout=180s
+      - run:
+          name: Install MongoDB Community Operator (CI)
+          # Install into its OWN namespace: the chart also renders a
+          # `mongodb-database` ServiceAccount/Role, which would otherwise collide
+          # in `default` with the one the stackstorm-ha release owns (its mongod
+          # pods use that SA). CRDs are installed above (community-operator-crds
+          # disabled). watchNamespace="*" still reconciles the CR in `default`.
+          command: helm install mongodb-operator mongodb/community-operator
+            --namespace mongodb-operator --create-namespace
+            --set community-operator-crds.enabled=false
+            --set operator.watchNamespace="*"
+            --wait
+      - run:
+          name: Install RabbitMQ Cluster Operator (CI)
+          # CRD already applied above; this adds the operator Deployment plus its
+          # cert-manager-issued certs. Retry to ride out cert-manager webhook warmup
+          # (Certificate/Issuer admission can briefly refuse connections).
+          command: |
+            for i in 1 2 3 4 5 6; do
+              kubectl apply -f https://github.com/rabbitmq/cluster-operator/releases/latest/download/cluster-operator.yml && break
+              echo "apply failed (attempt $i), retrying in 10s..."
+              sleep 10
+            done
+      - run:
+          name: Wait for operators to be ready
+          command: |
+            kubectl -n mongodb-operator rollout status deploy/mongodb-kubernetes-operator --timeout=180s
+            kubectl -n rabbitmq-system rollout status deploy/rabbitmq-cluster-operator --timeout=180s
+      - run:
+          # CI-only fakes so --wait installs can render/boot. Production must instead
+          # set real valkey auth, real rabbitmq.auth.existingSecret, and real MongoDB
+          # user secrets (mongodbCommunity.auth.{st2User,adminUser}.existingSecret).
+          name: Create fake credentials secrets (CI only)
+          command: |
+            kubectl create secret generic st2-rabbitmq-ci \
+              --from-literal=username=st2-rabbitmq \
+              --from-literal=password=ci-fake-password \
+              --from-literal=erlangCookie=ci-fake-erlang-cookie \
+              --from-literal=default_user.conf=$'default_user = st2-rabbitmq\ndefault_pass = ci-fake-password'
+            kubectl create secret generic mongodb-st2-user-secret --from-literal=password=ci-fake-password
+            kubectl create secret generic mongodb-admin-user-secret --from-literal=password=ci-fake-password
       - run:
           name: Helm install stackstorm-ha chart
-          command: helm install --timeout 15m0s --debug --wait --name-template stackstorm-ha .
+          command: helm install --timeout 5m0s --debug --wait --name-template stackstorm-ha
+            --set valkey.auth.enabled=false
+            --set rabbitmq.auth.existingSecret=st2-rabbitmq-ci
+            --set mongodbCommunity.members=1
+            --set mongodbCommunity.resources.requests.cpu=100m
+            --set mongodbCommunity.resources.requests.memory=124Mi
+            --set rabbitmq.replicas=1
+            --set rabbitmq.resources.requests.cpu=100m
+            --set rabbitmq.resources.requests.memory=124Mi
+            --set rabbitmq.persistence.enabled=false
+            --set valkey.replica.replicas=1
+            --set valkey.resources.requests.cpu=100m
+            --set valkey.resources.requests.memory=124Mi
+            .
       - run:
           name: Helm test
           command: helm test stackstorm-ha
       - run:
           name: Helm upgrade with RBAC enabled
-          command: helm upgrade --set st2.rbac.enabled=true --timeout 5m0s --debug --wait stackstorm-ha .
+          command: helm upgrade --set st2.rbac.enabled=true --timeout 15m0s --debug --wait
+            --set valkey.auth.enabled=false
+            --set rabbitmq.auth.existingSecret=st2-rabbitmq-ci
+            --set mongodbCommunity.members=1
+            --set mongodbCommunity.resources.requests.cpu=100m
+            --set mongodbCommunity.resources.requests.memory=124Mi
+            --set rabbitmq.replicas=1
+            --set rabbitmq.resources.requests.cpu=100m
+            --set rabbitmq.resources.requests.memory=124Mi
+            --set rabbitmq.persistence.enabled=false
+            --set valkey.replica.replicas=1
+            --set valkey.resources.requests.cpu=100m
+            --set valkey.resources.requests.memory=124Mi
+            stackstorm-ha .
       - run:
           name: Helm test with RBAC enabled
           command: helm test stackstorm-ha
+      - run:
+          when: on_fail
+          name: Diagnostics on failure
+          # Fires whenever a prior step fails (e.g. `helm install --wait` timing out
+          # because st2auth never becomes ready). Every command is best-effort so the
+          # dump always completes. Focus: pod state, events, logs, and the datastore
+          # dependencies (mongo/rabbitmq/valkey) that st2 pods block on at startup.
+          command: |
+            set +e
+            echo "==================== helm releases ===================="
+            helm list -A
+            echo "==================== helm status stackstorm-ha ===================="
+            helm status stackstorm-ha --show-resources
+            echo "==================== all pods (wide) ===================="
+            kubectl get pods -A -o wide
+            echo "==================== workloads ===================="
+            kubectl get sts,deploy,rs -A
+            echo "==================== recent events ===================="
+            kubectl get events -A --sort-by=.lastTimestamp | tail -n 80
+            echo "==================== MongoDBCommunity status ===================="
+            kubectl get mongodbcommunity -A -o wide
+            kubectl describe mongodbcommunity -A
+            echo "==================== RabbitmqCluster status ===================="
+            kubectl get rabbitmqclusters -A -o wide
+            kubectl describe rabbitmqclusters -A
+            echo "==================== PVCs / secrets (default) ===================="
+            kubectl get pvc -A
+            kubectl get secret -n default
+            echo "==================== describe + logs: non-running pods in default ===================="
+            for p in $(kubectl get pods -n default --no-headers 2>/dev/null | awk '$3!="Running" && $3!="Completed" {print $1}'); do
+              echo "----- describe pod/$p -----"
+              kubectl describe pod -n default "$p"
+              echo "----- logs pod/$p (all containers) -----"
+              kubectl logs -n default "$p" --all-containers --tail=150 --prefix
+              echo "----- previous logs pod/$p -----"
+              kubectl logs -n default "$p" --all-containers --previous --tail=150 --prefix
+            done
+            echo "==================== st2auth pod logs ===================="
+            for p in $(kubectl get pods -n default --no-headers -o custom-columns=:metadata.name 2>/dev/null | grep -i auth); do
+              echo "----- pod/$p -----"
+              kubectl describe pod -n default "$p"
+              kubectl logs -n default "$p" --all-containers --tail=200 --prefix
+              kubectl logs -n default "$p" --all-containers --previous --tail=200 --prefix
+            done
+            true
       - run:
           when: always
           name: Show created K8s resources
@@ -63,9 +206,9 @@ workflows:
             parameters:
               # https://kubernetes.io/releases
               kubernetes-version:
-                - "v1.28.3"
-                - "v1.27.7"
-                - "v1.26.10"
+                - "v1.37.0"
+                - "v1.36.4"
+                - "v1.35.8"
   # Run periodic nightly Helm tests to ensure there are no regressions
   e2e-nightly:
     jobs:
@@ -74,9 +217,9 @@ workflows:
             parameters:
               # https://kubernetes.io/releases
               kubernetes-version:
-                - "v1.28.3"
-                - "v1.27.7"
-                - "v1.26.10"
+                - "v1.37.0"
+                - "v1.36.4"
+                - "v1.35.8"
     triggers:
       - schedule:
           cron: "0 1 * * *"

.gitignore

@@ -1,3 +1,4 @@
-charts
 *.lock
 .DS_Store
+*.swp
+docs/

Chart.yaml

@@ -1,12 +1,18 @@
 apiVersion: v2
 # StackStorm version which refers to Docker images tag
-appVersion: "3.8"
+appVersion: "3.9"
 name: stackstorm-ha
-version: 1.1.0
-description: StackStorm K8s Helm Chart, optimized for running StackStorm in HA environment.
+version: 1.1.6
+description: >-
+  StackStorm K8s Helm Chart, optimized for running StackStorm in HA environment.
+  This Helm chart is a fully installable app that codifies StackStorm cluster
+  deployment optimized for HA and K8s environment. MongoDB-HA is deployed using
+  the MongoDB Community Operator (must be pre-installed), val-key operator, and
+  rabbitmq operator. For configuration details please check default values.yaml
+  and README.
 home: https://stackstorm.com/
 icon: https://landscape.cncf.io/logos/stack-storm.svg
-source:
+sources:
   - https://github.com/stackstorm/stackstorm-k8s
 keywords:
   - st2
@@ -23,24 +29,12 @@ maintainers:
   - name: The StackStorm Authors
     email: info@stackstorm.com
     url: https://github.com/StackStorm
-details:
-  This Helm chart is a fully installable app that codifies StackStorm cluster deployment optimized for HA and K8s environment.
-  RabbitMQ-HA, MongoDB-HA clusters and Redis coordination backend st2 relies on will be deployed as 3rd party chart dependencies.
-  For configuration details please check default values.yaml and README.
 dependencies:
-  - name: rabbitmq
-    version: 8.0.2
-    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-    condition: rabbitmq.enabled
-  - name: mongodb
-    version: 10.0.1
-    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-    condition: mongodb.enabled
   - name: external-dns
-    version: 4.0.0
-    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+    version: "*"
+    repository: alias:bitnami-mirror
     condition: external-dns.enabled
-  - name: redis
-    version: 12.3.2
-    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-    condition: redis.enabled
+  - name: valkey
+    version: 0.9.3
+    repository: alias:valkey-mirror
+    condition: valkey.enabled