DATAGO-73329 | Support for Toleration and GHA pipeline to automate to…

…leration test
SolaceDev · Apr 1, 2024 · ef9204b · ef9204b
1 parent 6b68e51
commit ef9204b
Show file tree

Hide file tree

Showing 32 changed files with 3,603 additions and 4,465 deletions.
diff --git a/.github/workflows/build-test-dev.yml b/.github/workflows/build-test-dev.yml
@@ -6,7 +6,7 @@ on:
       - main
 
 env:
-  VERSION: 1.0.2-dev
+  VERSION: 1.0.3-dev
   IMAGE_NAME: pubsubplus-eventbroker-operator
   VAULT_ADDR: https://vault.maas-vault-prod.solace.cloud:8200
   GCLOUD_PROJECT_ID_DEV: stellar-arcadia-205014
@@ -25,12 +25,17 @@ jobs:
       id-token: write
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version: "1.21"
+        env:
+          GOTOOLCHAIN: go1.21rc3
 
       - name: Check out code
         uses: actions/checkout@v1
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - name: Login to Github Packages
         uses: docker/login-action@v2
@@ -40,19 +45,22 @@ jobs:
           password: ${{ secrets.RELEASE_GITHUB_TOKEN }}
 
       - name: Create k8s Kind Cluster for Unit Tests
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: helm/[email protected]
 
       - name: Create Secret
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         run: |
           kubectl create secret generic regcred --from-file=.dockerconfigjson=${HOME}/.docker/config.json --type=kubernetes.io/dockerconfigjson
 
       - name: Unit Test and Coverage Report.
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         run: |
             mkdir -p reports
             make test-coverage
 
       - name: Upload Test coverage Reports
-        if: ${{ always() }}
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: actions/upload-artifact@v3
         with:
           name: code-coverage-report
@@ -61,11 +69,13 @@ jobs:
             ./reports/cover.html
 
       - name: Go Coverage Badge
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: tj-actions/coverage-badge-go@v2
         with:
           filename: ./reports/coverage.out
 
       - name: Verify Changed files
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: tj-actions/verify-changed-files@v12
         id: verify-changed-files
         with:
@@ -94,9 +104,11 @@ jobs:
 
       - name: Set up Docker Buildx
         id: buildx
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: docker/setup-buildx-action@v2
 
       - name: Checkout SolaceDev/maas-build-actions
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: actions/checkout@v2
         with:
           repository: SolaceDev/maas-build-actions
@@ -107,6 +119,7 @@ jobs:
 
       - name: Retrieve google container registry secrets
         id: docker_registry_secrets
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: hashicorp/[email protected]
         with:
           url: "${{ env.VAULT_ADDR }}"
@@ -120,13 +133,15 @@ jobs:
 
       - name: Log in to gcr development docker registry
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         with:
           registry: gcr.io
           username: _json_key
           password: ${{ steps.docker_registry_secrets.outputs.GCP_DEV_SERVICE_ACCOUNT }}
 
       - name: Build image and push Google Container Registry
         uses: docker/build-push-action@v2
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         with:
           context: ./
           tags: |
@@ -135,6 +150,7 @@ jobs:
 
       - name: Run Vulnerability PreCheck for Prisma
         uses: ./maas-build-actions/.github/actions/prisma-vulnerability-checker
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         with:
           docker_image_to_check: gcr.io/${{ env.GCLOUD_PROJECT_ID_DEV }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           prisma_repository_name: "${{ env.GCLOUD_PROJECT_ID_DEV }}/${{ env.IMAGE_NAME }}"
@@ -146,6 +162,7 @@ jobs:
 
       - name: Run Whitesource Action
         uses: SolaceDev/[email protected]
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         with:
           wssURL: https://saas.whitesourcesoftware.com/agent
           apiKey: ${{ secrets.WSS_API_KEY }}
@@ -155,29 +172,31 @@ jobs:
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         with:
           image-ref: ghcr.io/solacedev/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           format: 'sarif'
           severity: 'CRITICAL,HIGH'
           output: 'trivy-results.sarif'
 
       - name: Uploads Trivy Scan Reports
-        if: ${{ always() }}
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: actions/upload-artifact@v2
         with:
           path: |
             trivy-results.sarif
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         with:
           image-ref: ghcr.io/solacedev/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           format: 'sarif'
           severity: 'CRITICAL,HIGH'
           output: 'trivy-results.sarif'
 
       - name: Uploads Trivy Scan Reports
-        if: ${{ !startsWith(github.ref_name, '1.') }}
+        if: ${{ startsWith(github.ref_name, 'dev1.') && (github.ref_name != 'main') }}
         uses: actions/upload-artifact@v2
         with:
           path: |
@@ -225,3 +244,9 @@ jobs:
     needs: build
     uses: ./.github/workflows/test-broker-chaos-situation.yml
     secrets: inherit
+
+  taints-and-tolerations:
+    # if: ${{ false }}  # disable for now
+    needs: build
+    uses: ./.github/workflows/test-taints-and-toleration.yml
+    secrets: inherit
diff --git a/.github/workflows/prep-release.yml b/.github/workflows/prep-release.yml
@@ -22,9 +22,9 @@ jobs:
 
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19
+          go-version: 1.21
 
       - name: Login to Github Packages
         uses: docker/login-action@v2

diff --git a/.github/workflows/test-broker-chaos-situation.yml b/.github/workflows/test-broker-chaos-situation.yml
@@ -20,6 +20,9 @@ jobs:
 
       - name: Check out code
         uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'

diff --git a/.github/workflows/test-broker-upgrade-update.yml b/.github/workflows/test-broker-upgrade-update.yml
@@ -9,6 +9,9 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v1
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]

diff --git a/.github/workflows/test-full-params-ha.yml b/.github/workflows/test-full-params-ha.yml
@@ -20,6 +20,9 @@ jobs:
 
       - name: Check out code
         uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'

diff --git a/.github/workflows/test-full-params.yml b/.github/workflows/test-full-params.yml
@@ -9,6 +9,9 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v1
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]

diff --git a/.github/workflows/test-helm-upgrade.yml b/.github/workflows/test-helm-upgrade.yml
@@ -9,6 +9,9 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v1
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]

diff --git a/.github/workflows/test-minimal-params-ha.yml b/.github/workflows/test-minimal-params-ha.yml
@@ -20,6 +20,9 @@ jobs:
 
       - name: Check out code
         uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'

diff --git a/.github/workflows/test-minimal-params.yml b/.github/workflows/test-minimal-params.yml
@@ -9,6 +9,11 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v1
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+        env:
+          GOTOOLCHAIN: go1.21rc3
 
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]
@@ -28,6 +33,9 @@ jobs:
           echo "environment-kubeconfig:" ${KUBECONFIG}
           kubectl create ns pubsubplus-operator-system --save-config
           kubectl create secret generic regcred --from-file=.dockerconfigjson=${HOME}/.docker/config.json --type=kubernetes.io/dockerconfigjson -n pubsubplus-operator-system
+          echo "$(ls -lh)"
+          go mod tidy
+          go mod vendor
           make deploy | grep created
           kubectl rollout status deployment pubsubplus-eventbroker-operator -n pubsubplus-operator-system --timeout=30s
           kubectl get crd | grep eventbrokers

diff --git a/.github/workflows/test-taints-and-toleration.yml b/.github/workflows/test-taints-and-toleration.yml
@@ -0,0 +1,77 @@
+name: Integration Test for FULL HA Taints and Toleration Support
+
+on: workflow_call
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+      packages: 'read'
+
+    steps:
+      - name: Set env and tools
+        run: |
+          echo "TESTNAMESPACE=op-test-full-$(date +%s)" >> $GITHUB_ENV
+
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/[email protected]'
+        with:
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          access_token_lifetime: 600s
+
+      - name: Use the GKE Autopilot test cluster
+        uses: 'google-github-actions/[email protected]'
+        with:
+          cluster_name: 'dev-integrationtesting'
+          location: 'us-central1'
+
+      - name: Login to Github Packages
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Testing operator deployment
+        run: |
+          kubectl cluster-info
+          kubectl get pods -n kube-system
+          echo "current-context:" $(kubectl config current-context)
+          echo "environment-kubeconfig:" ${KUBECONFIG}
+          kubectl create ns pubsubplus-operator-system --save-config --dry-run=client -o yaml | kubectl apply -f -
+          kubectl create secret generic regcred --from-file=.dockerconfigjson=${HOME}/.docker/config.json --type=kubernetes.io/dockerconfigjson -n pubsubplus-operator-system --dry-run=client -o yaml | kubectl apply -f -
+          make deploy | grep 'created \| configured'
+          kubectl rollout status deployment pubsubplus-eventbroker-operator -n pubsubplus-operator-system --timeout=240s
+          kubectl get crd | grep eventbrokers
+
+      - name: Testing the Operator - HA Tolerations
+        run: |
+          kubectl create ns $TESTNAMESPACE; kubectl config set-context --current --namespace=$TESTNAMESPACE
+          kubectl apply -f ci/manifests/eventbroker-ha-taints.yaml | grep "ha-toleration-test created"
+          sleep 10 ; kubectl get all
+          kubectl wait pods --selector app.kubernetes.io/instance=test-ha --for condition=Ready --timeout=300s
+          kubectl get po --show-labels -n $TESTNAMESPACE | grep test-ha | grep "1/1"
+          kubectl get po --show-labels -n $TESTNAMESPACE | grep test-ha | grep active=true
+          kubectl get sts ha-toleration-test-pubsubplus-p -o jsonpath='{.spec.template.spec.tolerations}' | grep "value":"primary"
+          kubectl get sts ha-toleration-test-pubsubplus-b -o jsonpath='{.spec.template.spec.tolerations}' | grep "value":"backup"
+          kubectl get sts ha-toleration-test-pubsubplus-m -o jsonpath='{.spec.template.spec.tolerations}' | grep "value":"monitor"        
+          kubectl delete eventbroker test-ha -n $TESTNAMESPACE | grep deleted
+
+      - name: Cleanup
+        if: ${{ always() }}
+        run: |
+          kubectl delete pvc -n $TESTNAMESPACE --all --ignore-not-found
+          kubectl delete ns $TESTNAMESPACE --ignore-not-found
diff --git a/.github/workflows/vulncheck_periodic.yml b/.github/workflows/vulncheck_periodic.yml
@@ -18,9 +18,9 @@ jobs:
       id-token: write
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version: "1.21"
 
       - name: Check out code
         uses: actions/checkout@v1

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20.12 as builder
+FROM golang:1.21 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -21,12 +21,10 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
 
-RUN microdnf install -y curl-minimal-7.76.1-26.el9_3.2 libcurl-minimal-7.76.1-26.el9_3.2
-
 LABEL name="solace/pubsubplus-eventbroker-operator"
 LABEL vendor="Solace Corporation"
-LABEL version="1.0.2"
-LABEL release="1.0.2"
+LABEL version="1.0.3"
+LABEL release="1.0.3"
 LABEL summary="Solace PubSub+ Event Broker Kubernetes Operator"
 LABEL description="The Solace PubSub+ Event Broker Kubernetes Operator deploys and manages the lifecycle of PubSub+ Event Brokers"
 

diff --git a/Makefile b/Makefile
@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION ?= 1.0.2-dev
+VERSION ?= 1.0.3-dev
 
 # API_VERSION defines the API version for the PubSubPlusEventBroker CRD
 API_VERSION ?= v1beta1
@@ -182,7 +182,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v4.5.7
-CONTROLLER_TOOLS_VERSION ?= v0.11.3
+CONTROLLER_TOOLS_VERSION ?= v0.14.0
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize
@@ -198,7 +198,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
-	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240313184151-cb5107b36b64
 
 .PHONY: bundle
 bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files.