E2E Upgrade Module Test when BtpOperator CR is in Deleting state (#591)

* Set envs for tests

* Set images references accordingly to tags

* Replace script agrs from image tags to image urls

* Log info about module images used in tests

* Deploy base image and set BtpOperator CR in Deleting state

* Upgrade the module and check chart version after upgrade

* Add reusable workflow

* Run workflow on pull request

* Run workflow when creating release

* Trigger workflow on PR status change from draft to ready to review

* Fix input for await_image

* Ensure SAP BTP Service Operator deployment resource version is different after upgrade

* Enhance e2e module tests - add reconciliation check when CR is in Deleting state

* Add timeout for module resource reconciliation check

* Fix log

* Trigger reconciliation by annotating CR

* Wait until operand deployment is deleted before triggering reconciliation

.github/workflows/create-release.yaml

@@ -220,6 +220,15 @@ jobs:
       image-tag: ${{ github.event.inputs.name }}
       image-repo: europe-docker.pkg.dev/kyma-project/prod/btp-manager
 
+  run-e2e-upgrade-while-deleting-tests:
+    name: E2E upgrade while deleting tests
+    needs: wait-for-prow-jobs
+    secrets: inherit
+    uses: "./.github/workflows/run-e2e-upgrade-while-deleting-tests-reusable.yaml"
+    with:
+      image-registry: europe-docker.pkg.dev/kyma-project/prod/btp-manager
+      image-tag: ${{ github.event.inputs.name }}
+
   create-draft:
     name: Create draft release
     needs: bump-sec-scanners-config

.github/workflows/run-e2e-upgrade-while-deleting-tests-reusable.yaml

@@ -0,0 +1,36 @@
+name: Run E2E upgrade while deleting tests on the cluster (reusable)
+on:
+  workflow_call:
+    inputs:
+      image-registry:
+        description: Binary image registry reference
+        required: true
+        type: string
+      image-tag:
+        description: Binary image tag
+        required: true
+        type: string
+jobs:
+  run-e2e-upgrade-while-deleting-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up go environment
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Prepare k3s cluster and docker registry
+        run: "./scripts/testing/k3s-setup.sh --wait"
+
+      - name: Wait for artifacts
+        env:
+          IMAGE_REPO: ${{ inputs.image-registry }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: "./scripts/await_image.sh ${{ inputs.image-tag }}"
+
+      - name: Run tests
+        run: "./scripts/testing/run_e2e_module_upgrade_during_deletion_tests.sh ${{ inputs.image-registry }}:${{ inputs.image-tag }}"

.github/workflows/run-e2e-upgrade-while-deleting-tests.yaml

@@ -0,0 +1,18 @@
+name: Run E2E upgrade while deleting tests
+on:
+  pull_request:
+    branches: [ main ]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+jobs:
+  run-e2e-upgrade-while-deleting-tests:
+    name: E2E upgrade while deleting tests
+    if: ${{ !github.event.pull_request.draft }}
+    secrets: inherit
+    uses: "./.github/workflows/run-e2e-upgrade-while-deleting-tests-reusable.yaml"
+    with:
+      image-registry: europe-docker.pkg.dev/kyma-project/dev/btp-manager
+      image-tag: PR-${{ github.event.number }}

scripts/testing/install_module.sh

@@ -20,8 +20,8 @@ set -E          # needs to be set if we want the ERR trap
 set -o pipefail # prevents errors in a pipeline from being masked
 
 IMAGE_NAME=$1
-YAML_DIR="scripts/testing/yaml"
 CREDENTIALS=$2
+YAML_DIR="scripts/testing/yaml"
 
 # installing prerequisites, on production environment these are present before chart is used
 kubectl apply -f ./deployments/prerequisites.yaml
@@ -52,7 +52,7 @@ kubectl apply -f ${YAML_DIR}/e2e-test-btpoperator.yaml
 while [[ $(kubectl get btpoperators/e2e-test-btpoperator -ojson| jq '.status.conditions[] | select(.type=="Ready") |.status+.reason'|xargs)  != "TrueReconcileSucceeded" ]];
 do echo -e "\n---Waiting for BTP Operator to be ready and reconciled"; sleep 5; done
 
-# verifying whether service instance and service binding resources were created
+# verifying whether service instance and service binding custom resources were created
 echo -e "\n---Checking if serviceinstances and servicebindings CRDs are created"
 
 CRDS=$(kubectl get crds|awk '/(servicebindings|serviceinstances)/{print $1}')

scripts/testing/run_e2e_module_tests.sh

@@ -18,6 +18,7 @@ set -o pipefail # prevents errors in a pipeline from being masked
 
 CREDENTIALS=$1
 YAML_DIR="scripts/testing/yaml"
+SAP_BTP_OPERATOR_DEPLOYMENT_NAME=sap-btp-operator-controller-manager
 
 [[ -z ${GITHUB_RUN_ID} ]] && echo "required variable GITHUB_RUN_ID not set" && exit 1
 
@@ -109,6 +110,30 @@ fi
 
 echo -e "\n--- Deprovisioning safety measures work"
 
+echo -e "\n--- Checking module resources reconciliation when BtpOperator CR is in Deleting state"
+
+echo "Deleting ${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} deployment"
+kubectl delete -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME}
+
+while [[ "$(kubectl get -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} 2>&1)" != *"Error from server (NotFound)"* ]];
+do echo -e "\n--- Waiting for ${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} deployment deletion"; sleep 5; done
+
+echo -e "\n--- Triggering reconciliation by annotating BtpOperator CR"
+kubectl annotate --overwrite -f ${YAML_DIR}/e2e-test-btpoperator.yaml last-manual-reconciliation-timestamp="$(date -u -Iseconds)"
+
+echo -e "\n--- Waiting for reconciliation (${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} deployment existence)"
+SECONDS=0
+TIMEOUT=120
+until kubectl get -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME}
+do
+  if [[ ${SECONDS} -ge ${TIMEOUT} ]]; then
+    echo "timed out after ${TIMEOUT}s" && exit 1
+  fi
+  sleep 5
+done
+
+echo -e "\n--- ${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} deployment has been reconciled"
+
 echo -e "\n--- Adding force delete label"
 kubectl label -f ${YAML_DIR}/e2e-test-btpoperator.yaml force-delete=true
 

scripts/testing/run_e2e_module_upgrade_during_deletion_tests.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# This script has the following arguments:
+#     - link to the upgrade image (optional),
+#     - link to the base image (optional),
+# ./run_e2e_module_upgrade_during_deletion_tests.sh [upgrade-image] [base-image]
+# ./run_e2e_module_upgrade_during_deletion_tests.sh europe-docker.pkg.dev/kyma-project/prod/btp-manager:1.1.2 europe-docker.pkg.dev/kyma-project/prod/btp-manager:1.0.0
+
+# standard bash error handling
+set -o nounset  # treat unset variables as an error and exit immediately.
+set -o errexit  # exit immediately when a command fails.
+set -E          # needs to be set if we want the ERR trap
+set -o pipefail # prevents errors in a pipeline from being masked
+
+REGISTRY=europe-docker.pkg.dev/kyma-project/prod/btp-manager
+SAP_BTP_OPERATOR_DEPLOYMENT_NAME=sap-btp-operator-controller-manager
+BTP_MANAGER_DEPLOYMENT_NAME=btp-manager-controller-manager
+EXPECTED_SAP_BTP_SERVICE_OPERATOR_CHART_VER=$(yq '.version' module-chart/chart/Chart.yaml)
+YAML_DIR="scripts/testing/yaml"
+
+if [[ $# -eq 2 ]]; then
+  # upgrade from one given version to another given version
+  UPGRADE_IMAGE=${1}
+  BASE_IMAGE=${2}
+elif [[ $# -eq 1 ]]; then
+  # upgrade from the latest release to the given version
+  UPGRADE_IMAGE=${1}
+  REPOSITORY=${REPOSITORY:-kyma-project/btp-manager}
+  GITHUB_URL=https://api.github.com/repos/${REPOSITORY}
+  BASE_IMAGE_TAG=$(curl -sS "${GITHUB_URL}/releases/latest" | jq -r '.tag_name')
+  BASE_IMAGE=${REGISTRY}:${BASE_IMAGE_TAG}
+elif [[ $# -eq 0 ]]; then
+  # upgrade from the pre-latest release to the latest release
+  REPOSITORY=${REPOSITORY:-kyma-project/btp-manager}
+  GITHUB_URL=https://api.github.com/repos/${REPOSITORY}
+  UPGRADE_IMAGE_TAG=$(curl -sS "${GITHUB_URL}/releases/latest" | jq -r '.tag_name')
+  BASE_IMAGE_TAG=$(curl -sS "${GITHUB_URL}/tags" | jq -r '.[].name' | grep -A1 "${UPGRADE_IMAGE_TAG}" | grep -v "${UPGRADE_IMAGE_TAG}")
+  UPGRADE_IMAGE=${REGISTRY}:${UPGRADE_IMAGE_TAG}
+  BASE_IMAGE=${REGISTRY}:${BASE_IMAGE_TAG}
+else
+  echo "wrong number of arguments" && exit 1
+fi
+
+echo "--- E2E Module Upgrade Test when BtpOperator CR is in Deleting state"
+echo "-- FROM: ${BASE_IMAGE}"
+echo "-- TO: ${UPGRADE_IMAGE}"
+
+echo -e "\n--- PREPARING ENVIRONMENT"
+
+# deploy base image
+scripts/testing/install_module.sh "${BASE_IMAGE}" dummy
+
+SI_NAME=auditlog-management-si-dummy
+export SI_NAME
+
+echo -e "\n--- Creating ServiceInstance: ${SI_NAME}"
+envsubst <${YAML_DIR}/e2e-test-service-instance.yaml | kubectl apply -f -
+
+echo -e "\n--- Waiting for ServiceInstance existence"
+until kubectl get serviceinstances.services.cloud.sap.com/${SI_NAME}; do sleep 5; done
+
+# set BtpOperator CR in Deleting state
+echo -e "\n--- Deleting BtpOperator CR (setting Deleting state)"
+kubectl delete btpoperators/e2e-test-btpoperator &
+
+echo -e "\n--- Waiting for ServiceInstancesAndBindingsNotCleaned reason"
+while [[ $(kubectl get btpoperators/e2e-test-btpoperator -o json| jq '.status.conditions[] | select(.type=="Ready") |.status+.reason'|xargs) != "FalseServiceInstancesAndBindingsNotCleaned" ]];
+do sleep 5; done
+
+BASE_SAP_BTP_OPERATOR_CHART_VER=$(kubectl get -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} -o jsonpath='{.metadata.labels.chart-version}')
+OLD_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER=$(kubectl get -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} -o jsonpath='{.metadata.resourceVersion}')
+echo -e "\n--- SAP BTP Service Operator chart version before upgrade: ${BASE_SAP_BTP_OPERATOR_CHART_VER}"
+echo -e "\n--- Current ${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} deployment resource version: ${OLD_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER}"
+
+echo -e "\n--- UPGRADING MODULE"
+
+# deploy upgrade image
+echo -e "\n--- Deploying module with image: ${UPGRADE_IMAGE} - invoking make"
+IMG=${UPGRADE_IMAGE} make deploy
+
+echo -e "\n--- Waiting for BTP Manager deployment to be available"
+while [[ $(kubectl get deployment/${BTP_MANAGER_DEPLOYMENT_NAME} -n kyma-system -o 'jsonpath={..status.conditions[?(@.type=="Available")].status}') != "True" ]];
+do sleep 5; done
+
+echo -e "\n--- Expected SAP BTP Service Operator chart version after upgrade: ${EXPECTED_SAP_BTP_SERVICE_OPERATOR_CHART_VER}"
+
+ACTUAL_SAP_BTP_SERVICE_OPERATOR_CHART_VER=""
+ACTUAL_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER=""
+
+echo -e "\n--- Triggering reconciliation by annotating BtpOperator CR"
+kubectl annotate --overwrite -f ${YAML_DIR}/e2e-test-btpoperator.yaml last-manual-reconciliation-timestamp="$(date -u -Iseconds)"
+
+echo -e "\n--- Waiting for SAP BTP Service Operator deployment reconciliation"
+SECONDS=0
+TIMEOUT=120
+while [[ "${ACTUAL_SAP_BTP_SERVICE_OPERATOR_CHART_VER}" != "${EXPECTED_SAP_BTP_SERVICE_OPERATOR_CHART_VER}" || "${ACTUAL_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER}" == "${OLD_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER}" ]]
+do
+  if [[ ${SECONDS} -ge ${TIMEOUT} ]]; then
+    echo "timed out after ${TIMEOUT}s" && exit 1
+  fi
+  ACTUAL_SAP_BTP_SERVICE_OPERATOR_CHART_VER=$(kubectl get -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} -o jsonpath='{.metadata.labels.chart-version}')
+  ACTUAL_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER=$(kubectl get -n kyma-system deployment/${SAP_BTP_OPERATOR_DEPLOYMENT_NAME} -o jsonpath='{.metadata.resourceVersion}')
+  sleep 5
+done
+
+echo -e "\n--- SAP BTP Service Operator deployment has been reconciled"
+echo "-- Current chart version: ${ACTUAL_SAP_BTP_SERVICE_OPERATOR_CHART_VER}"
+echo "-- Current deployment resource version: ${ACTUAL_SAP_BTP_SERVICE_OPERATOR_DEPLOY_RES_VER}"
+
+echo -e "\n--- CLEANING UP"
+
+echo -e "\n--- Adding force delete label"
+kubectl label -f ${YAML_DIR}/e2e-test-btpoperator.yaml force-delete=true
+
+while [[ "$(kubectl get btpoperators/e2e-test-btpoperator 2>&1)" != *"Error from server (NotFound)"* ]];
+do echo -e "\n--- Waiting for BtpOperator CR to be removed"; sleep 5; done
+
+echo -e "\n--- BtpOperator CR has been removed"
+
+echo -e "\n--- Checking if ServiceInstance CRD was removed"
+[[ "$(kubectl get crd serviceinstances 2>&1)" != *"Error from server (NotFound)"* ]] \
+&& echo "ServiceInstance CRD still exists when it should have been removed" && exit 1
+
+echo -e "\n--- ServiceInstance CRD has been removed"
+
+echo -e "\n--- Checking if ServiceBinding CRD was removed"
+[[ "$(kubectl get crd servicebindings 2>&1)" != *"Error from server (NotFound)"* ]] \
+&& echo "ServiceBinding CRD still exists when it should have been removed" && exit 1
+
+echo -e "\n--- ServiceBinding CRD has been removed"
+
+echo -e "\n--- BTP Operator deprovisioning succeeded"
+
+echo -e "\n--- Uninstalling BTP Manager"
+
+# uninstall btp-manager
+make undeploy
+
+#clean up and ignore errors
+kubectl delete -f ./examples/btp-manager-secret.yaml || echo "ignoring failure during secret removal"
+kubectl delete -f ./deployments/prerequisites.yaml || echo "ignoring failure during prerequisites removal"