Go #964
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2021 ScyllaDB | |
| name: Go | |
| on: | |
| push: | |
| # Restrict the branches to only those we want to promote from. | |
| branches: | |
| - 'master' | |
| - 'v[0-9]+.[0-9]+' | |
| # Act only on temporary tags, the final ones are created by manually promoting | |
| # an existing candidate image, after it goes through QA. We need to be carefull | |
| # to avoid overwritting those, building from the same sources may not result | |
| # in the same image content that was tested, e.g. if a base image changed in the meantime. | |
| tags: | |
| - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+' | |
| - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+' | |
| - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' | |
| pull_request: | |
| branches: | |
| - '**' | |
| types: | |
| - opened | |
| - edited | |
| - reopened | |
| - synchronize | |
| schedule: | |
| - cron: '0 23 * * *' # daily at 11pm | |
| env: | |
| go_version: 1.17.5 | |
| GOPATH: ${{ github.workspace }}/go | |
| git_repo_path: ${{ github.workspace }}/go/src/github.com/scylladb/scylla-operator | |
| image_repo_ref: docker.io/scylladb/scylla-operator | |
| retention_days: 90 | |
| defaults: | |
| run: | |
| shell: bash | |
| working-directory: "./go/src/github.com/scylladb/scylla-operator" | |
| jobs: | |
| verify: | |
| name: Verify | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| - name: Setup go | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Install tools | |
| run: | | |
| set -x | |
| go install github.com/mikefarah/yq/[email protected] | |
| - name: Verify | |
| run: make verify --warn-undefined-variables | |
| verify-deps: | |
| name: Verify dependencies | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| - name: Setup go | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Verify dependencies | |
| run: make verify-deps --warn-undefined-variables | |
| build-and-test: | |
| name: Build and test | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| fetch-depth: 0 | |
| - name: Setup git tags | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| run: ./hack/ci-detect-tags.sh | |
| - name: Setup go | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Build | |
| run: make --warn-undefined-variables | |
| - name: Make sure new unit tests aren't flaky | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| echo "GO_TEST_COUNT=-count=3" | tee -a ${GITHUB_ENV} | |
| - name: Test | |
| run: make test --warn-undefined-variables | |
| images: | |
| name: Build images | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| fetch-depth: 0 # also fetch tags | |
| - name: Setup git tags | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| run: ./hack/ci-detect-tags.sh | |
| - name: Install podman | |
| run: | | |
| set -x | |
| # Install nativelly when we have Ubuntu >= 20.10 | |
| . /etc/os-release | |
| echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list | |
| curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - | |
| sudo apt-get update | |
| sudo apt-get -y install podman | |
| - name: Build image | |
| run: | | |
| set -x | |
| # Avoid podman permission error on Ubuntu 20.04 by using it as root, although it shouldn't be needed. | |
| # Use --format=docker to support SHELL instruction in the Dockerfile. (SHELL didn't make it to OCI spec.) | |
| sudo podman build --format=docker --squash -f ./Dockerfile -t '${{ env.image_repo_ref }}:ci' . | |
| sudo podman images '${{ env.image_repo_ref }}:ci' | |
| sudo podman save '${{ env.image_repo_ref }}:ci' | lz4 - ~/operatorimage.tar.lz4 | |
| - name: Upload image artifact | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: operatorimage.tar.lz4 | |
| path: ~/operatorimage.tar.lz4 | |
| if-no-files-found: error | |
| retention-days: ${{ env.retention_days }} | |
| charts: | |
| name: Build Helm charts | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| fetch-depth: 0 # also fetch tags | |
| - name: Setup git tags | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| run: ./hack/ci-detect-tags.sh | |
| - name: Setup go | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Build Helm charts | |
| run: make helm-build | |
| - name: Upload charts artifacts | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: helm-charts.zip | |
| path: ${{ env.git_repo_path }}/helm/build/* | |
| if-no-files-found: error | |
| retention-days: ${{ env.retention_days }} | |
| test-e2e: | |
| name: Test e2e | |
| runs-on: ubuntu-20.04 | |
| needs: images | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| fetch-depth: 0 # also fetch tags | |
| - name: Setup git tags | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| run: ./hack/ci-detect-tags.sh | |
| - name: Create artifacts dir | |
| env: | |
| ARTIFACTS_DIR: ${{ runner.temp }}/e2e-artifacts | |
| run: | | |
| set -x | |
| mkdir "${ARTIFACTS_DIR}" | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" | tee -a ${GITHUB_ENV} | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: operatorimage.tar.lz4 | |
| path: ~/ | |
| - name: Load image | |
| run: | | |
| set -x | |
| unlz4 ~/operatorimage.tar.lz4 - | docker load | |
| # docker looses the registry part on save/load | |
| docker tag "$( echo "${image_repo_ref}:ci" | sed -E -e 's~[^/]+/(.*)~\1~' )" "${image_repo_ref}:ci" | |
| docker images '${{ env.image_repo_ref }}:ci' | |
| - name: Setup go | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Install tools | |
| run: | | |
| set -x | |
| go install github.com/mikefarah/yq/[email protected] | |
| - name: Setup minikube | |
| uses: ./go/src/github.com/scylladb/scylla-operator/.github/actions/setup-minikube | |
| - name: Deploy scylla-operator | |
| run: | | |
| set -x | |
| timeout 10m ./hack/ci-deploy.sh '${{ env.image_repo_ref }}:ci' | |
| # Raise loglevel in CI. | |
| # TODO: Replace it with ScyllaOperatorConfig field when available. | |
| kubectl -n scylla-operator patch deployment/scylla-operator --type=json -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--loglevel=4"}]' | |
| kubectl -n scylla-operator rollout status deployment/scylla-operator | |
| kubectl get pods -A | |
| - name: Tolerate flakes on promotion jobs | |
| if: ${{ github.event_name != 'pull_request' }} | |
| run: | | |
| echo "FLAKE_ATTEMPTS=5" | tee -a ${GITHUB_ENV} | |
| - name: Run e2e | |
| run: | | |
| set -euExo pipefail | |
| e2e_timeout_minutes=120 | |
| flake_attempts=0 | |
| if [[ -v 'FLAKE_ATTEMPTS' ]]; then | |
| flake_attempts="${FLAKE_ATTEMPTS}" | |
| e2e_timeout_minutes="$(( ${e2e_timeout_minutes} + ${flake_attempts} * 10 ))" | |
| fi | |
| docker run --user="$( id -u ):$( id -g )" --rm \ | |
| --entrypoint=/usr/bin/scylla-operator-tests \ | |
| -v="${ARTIFACTS_DIR}:${ARTIFACTS_DIR}:rw" \ | |
| -v="${HOME}/.kube/config:/kubeconfig:ro" -e='KUBECONFIG=/kubeconfig' \ | |
| '${{ env.image_repo_ref }}:ci' \ | |
| run all \ | |
| --artifacts-dir="${ARTIFACTS_DIR}" \ | |
| --flake-attempts="${flake_attempts}" \ | |
| --timeout="${e2e_timeout_minutes}m" | |
| - name: Dump cluster state | |
| if: ${{ always() }} | |
| working-directory: ${{ runner.temp }} | |
| run: timeout 10m ${{ env.git_repo_path }}/hack/ci-gather-artifacts.sh | |
| - name: Get machine logs and info | |
| if: ${{ always() }} | |
| working-directory: ${{ runner.temp }}/e2e-artifacts | |
| run: | | |
| set -euExo pipefail | |
| shopt -s inherit_errexit | |
| docker info > docker.info | |
| docker images -a > docker.images | |
| docker stats -a --no-stream --no-trunc > docker.stats | |
| free -h > free | |
| journalctl -u kubelet > kubelet.log | |
| sudo tar -c --use-compress-program=lz4 -f ./kubernetes.tar.lz4 "/etc/kubernetes" | |
| mkdir container-logs | |
| for ns in kube-system; do | |
| mkdir "container-logs/${ns}" | |
| for cid in $( sudo crictl ps --label="io.kubernetes.pod.namespace=${ns}" -a -q ); do | |
| cname=$( sudo crictl inspect -o go-template --template='{{ .status.metadata.name }}' "${cid}" ) | |
| sudo crictl logs "${cid}" 1>"container-logs/${ns}/${cname}_${cid}.log" 2>&1 | |
| done | |
| done | |
| - name: Collect audit logs | |
| if: ${{ always() }} | |
| working-directory: ${{ runner.temp }}/e2e-artifacts | |
| run: | | |
| set -euEx -o pipefail | |
| sudo cat $( ls /var/log/kube-apiserver-audit*.log | sort -n ) > ./kube-apiserver-audit.log | |
| jq -s 'group_by(.user.username) | map({"user": .[0].user.username, "total": length, "verbs": (group_by(.verb) | map({"key":.[0].verb, "value": length}) | from_entries)}) | sort_by(.total) | reverse' ./kube-apiserver-audit.log > ./api-call-stats.json | |
| - name: Compress artifacts | |
| if: ${{ always() }} | |
| working-directory: ${{ runner.temp }} | |
| run: | | |
| set -x | |
| tar -c --use-compress-program=lz4 -f ./e2e-artifacts.tar.lz4 "e2e-artifacts/" | |
| - name: Upload artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: e2e-artifacts.tar.lz4 | |
| path: ${{ runner.temp }}/e2e-artifacts.tar.lz4 | |
| if-no-files-found: error | |
| retention-days: ${{ env.retention_days }} | |
| # TODO: Add upgrade-e2e - use the same image sha from images step | |
| # Dummy step for different promotion jobs to depend on | |
| success: | |
| name: All tests successfull | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - verify | |
| - verify-deps | |
| - build-and-test | |
| - images | |
| - charts | |
| - test-e2e | |
| # TODO: Depend on upgrade-e2e when available | |
| steps: | |
| - name: Always succeed | |
| working-directory: . | |
| run: true | |
| promote: | |
| name: Promote artifacts | |
| runs-on: ubuntu-20.04 | |
| needs: [success] | |
| if: ${{ github.event_name != 'pull_request' }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| path: ${{ env.git_repo_path }} | |
| # Helm Chart version needs to be semantic, we need tags in checked out repo to determine latest one. | |
| fetch-depth: 0 | |
| - name: Setup git tags | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| run: ./hack/ci-detect-tags.sh | |
| - name: Setup go | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: ${{ env.go_version }} | |
| - name: Determine promotion tag | |
| if: ${{ github.event_name != 'schedule' }} | |
| run: | | |
| source ./hack/lib/tag-from-gh-ref.sh | |
| CI_IMAGE_TAG=$( tag_from_gh_ref "${GITHUB_REF}" ) | |
| echo "CI_IMAGE_TAG=${CI_IMAGE_TAG}" | tee -a ${GITHUB_ENV} | |
| - name: Determine promotion tag for scheduled job | |
| if: ${{ github.event_name == 'schedule' }} | |
| run: | | |
| CI_IMAGE_TAG=nightly | |
| echo "CI_IMAGE_TAG=${CI_IMAGE_TAG}" | tee -a ${GITHUB_ENV} | |
| echo "HELM_CHART_VERSION_SUFFIX=-${CI_IMAGE_TAG}" | tee -a ${GITHUB_ENV} | |
| - uses: actions/download-artifact@v2 | |
| with: | |
| name: operatorimage.tar.lz4 | |
| path: ~/ | |
| - name: Load image | |
| run: | | |
| set -x | |
| unlz4 ~/operatorimage.tar.lz4 - | docker load | |
| # docker looses the registry part on save/load | |
| docker tag "$( echo "${image_repo_ref}:ci" | sed -E -e 's~[^/]+/(.*)~\1~' )" "${image_repo_ref}:ci" | |
| docker images '${{ env.image_repo_ref }}:ci' | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Promote image | |
| run: | | |
| set -x | |
| docker tag '${{ env.image_repo_ref }}:ci' '${{ env.image_repo_ref }}:${{ env.CI_IMAGE_TAG }}' | |
| docker push '${{ env.image_repo_ref }}:${{ env.CI_IMAGE_TAG }}' | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@master | |
| with: | |
| service_account_key: ${{ secrets.GCP_SA_KEY }} | |
| - name: Publish Helm Chart | |
| env: | |
| HELM_CHANNEL: latest | |
| HELM_APP_VERSION: ${{ env.CI_IMAGE_TAG }} | |
| run: make helm-publish | |
| failure-notifications: | |
| name: Failure notifications | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - success | |
| - promote | |
| if: ${{ failure() && github.event_name != 'pull_request' }} | |
| steps: | |
| - name: Report failures to Slack | |
| if: ${{ always() }} | |
| working-directory: . | |
| run: | | |
| # We have to avoid printing the secret to logs. | |
| set +x | |
| curl -X POST -H 'Content-type: application/json' --data @<( cat <<-EOF | |
| { | |
| "text": ":warning: CI workflow \"${{ github.workflow }}\" triggered on \"${{ github.event_name }}\" event from ${{ github.ref }} (${{ github.sha }}) failed!\n:fire_extinguisher: See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for details.:fire:\ncc: <@U01L8R3RYFN> <@UN90LVATC>" | |
| } | |
| EOF | |
| ) '${{ secrets.SLACK_WEBHOOK_URL }}' |