Skip to content

Commit

Permalink
K8s integration test with different k8s versions (spiffe#4376)
Browse files Browse the repository at this point in the history
* Add k8s integration test with multiple versions

Signed-off-by: FedeNQ <[email protected]>
  • Loading branch information
FedeNQ committed Aug 24, 2023
1 parent dd57ddf commit 7076b51
Show file tree
Hide file tree
Showing 8 changed files with 267 additions and 1 deletion.
84 changes: 84 additions & 0 deletions .github/workflows/pr_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,26 @@ jobs:
name: images-windows
path: images-windows.tar.gz

build-matrix:
name: Build matrix
runs-on: ubuntu-20.04
needs: [cache-deps]
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- id: set-matrix
name: Collect versions
run: |
json_array=$(bash ./.github/workflows/scripts/find_k8s.sh)
echo "test=$json_array" >> $GITHUB_OUTPUT
echo "Collected tests: $json_array"
outputs:
test: ${{ steps.set-matrix.outputs.test }}


integration:
name: integration (linux)
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -275,6 +295,70 @@ jobs:
CICD_TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
run: ./.github/workflows/scripts/split.sh | xargs ./test/integration/test.sh



integration-k8s:
name: integration-k8s
runs-on: ubuntu-20.04
needs: [cache-deps, images, build-matrix]
timeout-minutes: 45

permissions:
contents: read

strategy:
fail-fast: false
matrix:
num_runners: [1]
runner_id: [1]
#Test elements should be added as [KubeCTLVersion, K8s-image, KindVersion]
test: ${{ fromJson(needs.build-matrix.outputs.test) }}
steps:
- name: Checkout
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
# The "upgrade" integration test needs the history to ensure
# that the version number in the source code has been bumped as
# expected. This action does not fetch tags unless we supply a
# fetch depth of zero.
fetch-depth: 0
- name: Setup go
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version: ${{ env.GO_VERSION }}
- name: Install regctl
uses: regclient/actions/regctl-installer@b6614f5f56245066b533343a85f4109bdc38c8cc # main
- name: Load cached deps
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Load cached build tools
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: .build
key: ${{ runner.os }}-tools-${{ hashFiles('.go-version','Makefile') }}
- name: Download archived images
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: images
path: .
- name: Load archived images
run: |
tar xvf images.tar.gz
make load-images
- name: Run k8s integration
env:
NUM_RUNNERS: ${{ matrix.num_runners }}
THIS_RUNNER: ${{ matrix.runner_id }}
KUBECTLVERSION: ${{ matrix.test[0] }}
K8SIMAGE: ${{ matrix.test[1] }}
KINDVERSION: ${{ matrix.test[2] }}
TERM: dumb
CICD_TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
run: ./.github/workflows/scripts/split_k8s.sh | xargs ./test/integration/test-k8s.sh


integration-windows:
name: integration (windows)
runs-on: windows-2022
Expand Down
80 changes: 80 additions & 0 deletions .github/workflows/release_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,25 @@ jobs:
name: images-windows
path: images-windows.tar.gz

build-matrix:
name: Build matrix
runs-on: ubuntu-20.04
needs: [cache-deps]
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- id: set-matrix
name: Collect versions
run: |
json_array=$(bash ./.github/workflows/scripts/find_k8s.sh)
echo "test=$json_array" >> $GITHUB_OUTPUT
echo "Collected tests: $json_array"
outputs:
test: ${{ steps.set-matrix.outputs.test }}

integration:
name: integration (linux)
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -272,6 +291,67 @@ jobs:
# CICD_TARGET_BRANCH:
run: ./.github/workflows/scripts/split.sh | xargs ./test/integration/test.sh

integration-k8s:
name: integration-k8s
runs-on: ubuntu-20.04
needs: [cache-deps, images, build-matrix]
timeout-minutes: 45

permissions:
contents: read

strategy:
fail-fast: false
matrix:
num_runners: [1]
runner_id: [1]
#Test elements should be added as [KubeCTLVersion, K8s-image, KindVersion]
test: ${{ fromJson(needs.build-matrix.outputs.test) }}
steps:
- name: Checkout
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
# The "upgrade" integration test needs the history to ensure
# that the version number in the source code has been bumped as
# expected. This action does not fetch tags unless we supply a
# fetch depth of zero.
fetch-depth: 0
- name: Setup go
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
with:
go-version: ${{ env.GO_VERSION }}
- name: Install regctl
uses: regclient/actions/regctl-installer@b6614f5f56245066b533343a85f4109bdc38c8cc # main
- name: Load cached deps
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Load cached build tools
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: .build
key: ${{ runner.os }}-tools-${{ hashFiles('.go-version','Makefile') }}
- name: Download archived images
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: images
path: .
- name: Load archived images
run: |
tar xvf images.tar.gz
make load-images
- name: Run k8s integration
env:
NUM_RUNNERS: ${{ matrix.num_runners }}
THIS_RUNNER: ${{ matrix.runner_id }}
KUBECTLVERSION: ${{ matrix.test[0] }}
K8SIMAGE: ${{ matrix.test[1] }}
KINDVERSION: ${{ matrix.test[2] }}
TERM: dumb
CICD_TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
run: ./.github/workflows/scripts/split_k8s.sh | xargs ./test/integration/test-k8s.sh

integration-windows:
name: integration (windows)
runs-on: windows-2022
Expand Down
67 changes: 67 additions & 0 deletions .github/workflows/scripts/find_k8s.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
kind_release_info=$(curl -s https://api.github.com/repos/kubernetes-sigs/kind/releases/latest)
kind_version=$(echo "$kind_release_info" | jq -r '.tag_name')

all_tags=()
# Currently we're taking the first 5 pages of the URL
for ((page=1; page<=5; page++)); do
# Fetch tags for the current page using curl and jq
tags=$(curl -s "https://hub.docker.com/v2/repositories/kindest/node/tags?page=$page" | jq -r '.results[].name')

# Check if the tags variable is empty
if [[ -z "$tags" ]]; then
break
fi

# Append the current page tags to the all_tags array
all_tags+=( "$tags" )
done

readarray -t tags_sorted < <(printf '%s\n' "${all_tags[@]}" | sort -V)

lowest_target_version=$(cat ./test/integration/suites/k8s/integration_k8s_min_version.txt)

declare -A tags_map
for element in "${tags_sorted[@]}"; do
# Element is in this form: "X.XX.YY"
# If not, continue
num_dots=$(echo "$element" | grep -o '\.' | wc -l)

# Continue to the next iteration if the number of dots is not equal to 2
if [[ "$num_dots" -ne 2 ]]; then
continue
fi

# Extract the "X.XX" part as the key for the map
key="${element%.*}"
key="${key//\"}"
# Check if the key is greater than or equal to "1.21"
if [[ $(printf "%s\n$lowest_target_version" "$key" | sort -V | head -n1) == "$lowest_target_version" ]]; then
# Extract the "YY" part as the value for the map
value="${element##*.}"
tags_map["$key"]=$value
fi
done

# Read the content of the array.txt file
# Currently we just have one row as example, add more if we need to test a specific version
# Test elements should be added as [KubeCTLVersion, K8s-image, KindVersion]
IFS= readarray -t matrix_lines < ./test/integration/suites/k8s/integration_k8s_versions.txt

# Convert each line of the file into a JSON array element
json_array="["
for line in "${matrix_lines[@]}"; do
json_array+="$line,"
done

# Add every version from tags_map
for key in "${!tags_map[@]}"; do
value="${tags_map[$key]}"
k8s_image="kindest/node:$key.$value"
new_version_row="[\"$key.$value\",\"$k8s_image\",\"$kind_version\"]"
json_array+="$new_version_row,"
done

json_array="${json_array%,}]"

echo "${json_array}"
26 changes: 26 additions & 0 deletions .github/workflows/scripts/split_k8s.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/bash

set -e

if [ -z "$NUM_RUNNERS" ]; then
echo "split.sh: NUM_RUNNERS environment variable must be set"
exit 1
fi

if [ -z "$THIS_RUNNER" ]; then
echo "split.sh: THIS_RUNNER environment variable must be set"
exit 1
fi

declare -a job_set
current_runner=1
for FILE in test/integration/suites/k8s*; do
job_set[$current_runner]+="${FILE##test/integration/} "

((current_runner++))
if [ $current_runner -gt "$NUM_RUNNERS" ]; then
current_runner=1
fi
done

echo "${job_set[$THIS_RUNNER]}"
1 change: 0 additions & 1 deletion test/integration/common
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,6 @@ download-kind() {
elif [ "${ARCH}" = "aarch64" ]; then
ARCH=arm64
fi

KINDURL="https://github.com/kubernetes-sigs/kind/releases/download/$KINDVERSION/kind-$UNAME-$ARCH"

local kind_path=$1
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
v1.21
1 change: 1 addition & 0 deletions test/integration/suites/k8s/integration_k8s_versions.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
["v1.20.15","kindest/node:v1.20.15","v0.12.0"]
8 changes: 8 additions & 0 deletions test/integration/test-k8s.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

cd "${DIR}" || fail-now "Unable to change to script directory"

export SUITES=suites/k8s/*
./test.sh $1

0 comments on commit 7076b51

Please sign in to comment.