diff --git a/.github/actions/kyverno-wait-ready/action.yaml b/.github/actions/kyverno-wait-ready/action.yaml new file mode 100644 index 0000000..edf36e3 --- /dev/null +++ b/.github/actions/kyverno-wait-ready/action.yaml @@ -0,0 +1,10 @@ +name: Kyverno pods ready + +description: Wait kyverno pods are ready + +runs: + using: composite + steps: + - shell: bash + run: | + kubectl wait --namespace kyverno --for=condition=ready pod --selector '!job-name' --timeout=60s diff --git a/.github/actions/setup-test-env/action.yaml b/.github/actions/setup-test-env/action.yaml new file mode 100644 index 0000000..8c74f4c --- /dev/null +++ b/.github/actions/setup-test-env/action.yaml @@ -0,0 +1,29 @@ +name: Setup test env + +description: Create kind cluster, deploy kyverno, and wait pods are ready. + +inputs: + version: + description: kubernetes version + default: v1.27.3 + free-disk-space: + description: free disk space + default: 'false' + +runs: + using: composite + steps: + - uses: jlumbroso/free-disk-space@76866dbe54312617f00798d1762df7f43def6e5c # v1.2.0 + if: ${{ inputs.free-disk-space == 'true' }} + with: + tool-cache: true + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: false + - shell: bash + run: | + export KIND_IMAGE=kindest/node:${{ inputs.version }} + make kind-create-cluster kind-deploy-kyverno + - uses: ./.github/actions/kyverno-wait-ready diff --git a/.github/workflows/load-test.yaml b/.github/workflows/load-test.yaml new file mode 100644 index 0000000..634a056 --- /dev/null +++ b/.github/workflows/load-test.yaml @@ -0,0 +1,53 @@ +name: load-test + +permissions: {} + +on: + pull_request: + branches: + - 'main' + - 'add_threshold' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-load-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - name: Setup build env + uses: ./.github/actions/setup-test-env + timeout-minutes: 10 + - name: Wait for kyverno ready + uses: ./.github/actions/kyverno-wait-ready + - name: Run local k6 test + shell: bash + run: | + cd k6 + export VUS=10 + export ITERATIONS=1000 + export SCRIPT=kyverno-pss.js + ./start.sh tests/$SCRIPT $VUS $ITERATIONS + + grep "level=error" "$SCRIPT-${VUS}vu-${ITERATIONS}it-logs.txt" + # Store the exit code of the grep command + exit_code=$? + + # Check if the exit code is 0 (match found) or 1 (no match found) + if [ $exit_code -eq 0 ]; then + echo "Error found in the file." + exit 1 + elif [ $exit_code -eq 1 ]; then + echo "No error found in the file." + exit 0 + else + echo "An error occurred while searching the file." + exit 1 + fi + cat kyverno-pss.js-10vu-1000it-logs.txt + # uses: grafana/k6-action@v0.3.0 + # with: + # filename: k6/tests/kyverno-pss.js 10 100 \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..934e5ca --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +############ +# DEFAULTS # +############ + +KIND_IMAGE ?= kindest/node:v1.27.3 +KIND_NAME ?= kind +KIND_CONFIG ?= default + +######### +# TOOLS # +######### + +TOOLS_DIR := $(PWD)/.tools +KIND := $(TOOLS_DIR)/kind +KIND_VERSION := v0.20.0 +HELM := $(TOOLS_DIR)/helm +HELM_VERSION := v3.12.3 +HELM_DOCS := $(TOOLS_DIR)/helm-docs +HELM_DOCS_VERSION := v1.11.0 + +$(KIND): + @echo Install kind... >&2 + @GOBIN=$(TOOLS_DIR) go install sigs.k8s.io/kind@$(KIND_VERSION) + +$(HELM): + @echo Install helm... >&2 + @GOBIN=$(TOOLS_DIR) go install helm.sh/helm/v3/cmd/helm@$(HELM_VERSION) + +######## +# HELM # +######## + +.PHONY: helm-add-repo # Add Kyverno chart repository +helm-add-repo: $(HELM) + @echo Add kyverno chart... >&2 + @$(HELM) repo add kyverno https://kyverno.github.io/kyverno/ + +.PHONY: helm-install-kyverno +helm-install-kyverno: helm-add-repo ## Install kyverno helm chart + @echo Install kyverno chart... >&2 + @$(HELM) upgrade --install kyverno --namespace kyverno --create-namespace --wait kyverno/kyverno --devel --values ./configs/kyverno/values.yaml + +######## +# KIND # +######## + +.PHONY: kind-create-cluster +kind-create-cluster: $(KIND) ## Create kind cluster + @echo Create kind cluster... >&2 + @$(KIND) create cluster --name $(KIND_NAME) --image $(KIND_IMAGE) --config ./configs/kind/default.yaml + +.PHONY: kind-deploy-kyverno +kind-deploy-kyverno: helm-add-repo helm-install-kyverno ## Deploy kyverno helm chart \ No newline at end of file diff --git a/configs/kind/default.yaml b/configs/kind/default.yaml new file mode 100644 index 0000000..9438061 --- /dev/null +++ b/configs/kind/default.yaml @@ -0,0 +1,36 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +kubeadmConfigPatches: + - |- + kind: ClusterConfiguration + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2382 + scheduler: + extraArgs: + bind-address: 0.0.0.0 + - |- + kind: KubeProxyConfiguration + metricsBindAddress: 0.0.0.0 +nodes: + - role: control-plane + kubeadmConfigPatches: + - |- + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - role: worker + - role: worker + - role: worker diff --git a/configs/kind/tracing.yaml b/configs/kind/tracing.yaml new file mode 100644 index 0000000..598a1af --- /dev/null +++ b/configs/kind/tracing.yaml @@ -0,0 +1,56 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +kubeadmConfigPatches: + - |- + kind: ClusterConfiguration + apiServer: + extraVolumes: + - name: tracing-configuration + hostPath: /opt/kube-apiserver/tracing-configuration.yaml + mountPath: /opt/kube-apiserver/tracing-configuration.yaml + readOnly: true + pathType: File + extraArgs: + tracing-config-file: /opt/kube-apiserver/tracing-configuration.yaml + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2382 + scheduler: + extraArgs: + bind-address: 0.0.0.0 + - |- + kind: KubeProxyConfiguration + metricsBindAddress: 0.0.0.0 + - |- + kind: KubeletConfiguration + featureGates: + KubeletTracing: true + tracing: + endpoint: localhost:4317 + samplingRatePerMillion: 1000000 +nodes: + - role: control-plane + kubeadmConfigPatches: + - |- + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraMounts: + - hostPath: ./scripts/config/kube-apiserver/tracing-configuration.yaml + containerPath: /opt/kube-apiserver/tracing-configuration.yaml + readOnly: true + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - role: worker + - role: worker + - role: worker diff --git a/configs/kind/vap-v1alpha1.yaml b/configs/kind/vap-v1alpha1.yaml new file mode 100644 index 0000000..b6d1c2a --- /dev/null +++ b/configs/kind/vap-v1alpha1.yaml @@ -0,0 +1,40 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +featureGates: + ValidatingAdmissionPolicy: true +runtimeConfig: + admissionregistration.k8s.io/v1alpha1: true +kubeadmConfigPatches: + - |- + kind: ClusterConfiguration + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2382 + scheduler: + extraArgs: + bind-address: 0.0.0.0 + - |- + kind: KubeProxyConfiguration + metricsBindAddress: 0.0.0.0 +nodes: + - role: control-plane + kubeadmConfigPatches: + - |- + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - role: worker + - role: worker + - role: worker diff --git a/configs/kind/vap-v1beta1.yaml b/configs/kind/vap-v1beta1.yaml new file mode 100644 index 0000000..8b9b433 --- /dev/null +++ b/configs/kind/vap-v1beta1.yaml @@ -0,0 +1,41 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +featureGates: + ValidatingAdmissionPolicy: true +runtimeConfig: + admissionregistration.k8s.io/v1beta1: true + admissionregistration.k8s.io/v1alpha1: true +kubeadmConfigPatches: + - |- + kind: ClusterConfiguration + controllerManager: + extraArgs: + bind-address: 0.0.0.0 + etcd: + local: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2382 + scheduler: + extraArgs: + bind-address: 0.0.0.0 + - |- + kind: KubeProxyConfiguration + metricsBindAddress: 0.0.0.0 +nodes: + - role: control-plane + kubeadmConfigPatches: + - |- + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - role: worker + - role: worker + - role: worker diff --git a/configs/kyverno/values.yaml b/configs/kyverno/values.yaml new file mode 100644 index 0000000..cd9d7a1 --- /dev/null +++ b/configs/kyverno/values.yaml @@ -0,0 +1,40 @@ +features: + admissionReports: + enabled: false + omitEvents: + eventTypes: + - PolicyViolation + - PolicyApplied + - PolicyError + - PolicySkipped + +admissionController: + + serviceMonitor: + enabled: true + + container: + image: + tag: release-1.11 + + resources: + limits: + memory: 2Gi + requests: + cpu: 1 + memory: 1Gi + +reportsController: + serviceMonitor: + enabled: true + + container: + image: + tag: release-1.11 + + resources: + limits: + memory: 10Gi + requests: + cpu: 1 + memory: 1Gi \ No newline at end of file diff --git a/k6/tests/kyverno-pss.js b/k6/tests/kyverno-pss.js index ed2ad48..ed6663d 100644 --- a/k6/tests/kyverno-pss.js +++ b/k6/tests/kyverno-pss.js @@ -17,6 +17,13 @@ import { buildKubernetesBaseUrl, generatePod, getParamsWithAuth, getTestNamespac const baseUrl = buildKubernetesBaseUrl(); const namespace = getTestNamespace(); +export const options = { + thresholds: { + http_req_failed: ['rate<0.01'], // http errors should be less than 1% + http_req_duration: [{ threshold: 'p(95)<200' , abortOnFail: true} ], // 95% of requests should be below 200ms + }, +}; + export default function() { const podName = `test-${randomString(8)}`; const pod = generatePod(podName);