use pvmigrate rbac in integration tests (#179)

* use pvmigrate rbac in integration tests

* f

* f

* use krew

* f

* create token help

* manually create secret

* add ttl-sh build

* run pvmigrate inside the cluster

* no .git

* welp

* end wait after failure

* why

* entrypoint?

.github/workflows/build-test.yaml

@@ -31,8 +31,12 @@ jobs:
         with:
           go-version: 1.21
 
-      - name: Build
-        run: make build
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          file: testing/Dockerfile
+          tags: ttl.sh/build-test/pvmigrate:latest
 
       - name: Create k8s Kind Cluster
         uses: helm/[email protected]
@@ -41,7 +45,7 @@ jobs:
         run: ./testing/init.sh
 
       - name: Run PVMigrate
-        run: ./bin/pvmigrate --source-sc int-source --dest-sc int-dest
+        run: kubectl apply -f testing/yaml/migrate-job.yaml
 
       - name: Validate Cluster End State
         run: ./testing/validate.sh

Makefile

@@ -2,6 +2,7 @@ SHELL := /bin/bash
 VERSION_PACKAGE = github.com/replicatedhq/pvmigrate/pkg/version
 VERSION ?=`git describe --tags --dirty`
 DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+CURRENT_USER := $(shell id -u -n)
 
 GIT_TREE = $(shell git rev-parse --is-inside-work-tree 2>/dev/null)
 ifneq "$(GIT_TREE)" ""
@@ -56,3 +57,8 @@ build: bin/pvmigrate
 
 bin/pvmigrate: cmd/main.go pkg/migrate/migrate.go pkg/version/version.go
 	go build ${LDFLAGS} -o bin/pvmigrate cmd/main.go
+
+.PHONY: ttl-sh
+ttl-sh:
+	docker build --pull -f testing/Dockerfile -t ttl.sh/${CURRENT_USER}/pvmigrate:${VERSION} .
+	docker push ttl.sh/${CURRENT_USER}/pvmigrate:${VERSION}

testing/Dockerfile

@@ -0,0 +1,17 @@
+
+FROM golang:1.21-alpine AS build
+
+RUN apk add --no-cache make git bash
+
+WORKDIR /go/src/github.com/replicatedhq/pvmigrate
+COPY cmd cmd
+COPY pkg pkg
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY Makefile Makefile
+
+RUN make build
+
+FROM scratch
+COPY --from=build /go/src/github.com/replicatedhq/pvmigrate/bin/pvmigrate /pvmigrate
+ENTRYPOINT ["/pvmigrate"]

testing/init.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set +e
+
 # this waits for a deployment to have all replicas up-to-date and available
 function deployment_fully_updated() {
     x_fully_updated "$1" deployment "$2"
@@ -28,15 +30,15 @@ function x_fully_updated() {
     local updatedReplicas
     updatedReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.updatedReplicas}')
 
-    if [ "$desiredReplicas" != "$availableReplicas" ] ; then
+    if [ "$desiredReplicas" != "$availableReplicas" ]; then
         return 1
     fi
 
-    if [ "$desiredReplicas" != "$readyReplicas" ] ; then
+    if [ "$desiredReplicas" != "$readyReplicas" ]; then
         return 1
     fi
 
-    if [ "$desiredReplicas" != "$updatedReplicas" ] ; then
+    if [ "$desiredReplicas" != "$updatedReplicas" ]; then
         return 1
     fi
 
@@ -103,3 +105,8 @@ spinner_until 120 deployment_fully_updated default short-pvc-name
 echo ""
 echo "'short-pvc-name' deployment healthy"
 
+echo ""
+echo "setting up rbac for the testing service account"
+echo ""
+kubectl apply -f ./rbac.yaml              # the ClusterRole
+kubectl apply -f ./testing/yaml/rbac.yaml # the ClusterRoleBinding and ServiceAccount

testing/validate.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set +e
+
 # this waits for a deployment to have all replicas up-to-date and available
 function deployment_fully_updated() {
     x_fully_updated "$1" deployment "$2"
@@ -28,21 +30,42 @@ function x_fully_updated() {
     local updatedReplicas
     updatedReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.updatedReplicas}')
 
-    if [ "$desiredReplicas" != "$availableReplicas" ] ; then
+    if [ "$desiredReplicas" != "$availableReplicas" ]; then
         return 1
     fi
 
-    if [ "$desiredReplicas" != "$readyReplicas" ] ; then
+    if [ "$desiredReplicas" != "$readyReplicas" ]; then
         return 1
     fi
 
-    if [ "$desiredReplicas" != "$updatedReplicas" ] ; then
+    if [ "$desiredReplicas" != "$updatedReplicas" ]; then
         return 1
     fi
 
     return 0
 }
 
+function job_completed() {
+    local namespace=$1
+    local name=$2
+
+    local succeeded
+    succeeded=$(kubectl get job -n "$namespace" "$name" -o jsonpath='{.status.succeeded}')
+
+    local failed
+    failed=$(kubectl get job -n "$namespace" "$name" -o jsonpath='{.status.failed}')
+
+    if [ "$succeeded" == "1" ]; then
+        return 0
+    fi
+
+    if [ "$failed" == "1" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
 # Run a test every second with a spinner until it succeeds
 function spinner_until() {
     local timeoutSeconds="$1"
@@ -70,6 +93,11 @@ function spinner_until() {
     done
 }
 
+kubectl get pods
+echo "waiting for the pvmigrate job to complete"
+spinner_until 240 job_completed default pvmigrate
+kubectl get pods
+
 kubectl get statefulsets
 kubectl get deployments
 
@@ -89,6 +117,6 @@ kubectl get deployments
 kubectl get pvc
 
 if kubectl get pvc | grep -q int-source; then
-  echo "found PVCs in the int-source namespace"
-  exit 1
+    echo "found PVCs in the int-source namespace"
+    exit 1
 fi

testing/yaml/migrate-job.yaml

@@ -0,0 +1,16 @@
+kind: Job
+apiVersion: batch/v1
+metadata:
+  name: pvmigrate
+  namespace: default
+spec:
+  completions: 1
+  template:
+    spec:
+      serviceAccountName: pvmigrate
+      restartPolicy: Never
+      containers:
+        - name: pvmigrate
+          image: ttl.sh/build-test/pvmigrate:latest
+          command: [ "/pvmigrate" ]
+          args: [ "--source-sc", "int-source", "--dest-sc", "int-dest" ]

testing/yaml/rbac.yaml

@@ -0,0 +1,27 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: pvmigrate
+  namespace: default
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pvmigrate
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pvmigrate
+subjects:
+  - kind: ServiceAccount
+    name: pvmigrate
+    namespace: default
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: pvmigrate-secret
+  annotations:
+    kubernetes.io/service-account.name: pvmigrate
+type: kubernetes.io/service-account-token
+---