diff --git a/Makefile b/Makefile index b324f38c..4d052799 100644 --- a/Makefile +++ b/Makefile @@ -76,12 +76,12 @@ mod-tidy: ## Run go mod tidy against code. test: manifests generate fmt vet envtest ## Run tests. @echo "Check for kubernetes version $(K8S_VERSION_TRIMMED_V) in $(ENVTEST)" @$(ENVTEST) list | grep -q $(K8S_VERSION_TRIMMED_V) - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(K8S_VERSION_TRIMMED_V) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(K8S_VERSION_TRIMMED_V) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -v # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v +test-e2e: ginkgo + $(GINKGO) -v ./test/e2e/ .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint @@ -203,11 +203,11 @@ kind-load: docker-build kind ## Build and upload docker image to the local Kind .PHONY: kind-create kind-create: kind yq ## Create kubernetes cluster using Kind. @if ! $(KIND) get clusters | grep -q $(KIND_CLUSTER_NAME); then \ - $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION); \ + $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION) --config=test/e2e/testdata/kind.yaml; \ fi @if ! $(CONTAINER_TOOL) container inspect $$($(KIND) get nodes) | $(YQ) e '.[0].Config.Image' | grep -q $(K8S_VERSION); then \ $(KIND) delete cluster --name $(KIND_CLUSTER_NAME); \ - $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION); \ + $(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION) --config=test/e2e/testdata/kind.yaml; \ fi .PHONY: kind-delete @@ -249,6 +249,7 @@ HELM ?= $(LOCALBIN)/helm HELM_DOCS ?= $(LOCALBIN)/helm-docs YQ = $(LOCALBIN)/yq CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs +GINKGO ?= $(LOCALBIN)/ginkgo ## Tool Versions # renovate: datasource=github-tags depName=kubernetes-sigs/kustomize @@ -268,6 +269,8 @@ HELM_SCHEMA_VERSION ?= v1.4.1 HELM_DOCS_VERSION ?= v1.13.1 # renovate: datasource=github-tags depName=mikefarah/yq YQ_VERSION ?= v4.44.1 +# renovate: datasource=github-tags depName=onsi/ginkgo +GINKGO_VERSION ?= v2.19.0 ## Tool install scripts KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" @@ -298,6 +301,11 @@ golangci-lint: $(LOCALBIN) @test -x $(GOLANGCI_LINT) && $(GOLANGCI_LINT) version | grep -q $(GOLANGCI_LINT_VERSION) || \ GOBIN=$(LOCALBIN) go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) +.PHONY: ginkgo +ginkgo: $(LOCALBIN) + @test -x $(GINKGO) && $(GINKGO) version | grep -q $(GINKGO_VERSION) || \ + GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION) + .PHONY: nilaway nilaway: $(LOCALBIN) @test -x $(NILAWAY_LINT) || GOBIN=$(LOCALBIN) go install go.uber.org/nilaway/cmd/nilaway@latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d657a5e9..4d4f9b29 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -88,7 +88,7 @@ spec: httpGet: path: /readyz port: 8081 - initialDelaySeconds: 5 + initialDelaySeconds: 0 periodSeconds: 10 # TODO(user): Configure the resources accordingly based on the project requirements. # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ diff --git a/go.mod b/go.mod index d74f4195..95f6139b 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.3.0 // indirect diff --git a/go.sum b/go.sum index a13159d3..aaf91b21 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c9967858..3d723d29 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -20,7 +20,7 @@ import ( "fmt" "os" "os/exec" - "sync" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -31,6 +31,7 @@ import ( ) var _ = Describe("etcd-operator", Ordered, func() { + dir, _ := utils.GetProjectDir() BeforeAll(func() { var err error @@ -55,7 +56,7 @@ var _ = Describe("etcd-operator", Ordered, func() { By("wait while etcd-operator is ready", func() { cmd := exec.Command("kubectl", "wait", "--namespace", "etcd-operator-system", "deployment/etcd-operator-controller-manager", - "--for", "jsonpath={.status.availableReplicas}=1", "--timeout=5m") + "--for", "jsonpath={.status.readyReplicas}=1", "--timeout=5m") _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) @@ -73,17 +74,10 @@ var _ = Describe("etcd-operator", Ordered, func() { Context("Simple", func() { It("should deploy etcd cluster", func() { - var err error const namespace = "test-simple-etcd-cluster" - var wg sync.WaitGroup - wg.Add(1) - By("create namespace", func() { - cmd := exec.Command("sh", "-c", - fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) // nolint:lll - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - }) + CreateNamespace(namespace) + DeferCleanup(DeleteNamespaceCB(namespace)) By("apply simple etcd cluster manifest", func() { dir, _ := utils.GetProjectDir() @@ -91,30 +85,25 @@ var _ = Describe("etcd-operator", Ordered, func() { "--filename", dir+"/examples/manifests/etcdcluster-simple.yaml", "--namespace", namespace, ) - _, err = utils.Run(cmd) + _, err := utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - By("wait for statefulset is ready", func() { - cmd := exec.Command("kubectl", "wait", - "statefulset/test", - "--for", "jsonpath={.status.readyReplicas}=3", - "--namespace", namespace, - "--timeout", "5m", - ) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - }) + WaitSTSReady("statefulset/test", namespace) + + var etcdClient *clientv3.Client - client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + // By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) Expect(err).NotTo(HaveOccurred()) defer func() { - err := client.Close() + err := etcdClient.Close() Expect(err).NotTo(HaveOccurred()) }() + // }) By("check etcd cluster is healthy", func() { - Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue()) + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) }) }) @@ -124,8 +113,6 @@ var _ = Describe("etcd-operator", Ordered, func() { It("should deploy etcd cluster with auth", func() { var err error const namespace = "test-tls-auth-etcd-cluster" - var wg sync.WaitGroup - wg.Add(1) By("create namespace", func() { cmd := exec.Command("sh", "-c", fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) //nolint:lll @@ -146,7 +133,7 @@ var _ = Describe("etcd-operator", Ordered, func() { By("wait for statefulset is ready", func() { cmd := exec.Command("kubectl", "wait", "statefulset/test", - "--for", "jsonpath={.status.availableReplicas}=3", + "--for", "jsonpath={.status.readyReplicas}=3", "--namespace", namespace, "--timeout", "5m", ) @@ -154,18 +141,22 @@ var _ = Describe("etcd-operator", Ordered, func() { ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + var etcdClient *clientv3.Client + + // By("get etcd client", func() { + etcdClient, err = utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) Expect(err).NotTo(HaveOccurred()) defer func() { - err := client.Close() + err := etcdClient.Close() Expect(err).NotTo(HaveOccurred()) }() + // }) By("check etcd cluster is healthy", func() { - Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue()) + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) }) - auth := clientv3.NewAuth(client) + auth := clientv3.NewAuth(etcdClient) By("check root role is created", func() { _, err = auth.RoleGet(ctx, "root") @@ -187,4 +178,142 @@ var _ = Describe("etcd-operator", Ordered, func() { }) }) + DescribeTable("Upgrade", + func(namespace string, fileName string, version string) { + CreateNamespace(namespace) + DeferCleanup(DeleteNamespaceCB(namespace)) + + By("apply upgrade etcd cluster manifest", func() { + cmd := exec.Command("kubectl", "apply", + "--filename", dir+fileName, + "--namespace", namespace, + ) + _, err := utils.Run(cmd) + + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + var etcdClient *clientv3.Client + + // By("get etcd client", func() { + etcdClient, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + err := etcdClient.Close() + Expect(err).NotTo(HaveOccurred()) + }() + // }) + + WaitSTSReady("statefulset/test", namespace) + + By("check etcd cluster is healthy", func() { + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) + }) + By("upgrade etcd cluster to one patch version", func() { + cmd := exec.Command("kubectl", "patch", + "etcdcluster/test", + "--type", "json", + // Strategic Merge Patch is not currently supported by the etcd-operator + // "--patch", + // fmt.Sprintf( + // "{\"spec\":{\"podTemplate\":{\"containers\":[{\"name\":\"etcd\",\"image\":\"quay.io/coreos/etcd:%s\"}]}}}", + // version, + // ), + "--patch", + fmt.Sprintf( + "[{\"op\": \"replace\", \"path\": \"/spec/podTemplate/spec/containers/0/image\", "+ + "\"value\":\"quay.io/coreos/etcd:%s\"}]", + version, + ), + "--namespace", namespace, + ) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + + // Give the operator some time to update the sts + time.Sleep(2 * time.Second) + + WaitSTSReady("statefulset/test", namespace) + By("check etcd cluster is healthy", func() { + Expect(utils.IsEtcdClusterHealthy(ctx, etcdClient)).To(BeTrue()) + }) + }, + Entry( + "should upgrade etcd cluster patch version", + "test-upgrade-3-5-11--3-5-12", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.5.12", + ), + Entry( + "should downgrade etcd cluster patch version", + "test-downgrade-3-5-11--3-5-10", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.5.10", + ), + Entry( + "should upgrade etcd cluster to one minor version", + "test-upgrade-3-4-32--3-5-10", + "/test/e2e/testdata/etcdcluster-3.4.yaml", + "v3.5.10", + ), + Entry( + "should downgrade etcd cluster to one minor version", + "test-downgrade-3-5-11--3-4-32", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.4.32", + ), + Entry( + "should upgrade etcd cluster to multiple minor version", + "test-upgrade-3-3-27--3-5-11", + "/test/e2e/testdata/etcdcluster-3.3.yaml", + "v3.5.11", + ), + Entry( + "should downgrade etcd cluster to multiple minor version", + "test-downgrade-3-5-11--3-3-27", + "/test/e2e/testdata/etcdcluster-3.5.yaml", + "v3.3.27", + ), + ) + }) + +func DeleteNamespaceCB(namespace string) func() error { + return func() error { + return DeleteNamespace(namespace) + } +} + +func DeleteNamespace(namespace string) error { + By("delete namespace") + + cmd := exec.Command("kubectl", "delete", "namespace", namespace) + _, err := utils.Run(cmd) + + return err +} + +func CreateNamespace(namespace string) { + By("create namespace", func() { + cmd := exec.Command("kubectl", "create", "namespace", namespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + +} + +func WaitSTSReady(stsName, namespace string) { + By("wait for statefulset is ready", func() { + cmd := exec.Command("kubectl", "wait", + stsName, + "--for", "jsonpath={.status.readyReplicas}=3", + "--namespace", namespace, + "--timeout", "5m", + ) + _, err := utils.Run(cmd) + + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + }) + +} diff --git a/test/e2e/testdata/etcdcluster-3.3.yaml b/test/e2e/testdata/etcdcluster-3.3.yaml new file mode 100644 index 00000000..d8bf5b9f --- /dev/null +++ b/test/e2e/testdata/etcdcluster-3.3.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: etcd + image: "quay.io/coreos/etcd:v3.3.27" +# volumes: [] +# storage: +# emptyDir: {} diff --git a/test/e2e/testdata/etcdcluster-3.4.yaml b/test/e2e/testdata/etcdcluster-3.4.yaml new file mode 100644 index 00000000..9e80cce4 --- /dev/null +++ b/test/e2e/testdata/etcdcluster-3.4.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: etcd + image: "quay.io/coreos/etcd:v3.4.32" +# volumes: [] +# storage: +# emptyDir: {} diff --git a/test/e2e/testdata/etcdcluster-3.5.yaml b/test/e2e/testdata/etcdcluster-3.5.yaml new file mode 100644 index 00000000..71bf3f38 --- /dev/null +++ b/test/e2e/testdata/etcdcluster-3.5.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: etcd + image: "quay.io/coreos/etcd:v3.5.11" +# volumes: [] +# storage: +# emptyDir: {} diff --git a/test/e2e/testdata/etcdcluster-simple.yaml b/test/e2e/testdata/etcdcluster-simple.yaml new file mode 100644 index 00000000..f1bff9e4 --- /dev/null +++ b/test/e2e/testdata/etcdcluster-simple.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test +spec: + replicas: 3 diff --git a/test/e2e/testdata/kind.yaml b/test/e2e/testdata/kind.yaml new file mode 100644 index 00000000..afe81483 --- /dev/null +++ b/test/e2e/testdata/kind.yaml @@ -0,0 +1,4 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + kubeProxyMode: "ipvs"