Update controller-runtime (#205)

* add tests for multiple k8s versions

* bump controller-runtime to 0.15.0; bump controller-tools to 0.12.1; bump k8s versions to 0.27.3; bump go version to 1.20

* remove deprecations and fix breaking change of controller-runtime

* add code suggestions from PR review

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3-labs
-ARG GO_VERSION=1.18
+ARG GO_VERSION=1.20
 
 # get modules, if they don't change the cache can be used for faster builds
 FROM golang:${GO_VERSION} AS base

Earthfile

@@ -1,5 +1,5 @@
 VERSION 0.6
-FROM golang:1.19
+FROM golang:1.20
 ARG DOCKER_REPO=ghcr.io/stackitcloud/yawol/
 ARG BINPATH=/usr/local/bin/
 ARG GOCACHE=/go-cache
@@ -262,6 +262,18 @@ test:
         eval `setup-envtest use -p env $KUBERNETES_VERSION` && \
         eval "$GO_TEST ./..."
 
+test-multiple-k8s-versions:
+    BUILD +test --KUBERNETES_VERSION=1.23.5
+    BUILD +test --KUBERNETES_VERSION=1.24.2
+    BUILD +test --KUBERNETES_VERSION=1.25.0
+    BUILD +test --KUBERNETES_VERSION=1.26.1
+    BUILD +test --KUBERNETES_VERSION=1.27.1
+
+list-available-k8s-envtest-versions:
+    FROM +deps
+    COPY +gotools/bin/setup-envtest $BINPATH
+    RUN setup-envtest list
+
 test-output:
     FROM +test --GO_TEST="go test -count 1 -coverprofile=cover.out"
     SAVE ARTIFACT cover.out
@@ -307,7 +319,7 @@ all-except-snyk:
     BUILD +generate
     BUILD +lint
     BUILD +coverage
-    BUILD +test
+    BUILD +test-multiple-k8s-versions
     BUILD +ci
 
 all:

cmd/yawol-cloud-controller/main.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
 	"github.com/stackitcloud/yawol/controllers/yawol-cloud-controller/controlcontroller"
@@ -138,10 +139,11 @@ func main() {
 	}
 
 	controlMgr, err := ctrl.NewManager(getConfigFromKubeconfigOrDie(controlKubeconfig), ctrl.Options{
-		Scheme:                        scheme,
-		MetricsBindAddress:            "0",
-		Port:                          9443,
-		Namespace:                     *infrastructureDefaults.Namespace,
+		Scheme:             scheme,
+		MetricsBindAddress: "0",
+		Cache: cache.Options{
+			Namespaces: []string{*infrastructureDefaults.Namespace},
+		},
 		LeaderElection:                controlEnableLeaderElection,
 		LeaderElectionReleaseOnCancel: true,
 		LeaderElectionID:              "4c878ae2.stackit.cloud",

cmd/yawol-controller/main.go

@@ -13,6 +13,7 @@ import (
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
@@ -153,15 +154,16 @@ func main() {
 		loadBalancerMgr, err = ctrl.NewManager(cfg, ctrl.Options{
 			Scheme:                        scheme,
 			MetricsBindAddress:            metricsAddrLb,
-			Port:                          9443,
 			LeaderElection:                enableLeaderElection,
 			LeaderElectionReleaseOnCancel: true,
 			LeaderElectionID:              "3a7ac996.stackit.cloud",
 			LeaseDuration:                 &leasesDuration,
 			RenewDeadline:                 &leasesRenewDeadline,
 			RetryPeriod:                   &leasesRetryPeriod,
 			LeaderElectionResourceLock:    leasesLeaderElectionResourceLock,
-			Namespace:                     clusterNamespace,
+			Cache: cache.Options{
+				Namespaces: []string{clusterNamespace},
+			},
 		})
 		if err != nil {
 			setupLog.Error(err, "unable to start manager")
@@ -189,15 +191,16 @@ func main() {
 		loadBalancerSetMgr, err = ctrl.NewManager(cfg, ctrl.Options{
 			Scheme:                        scheme,
 			MetricsBindAddress:            metricsAddrLbs,
-			Port:                          9444,
 			LeaderElection:                enableLeaderElection,
 			LeaderElectionReleaseOnCancel: true,
 			LeaderElectionID:              "rgp5vg43.stackit.cloud",
 			LeaseDuration:                 &leasesDuration,
 			RenewDeadline:                 &leasesRenewDeadline,
 			RetryPeriod:                   &leasesRetryPeriod,
 			LeaderElectionResourceLock:    leasesLeaderElectionResourceLock,
-			Namespace:                     clusterNamespace,
+			Cache: cache.Options{
+				Namespaces: []string{clusterNamespace},
+			},
 		})
 		if err != nil {
 			setupLog.Error(err, "unable to start manager")
@@ -240,15 +243,16 @@ func main() {
 		loadBalancerMachineMgr, err = ctrl.NewManager(cfg, ctrl.Options{
 			Scheme:                        scheme,
 			MetricsBindAddress:            metricsAddrLbm,
-			Port:                          9445,
 			LeaderElection:                enableLeaderElection,
 			LeaderElectionReleaseOnCancel: true,
 			LeaderElectionID:              "tanf7ges.stackit.cloud",
 			LeaseDuration:                 &leasesDuration,
 			RenewDeadline:                 &leasesRenewDeadline,
 			RetryPeriod:                   &leasesRetryPeriod,
 			LeaderElectionResourceLock:    leasesLeaderElectionResourceLock,
-			Namespace:                     clusterNamespace,
+			Cache: cache.Options{
+				Namespaces: []string{clusterNamespace},
+			},
 		})
 		if err != nil {
 			setupLog.Error(err, "unable to start manager")

cmd/yawollet/main.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/envoyproxy/go-control-plane/pkg/resource/v3"
 	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -187,21 +187,13 @@ func main() {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
-		Port:               9443,
 		LeaderElection:     false,
-		Namespace:          namespace,
-
-		NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
-			opts.SelectorsByObject = cache.SelectorsByObject{
-				&yawolv1beta1.LoadBalancer{}: cache.ObjectSelector{
-					Field: fields.SelectorFromSet(fields.Set{"metadata.name": loadbalancerName}),
-				},
-				&yawolv1beta1.LoadBalancerMachine{}: cache.ObjectSelector{
-					Field: fields.SelectorFromSet(fields.Set{"metadata.name": loadbalancerMachineName}),
-				},
-			}
-
-			return cache.New(config, opts)
+		Cache: cache.Options{
+			Namespaces: []string{namespace},
+			ByObject: map[client.Object]cache.ByObject{
+				&yawolv1beta1.LoadBalancer{}:        {Field: fields.SelectorFromSet(fields.Set{"metadata.name": loadbalancerName})},
+				&yawolv1beta1.LoadBalancerMachine{}: {Field: fields.SelectorFromSet(fields.Set{"metadata.name": loadbalancerMachineName})},
+			},
 		},
 	})
 	if err != nil {

controllers/yawol-cloud-controller/targetcontroller/suite_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"path/filepath"
 	"testing"
-	"time"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -48,12 +47,10 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:        []string{filepath.Join("..", "..", "..", "charts", "yawol-controller", "crds")},
-		ControlPlaneStartTimeout: time.Second * time.Duration(15),
+		CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "yawol-controller", "crds")},
 	}
 
 	args := testEnv.ControlPlane.GetAPIServer().Configure()
-	args.Append("feature-gates", "IPv6DualStack=true")
 	args.Append("service-cluster-ip-range", "10.244.0.0/16,fc00::0001:0000/112")
 
 	var err error

controllers/yawol-controller/loadbalancer/loadbalancer_controller.go

@@ -7,10 +7,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
-	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/stackitcloud/yawol/internal/helper"
 	"github.com/stackitcloud/yawol/internal/helper/kubernetes"
@@ -137,12 +135,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&yawolv1beta1.LoadBalancer{}).
-		// we can't use Owns because it filters for ownerReference.controller==true which is not the case for our objects (as of now)
-		Watches(
-			&source.Kind{Type: &yawolv1beta1.LoadBalancerSet{}},
-			&handler.EnqueueRequestForOwner{OwnerType: &yawolv1beta1.LoadBalancer{}},
-			builder.WithPredicates(LoadBalancerSetPredicate()),
-		).
+		Owns(&yawolv1beta1.LoadBalancerSet{}, builder.MatchEveryOwner, builder.WithPredicates(LoadBalancerSetPredicate())).
 		WithOptions(controller.Options{
 			MaxConcurrentReconciles: r.WorkerCount,
 			RateLimiter:             r.RateLimiter,

go.mod

@@ -1,6 +1,6 @@
 module github.com/stackitcloud/yawol
 
-go 1.19
+go 1.20
 
 require (
 	github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4
@@ -20,62 +20,47 @@ require (
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/ini.v1 v1.67.0
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.26.1
-	k8s.io/apimachinery v0.26.1
-	k8s.io/client-go v0.26.1
-	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
-	sigs.k8s.io/controller-runtime v0.13.1
-	sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20230326190753-3b0376c24e4f
-	sigs.k8s.io/controller-tools v0.11.3
-)
-
-replace (
-	k8s.io/api => k8s.io/api v0.24.12
-	k8s.io/apimachinery => k8s.io/apimachinery v0.24.12
-	k8s.io/client-go => k8s.io/client-go v0.24.12
+	k8s.io/api v0.27.3
+	k8s.io/apimachinery v0.27.3
+	k8s.io/client-go v0.27.3
+	k8s.io/utils v0.0.0-20230505201702-9f6742963106
+	sigs.k8s.io/controller-runtime v0.15.0
+	sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20230523032821-116a1b831fff // v0.15.0
+	sigs.k8s.io/controller-tools v0.12.1
 )
 
 require (
-	cloud.google.com/go/compute v1.19.1 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.0.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/fatih/color v1.13.0 // indirect
+	github.com/fatih/color v1.15.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-logr/zapr v1.2.3 // indirect
+	github.com/go-logr/zapr v1.2.4 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.20.0 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/gobuffalo/flect v0.3.0 // indirect
+	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/mattn/go-colorable v0.1.9 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -86,35 +71,33 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
-	github.com/spf13/cobra v1.6.1 // indirect
+	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/tools v0.9.3 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.26.1 // indirect
-	k8s.io/component-base v0.26.1 // indirect
-	k8s.io/klog/v2 v2.80.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	k8s.io/apiextensions-apiserver v0.27.2 // indirect
+	k8s.io/component-base v0.27.2 // indirect
+	k8s.io/klog/v2 v2.90.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )