Add ClusterAnnotations support.

Signed-off-by: xuezhaojun <[email protected]>

deploy/klusterlet/config/crds/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml

@@ -184,6 +184,13 @@ spec:
                       set.
                     format: int32
                     type: integer
+                  clusterAnnotations:
+                    additionalProperties:
+                      type: string
+                    description: ClusterAnnotations is annotations with the reserve
+                      prefix "agent.open-cluster-management.io" set on ManagedCluster
+                      when creating only, other actors can update it afterwards.
+                    type: object
                   featureGates:
                     description: 'FeatureGates represents the list of feature gates
                       for registration If it is set empty, default feature gates will

deploy/klusterlet/olm-catalog/klusterlet/manifests/operator.open-cluster-management.io_klusterlets.yaml

@@ -116,6 +116,11 @@ spec:
                     description: clientCertExpirationSeconds represents the seconds of a client certificate to expire. If it is not set or 0, the default duration seconds will be set by the hub cluster. If the value is larger than the max signing duration seconds set on the hub cluster, the max signing duration seconds will be set.
                     format: int32
                     type: integer
+                  clusterAnnotations:
+                    additionalProperties:
+                      type: string
+                    description: ClusterAnnotations is annotations with the reserve prefix "agent.open-cluster-management.io" set on ManagedCluster when creating only, other actors can update it afterwards.
+                    type: object
                   featureGates:
                     description: 'FeatureGates represents the list of feature gates for registration If it is set empty, default feature gates will be used. If it is set, featuregate/Foo is an example of one item in FeatureGates: 1. If featuregate/Foo does not exist, registration-operator will discard it 2. If featuregate/Foo exists and is false by default. It is now possible to set featuregate/Foo=[false|true] 3. If featuregate/Foo exists and is true by default. If a cluster-admin upgrading from 1 to 2 wants to continue having featuregate/Foo=false, he can set featuregate/Foo=false before upgrading. Let''s say the cluster-admin wants featuregate/Foo=false.'
                     items:

go.mod

@@ -28,7 +28,7 @@ require (
 	k8s.io/kube-aggregator v0.27.2
 	k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
 	open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63
-	open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02
+	open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249
 	sigs.k8s.io/controller-runtime v0.15.0
 	sigs.k8s.io/kube-storage-version-migrator v0.0.5
 )

go.sum

@@ -1158,8 +1158,8 @@ k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 h1:xMMXJlJbsU8w3V5N2FLDQ8YgU8s1E
 k8s.io/utils v0.0.0-20230313181309-38a27ef9d749/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63 h1:GCsAD1jb6wqhXTHdUM/HcWzv5b2NbZ6FxpLZcxa/jhI=
 open-cluster-management.io/addon-framework v0.7.1-0.20230705031704-6a328fa5cd63/go.mod h1:V+WUFC7GD89Lc68eXSN/FJebnCH4NjrfF44VsO0YAC8=
-open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02 h1:YdfpLV94HvYJexLeguX2WZ9Asszks0aNt2K4Ifd3Bfg=
-open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
+open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249 h1:Q3UCh10q8w0k/sx0YDer6svU44/puTZMmxVVFndGdUs=
+open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

manifests/klusterlet/management/klusterlet-agent-deployment.yaml

@@ -75,6 +75,9 @@ spec:
           {{if gt .ClientCertExpirationSeconds 0}}
           - "--client-cert-expiration-seconds={{ .ClientCertExpirationSeconds }}"
           {{end}}
+          {{if .ClusterAnnotationsString}}
+          - "--cluster-annotations={{ .ClusterAnnotationsString }}"
+          {{end}}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

manifests/klusterlet/management/klusterlet-registration-deployment.yaml

@@ -73,6 +73,9 @@ spec:
           {{if gt .ClientCertExpirationSeconds 0}}
           - "--client-cert-expiration-seconds={{ .ClientCertExpirationSeconds }}"
           {{end}}
+          {{if .ClusterAnnotationsString}}
+          - "--cluster-annotations={{ .ClusterAnnotationsString }}"
+          {{end}}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

pkg/common/helpers/annotations.go

@@ -0,0 +1,26 @@
+package helpers
+
+import (
+	"strings"
+
+	"k8s.io/klog/v2"
+
+	operatorv1 "open-cluster-management.io/api/operator/v1"
+)
+
+func FilterClusterAnnotations(annotations map[string]string) map[string]string {
+	clusterAnnotations := make(map[string]string)
+	if annotations == nil {
+		return clusterAnnotations
+	}
+
+	for k, v := range annotations {
+		if strings.HasPrefix(k, operatorv1.ClusterAnnotationsKeyPrefix) {
+			clusterAnnotations[k] = v
+		} else {
+			klog.Warningf("annotation %q is not prefixed with %q, it will be ignored", k, operatorv1.ClusterAnnotationsKeyPrefix)
+		}
+	}
+
+	return clusterAnnotations
+}

pkg/common/helpers/annotations_test.go

@@ -0,0 +1,71 @@
+package helpers
+
+import (
+	"reflect"
+	"testing"
+
+	operatorv1 "open-cluster-management.io/api/operator/v1"
+)
+
+func TestFilterClusterAnnotations(t *testing.T) {
+	tests := []struct {
+		name        string
+		annotations map[string]string
+		want        map[string]string
+	}{
+		{
+			name:        "empty annotations",
+			annotations: map[string]string{},
+			want:        map[string]string{},
+		},
+		{
+			name: "no cluster annotations",
+			annotations: map[string]string{
+				"foo": "bar",
+				"baz": "qux",
+			},
+			want: map[string]string{},
+		},
+		{
+			name: "one cluster annotation",
+			annotations: map[string]string{
+				operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
+				"baz": "qux",
+			},
+			want: map[string]string{
+				operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
+			},
+		},
+		{
+			name: "multiple cluster annotations",
+			annotations: map[string]string{
+				operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
+				operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
+				"quux": "corge",
+			},
+			want: map[string]string{
+				operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
+				operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
+			},
+		},
+		{
+			name: "all annotations are cluster annotations",
+			annotations: map[string]string{
+				operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
+				operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
+			},
+			want: map[string]string{
+				operatorv1.ClusterAnnotationsKeyPrefix + "foo": "bar",
+				operatorv1.ClusterAnnotationsKeyPrefix + "baz": "qux",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := FilterClusterAnnotations(tt.annotations); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FilterClusterAnnotations() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go

@@ -28,6 +28,7 @@ import (
 	ocmfeature "open-cluster-management.io/api/feature"
 	operatorapiv1 "open-cluster-management.io/api/operator/v1"
 
+	commonhelpers "open-cluster-management.io/ocm/pkg/common/helpers"
 	"open-cluster-management.io/ocm/pkg/common/patcher"
 	"open-cluster-management.io/ocm/pkg/common/queue"
 	"open-cluster-management.io/ocm/pkg/operator/helpers"
@@ -134,6 +135,7 @@ type klusterletConfig struct {
 	OperatorNamespace           string
 	Replica                     int32
 	ClientCertExpirationSeconds int32
+	ClusterAnnotationsString    string
 
 	ExternalManagedKubeConfigSecret             string
 	ExternalManagedKubeConfigRegistrationSecret string
@@ -234,6 +236,13 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 	if klusterlet.Spec.RegistrationConfiguration != nil {
 		registrationFeatureGates = klusterlet.Spec.RegistrationConfiguration.FeatureGates
 		config.ClientCertExpirationSeconds = klusterlet.Spec.RegistrationConfiguration.ClientCertExpirationSeconds
+
+		// construct cluster annotations string, the final format is "key1=value1,key2=value2" or ""(when no annotations)
+		var annotationsArray []string
+		for k, v := range commonhelpers.FilterClusterAnnotations(klusterlet.Spec.RegistrationConfiguration.ClusterAnnotations) {
+			annotationsArray = append(annotationsArray, fmt.Sprintf("%s=%s", k, v))
+		}
+		config.ClusterAnnotationsString = strings.Join(annotationsArray, ",")
 	}
 	config.RegistrationFeatureGates, registrationFeatureMsgs = helpers.ConvertToFeatureGateFlags("Registration",
 		registrationFeatureGates, ocmfeature.DefaultSpokeRegistrationFeatureGates)

pkg/registration/spoke/options.go

@@ -18,6 +18,7 @@ type SpokeAgentOptions struct {
 	ClusterHealthCheckPeriod    time.Duration
 	MaxCustomClusterClaims      int
 	ClientCertExpirationSeconds int32
+	ClusterAnnotations          map[string]string
 }
 
 func NewSpokeAgentOptions() *SpokeAgentOptions {
@@ -43,6 +44,8 @@ func (o *SpokeAgentOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&o.ClientCertExpirationSeconds, "client-cert-expiration-seconds", o.ClientCertExpirationSeconds,
 		"The requested duration in seconds of validity of the issued client certificate. If this is not set, "+
 			"the value of --cluster-signing-duration command-line flag of the kube-controller-manager will be used.")
+	fs.StringToStringVar(&o.ClusterAnnotations, "cluster-annotations", o.ClusterAnnotations, `the annotations with the reserve
+	 prefix "agent.open-cluster-management.io" set on ManagedCluster when creating only, other actors can update it afterwards.`)
 }
 
 // Validate verifies the inputs.

pkg/registration/spoke/registration/creating_controller.go

@@ -14,6 +14,8 @@ import (
 
 	clientset "open-cluster-management.io/api/client/cluster/clientset/versioned"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
+
+	commonhelpers "open-cluster-management.io/ocm/pkg/common/helpers"
 )
 
 var (
@@ -26,19 +28,22 @@ type managedClusterCreatingController struct {
 	clusterName             string
 	spokeExternalServerURLs []string
 	spokeCABundle           []byte
+	clusterAnnotations      map[string]string
 	hubClusterClient        clientset.Interface
 }
 
 // NewManagedClusterCreatingController creates a new managedClusterCreatingController on the managed cluster.
 func NewManagedClusterCreatingController(
-	clusterName string, spokeExternalServerURLs []string,
+	clusterName string, spokeExternalServerURLs []string, annotations map[string]string,
 	spokeCABundle []byte,
 	hubClusterClient clientset.Interface,
 	recorder events.Recorder) factory.Controller {
+
 	c := &managedClusterCreatingController{
 		clusterName:             clusterName,
 		spokeExternalServerURLs: spokeExternalServerURLs,
 		spokeCABundle:           spokeCABundle,
+		clusterAnnotations:      commonhelpers.FilterClusterAnnotations(annotations),
 		hubClusterClient:        hubClusterClient,
 	}
 
@@ -64,7 +69,8 @@ func (c *managedClusterCreatingController) sync(ctx context.Context, syncCtx fac
 	if errors.IsNotFound(err) {
 		managedCluster := &clusterv1.ManagedCluster{
 			ObjectMeta: metav1.ObjectMeta{
-				Name: c.clusterName,
+				Name:        c.clusterName,
+				Annotations: c.clusterAnnotations,
 			},
 		}
 

pkg/registration/spoke/registration/creating_controller_test.go

@@ -36,6 +36,14 @@ func TestCreateSpokeCluster(t *testing.T) {
 				actual := actions[1].(clienttesting.CreateActionImpl).Object
 				actualClientConfigs := actual.(*clusterv1.ManagedCluster).Spec.ManagedClusterClientConfigs
 				testinghelpers.AssertManagedClusterClientConfigs(t, actualClientConfigs, expectedClientConfigs)
+				clusterannotations := actual.(*clusterv1.ManagedCluster).Annotations
+				if len(clusterannotations) != 1 {
+					t.Errorf("expected cluster annotations %#v but got: %#v", 1, len(clusterannotations))
+				}
+				if value, ok := clusterannotations["agent.open-cluster-management.io/test"]; !ok || value != "true" {
+					t.Errorf("expected cluster annotations %#v but got: %#v", "agent.open-cluster-management.io/test",
+						clusterannotations["agent.open-cluster-management.io/test"])
+				}
 			},
 		},
 		{
@@ -55,6 +63,9 @@ func TestCreateSpokeCluster(t *testing.T) {
 				spokeExternalServerURLs: []string{testSpokeExternalServerUrl},
 				spokeCABundle:           []byte("testcabundle"),
 				hubClusterClient:        clusterClient,
+				clusterAnnotations: map[string]string{
+					"agent.open-cluster-management.io/test": "true",
+				},
 			}
 
 			syncErr := ctrl.sync(context.TODO(), testingcommon.NewFakeSyncContext(t, ""))

pkg/registration/spoke/spokeagent.go

@@ -173,7 +173,7 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
 
 	// start a SpokeClusterCreatingController to make sure there is a spoke cluster on hub cluster
 	spokeClusterCreatingController := registration.NewManagedClusterCreatingController(
-		o.agentOptions.SpokeClusterName, o.registrationOption.SpokeExternalServerURLs,
+		o.agentOptions.SpokeClusterName, o.registrationOption.SpokeExternalServerURLs, o.registrationOption.ClusterAnnotations,
 		spokeClusterCABundle,
 		bootstrapClusterClient,
 		recorder,

test/integration/operator/klusterlet_test.go

@@ -963,6 +963,7 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 			gomega.Expect(operatorClient.OperatorV1().Klusterlets().Delete(context.Background(),
 				klusterlet.Name, metav1.DeleteOptions{})).To(gomega.BeNil())
 		})
+
 		ginkgo.It("feature gates configuration is nil or empty", func() {
 			klusterlet.Spec.RegistrationConfiguration = nil
 			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{}
@@ -1083,6 +1084,10 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 						Mode:    operatorapiv1.FeatureGateModeTypeDisable,
 					},
 				},
+				ClusterAnnotations: map[string]string{
+					"foo":                                  "bar", // should be ignored
+					"agent.open-cluster-management.io/foo": "bar",
+				},
 			}
 			klusterlet.Spec.WorkConfiguration = &operatorapiv1.WorkConfiguration{
 				FeatureGates: []operatorapiv1.FeatureGate{
@@ -1141,6 +1146,10 @@ var _ = ginkgo.Describe("Klusterlet", func() {
 			gomega.Expect(registrationDeployment.Spec.Template.Spec.Containers[0].Args).Should(
 				gomega.ContainElement("--feature-gates=ClusterClaim=false"))
 
+			ginkgo.By("Check the registration-agent has the expected cluster-annotations")
+			gomega.Expect(registrationDeployment.Spec.Template.Spec.Containers[0].Args).Should(
+				gomega.ContainElement("--cluster-annotations=agent.open-cluster-management.io/foo=bar"))
+
 			ginkgo.By("Check the work-agent has the expected feature gates")
 			workDeployment, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(
 				context.Background(), workDeploymentName, metav1.GetOptions{})

vendor/modules.txt

@@ -1426,7 +1426,7 @@ open-cluster-management.io/addon-framework/pkg/index
 open-cluster-management.io/addon-framework/pkg/manager/controllers/addonconfiguration
 open-cluster-management.io/addon-framework/pkg/manager/controllers/addonowner
 open-cluster-management.io/addon-framework/pkg/utils
-# open-cluster-management.io/api v0.11.1-0.20230720020428-0336f2374d02
+# open-cluster-management.io/api v0.11.1-0.20230725140722-c0c9fb59d249
 ## explicit; go 1.19
 open-cluster-management.io/api/addon/v1alpha1
 open-cluster-management.io/api/client/addon/clientset/versioned