From e9dc9fa84549c0574c1b935b76f0f8c414bb1716 Mon Sep 17 00:00:00 2001 From: cbzzz <69888673+cbzzz@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:22:22 -0400 Subject: [PATCH] feat: cluster autoscaler (#251) * feat: add cluster autoscaler flavor Adds a new cluster-autoscaler flavor that provides an autoscaling add-on for workload cluster nodes via [Cluster Autoscaler](https://www.github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#cluster-autoscaler). Due to constraints with the Kubernetes RBAC system (i.e. [roles cannot be subdivided beyond namespace-granularity](https://www.github.com/kubernetes/kubernetes/issues/56582)), the Cluster Autoscaler add-on is deployed on the management cluster to prevent leaking Cluster API data between workload clusters. Currently, the Cluster Autoscaler reuses the `${CLUSTER_NAME}-kubeconfig` Secret generated by the bootstrap provider to interact with the workload cluster. The kubeconfig contents must be stored in a key named `value`. Due to this, all Cluster Autoscaler actions in the workload cluster are performed as the `cluster-admin` role. See: https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling#autoscaler-running-in-management-cluster-using-service-account-credentials-with-separate-workload-cluster * docs: add cluster autoscaler flavor --- docs/src/SUMMARY.md | 2 + docs/src/topics/autoscaling.md | 62 +++++++++++++++ docs/src/topics/flavors/cluster-autoscaler.md | 43 ++++++++++ .../cluster-autoscaler.yaml | 78 +++++++++++++++++++ .../cluster-autoscaler/kustomization.yaml | 4 + .../cluster-autoscaler/kustomization.yaml | 29 +++++++ 6 files changed, 218 insertions(+) create mode 100644 docs/src/topics/autoscaling.md create mode 100644 docs/src/topics/flavors/cluster-autoscaler.md create mode 100644 templates/addons/cluster-autoscaler/cluster-autoscaler.yaml create mode 100644 templates/addons/cluster-autoscaler/kustomization.yaml create mode 100644 templates/flavors/cluster-autoscaler/kustomization.yaml diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 5b686aac0..9613db2ec 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -10,6 +10,7 @@ - [Dual-stack (kubeadm)](./topics/flavors/dual-stack.md) - [Etcd-disk (kubeadm)](./topics/flavors/etcd-disk.md) - [ClusterClass kubeadm](./topics/flavors/clusterclass-kubeadm.md) + - [Cluster Autoscaler (kubeadm)](./topics/flavors/cluster-autoscaler.md) - [k3s](./topics/flavors/k3s.md) - [rke2](./topics/flavors/rke2.md) - [Etcd](./topics/etcd.md) @@ -19,6 +20,7 @@ - [OS Disk](./topics/disks/os-disk.md) - [Data Disks](./topics/disks/data-disks.md) - [Machine Health Checks](./topics/health-checking.md) + - [Autoscaling](./topics/autoscaling.md) - [Development](./developers/development.md) - [Releasing](./developers/releasing.md) - [Reference](./reference/reference.md) diff --git a/docs/src/topics/autoscaling.md b/docs/src/topics/autoscaling.md new file mode 100644 index 000000000..e12e5d4dd --- /dev/null +++ b/docs/src/topics/autoscaling.md @@ -0,0 +1,62 @@ +# Auto-scaling + +This guide covers auto-scaling for CAPL clusters. The recommended tool for auto-scaling on Cluster API is [Cluster +Autoscaler](https://www.github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#cluster-autoscaler). + +## Flavor + +The auto-scaling feature is provided by an add-on as part of the [Cluster Autoscaler +flavor](./flavors/cluster-autoscaler.md). + +## Configuration + +By default, the Cluster Autoscaler add-on [runs in the management cluster, managing an external workload +cluster](https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling#autoscaler-running-in-management-cluster-using-service-account-credentials-with-separate-workload-cluster). + +``` ++------------+ +----------+ +| mgmt | | workload | +| ---------- | kubeconfig | | +| autoscaler +------------>| | ++------------+ +----------+ +``` + +A separate Cluster Autoscaler is deployed for each workload cluster, configured to only monitor node groups for the +specific namespace and cluster name combination. + +## Role-based Access Control (RBAC) + +### Management Cluster + +Due to constraints with the Kubernetes RBAC system (i.e. [roles cannot be subdivided beyond +namespace-granularity](https://www.github.com/kubernetes/kubernetes/issues/56582)), the Cluster Autoscaler add-on is +deployed on the management cluster to prevent leaking Cluster API data between workload clusters. + +### Workload Cluster + +Currently, the Cluster Autoscaler reuses the `${CLUSTER_NAME}-kubeconfig` Secret generated by the bootstrap provider to +interact with the workload cluster. The kubeconfig contents must be stored in a key named `value`. Due to this, all +Cluster Autoscaler actions in the workload cluster are performed as the `cluster-admin` role. + +## Scale Down + +> Cluster Autoscaler decreases the size of the cluster when some nodes are consistently unneeded for a significant +> amount of time. A node is unneeded when it has low utilization and all of its important pods can be moved elsewhere. + +By default, Cluster Autoscaler scales down a node after it is marked as unneeded for 10 minutes. This can be adjusted +with the [`--scale-down-unneeded-time` +setting](https://www.github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#how-can-i-modify-cluster-autoscaler-reaction-time). + +### Kubernetes Cloud Controller Manager for Linode (CCM) + +The [Kubernetes Cloud Controller Manager for +Linode](https://www.github.com/linode/linode-cloud-controller-manager?tab=readme-ov-file#the-purpose-of-the-ccm) is +deployed on workload clusters and reconciles Kubernetes Node objects with their backing Linode infrastructure. When +scaling down a node group, the Cluster Autoscaler also deletes the Kubernetes Node object on the workload cluster. This +step preempts the Node-deletion in Kubernetes triggered by the CCM. + +## Additional Resources + +- [Autoscaling - The Cluster API Book](https://cluster-api.sigs.k8s.io/tasks/automated-machine-management/autoscaling) +- [Cluster Autoscaler + FAQ](https://www.github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#cluster-autoscaler) diff --git a/docs/src/topics/flavors/cluster-autoscaler.md b/docs/src/topics/flavors/cluster-autoscaler.md new file mode 100644 index 000000000..63ab643d2 --- /dev/null +++ b/docs/src/topics/flavors/cluster-autoscaler.md @@ -0,0 +1,43 @@ +# Cluster Autoscaler + +This flavor adds auto-scaling via [Cluster +Autoscaler](https://www.github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#cluster-autoscaler). + +## Specification + +| Control Plane | CNI | Default OS | Installs ClusterClass | IPv4 | IPv6 | +|---------------|--------|--------------|-----------------------|------|------| +| Kubeadm | Cilium | Ubuntu 22.04 | No | Yes | No | + +## Prerequisites + +[Quickstart](../getting-started.md) completed + +## Usage + +1. Set up autoscaling environment variables + > We recommend using Cluster Autoscaler with the Kubernetes control plane + > ... version for which it was meant. + > + > -- [Releases ยท kubernetes/autoscaler](https://www.github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#releases) + + ```sh + export CLUSTER_AUTOSCALER_VERSION=v1.29.0 + # Optional: If specified, these values must be explicitly quoted! + export WORKER_MACHINE_MIN='"1"' + export WORKER_MACHINE_MAX='"10"' + ``` + +2. Generate cluster yaml + + ```sh + clusterctl generate cluster test-cluster \ + --infrastructure linode:0.0.0 \ + --flavor cluster-autoscaler > test-cluster.yaml + ``` + +3. Apply cluster yaml + + ```sh + kubectl apply -f test-cluster.yaml + ``` diff --git a/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml b/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml new file mode 100644 index 000000000..0259cff97 --- /dev/null +++ b/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${CLUSTER_NAME}-cluster-autoscaler + labels: + app: ${CLUSTER_NAME}-cluster-autoscaler +spec: + selector: + matchLabels: + app: ${CLUSTER_NAME}-cluster-autoscaler + replicas: 1 + template: + metadata: + labels: + app: ${CLUSTER_NAME}-cluster-autoscaler + spec: + containers: + - image: registry.k8s.io/autoscaling/cluster-autoscaler:${CLUSTER_AUTOSCALER_VERSION} + name: cluster-autoscaler + command: + - /cluster-autoscaler + args: + - --cloud-provider=clusterapi + - --kubeconfig=/mnt/kubeconfig/value + - --clusterapi-cloud-config-authoritative + - --node-group-auto-discovery=clusterapi:namespace=${NAMESPACE},clusterName=${CLUSTER_NAME} + # See: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#my-cluster-is-below-minimum--above-maximum-number-of-nodes-but-ca-did-not-fix-that-why + - --enforce-node-group-min-size + volumeMounts: + - name: kubeconfig + readOnly: true + mountPath: /mnt/kubeconfig/ + serviceAccountName: ${CLUSTER_NAME}-cluster-autoscaler + terminationGracePeriodSeconds: 10 + volumes: + - name: kubeconfig + secret: + secretName: ${CLUSTER_NAME}-kubeconfig +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ${CLUSTER_NAME}-cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ${CLUSTER_NAME}-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: ${CLUSTER_NAME}-cluster-autoscaler +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ${CLUSTER_NAME}-cluster-autoscaler +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ${CLUSTER_NAME}-cluster-autoscaler +rules: +- apiGroups: + - cluster.x-k8s.io + - infrastructure.cluster.x-k8s.io + resources: + - machinedeployments + - machinedeployments/scale + - machines + - machinesets + - machinepools + - linodemachinetemplates + - linodemachines + verbs: + - get + - list + - update + - watch diff --git a/templates/addons/cluster-autoscaler/kustomization.yaml b/templates/addons/cluster-autoscaler/kustomization.yaml new file mode 100644 index 000000000..e84803310 --- /dev/null +++ b/templates/addons/cluster-autoscaler/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cluster-autoscaler.yaml diff --git a/templates/flavors/cluster-autoscaler/kustomization.yaml b/templates/flavors/cluster-autoscaler/kustomization.yaml new file mode 100644 index 000000000..4a2f43968 --- /dev/null +++ b/templates/flavors/cluster-autoscaler/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../default + - ../../addons/cluster-autoscaler +patches: + - target: + group: cluster.x-k8s.io + version: v1beta1 + kind: Cluster + patch: |- + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + metadata: + name: ${CLUSTER_NAME} + labels: + cluster-autoscaler: "true" + - target: + group: cluster.x-k8s.io + version: v1beta1 + kind: MachineDeployment + patch: |- + apiVersion: cluster.x-k8s.io/v1beta1 + kind: MachineDeployment + metadata: + name: ${CLUSTER_NAME}-md-0 + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: ${WORKER_MACHINE_MIN:-"1"} + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: ${WORKER_MACHINE_MAX:-"10"}