From e63837744bef7041b5ff30b654902f1362fa8a48 Mon Sep 17 00:00:00 2001 From: Andreas Sommer Date: Sun, 27 Aug 2023 23:05:02 +0200 Subject: [PATCH 1/3] Upgrade to Kubernetes 1.28 --- service/kubernetes/scripts/install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service/kubernetes/scripts/install.sh b/service/kubernetes/scripts/install.sh index 96738cc..83f72fd 100644 --- a/service/kubernetes/scripts/install.sh +++ b/service/kubernetes/scripts/install.sh @@ -2,7 +2,7 @@ set -e # https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-using-native-package-management (yes, `xenial` is correct even for newer Ubuntu versions) -curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg +curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmour -o /etc/apt/keyrings/docker.gpg @@ -13,8 +13,8 @@ apt-get update # Use `DEBIAN_FRONTEND=noninteractive` to avoid starting containerd already with Ubuntu's minimal config. # -# Kubernetes 1.26 requires at least containerd v1.6 -DEBIAN_FRONTEND=noninteractive apt-get install -y containerd.io=1.6.15-1 +# Kubernetes 1.26+ requires at least containerd v1.6. +DEBIAN_FRONTEND=noninteractive apt-get install -y containerd.io=1.6.22-1 containerd config default > /etc/containerd/config.toml sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml @@ -23,7 +23,7 @@ systemctl restart containerd # Pin Kubernetes major version since there are breaking changes between releases. # For example, Kubernetes 1.26 requires a newer containerd (https://kubernetes.io/blog/2022/11/18/upcoming-changes-in-kubernetes-1-26/#cri-api-removal). -apt-get install -y kubelet=1.26.0-00 kubeadm=1.26.0-00 kubectl=1.26.0-00 # kubernetes-cni package comes as dependency of the others +apt-get install -y kubelet=1.28.0-00 kubeadm=1.28.1-00 kubectl=1.28.1-00 # kubernetes-cni package comes as dependency of the others apt-mark hold kubelet kubeadm kubectl kubernetes-cni echo "Installation of packages done" From e1179b158a361a194b8b1c09c205b4b02dafa53b Mon Sep 17 00:00:00 2001 From: Andreas Sommer Date: Sun, 27 Aug 2023 23:28:05 +0200 Subject: [PATCH 2/3] Replace deprecated `null_data_source` by `local` --- security/ufw/main.tf | 28 ++++++++++++---------------- service/etcd/main.tf | 33 ++++++++++++--------------------- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/security/ufw/main.tf b/security/ufw/main.tf index 1fae387..23f7d9e 100644 --- a/security/ufw/main.tf +++ b/security/ufw/main.tf @@ -25,11 +25,21 @@ variable "additional_rules" { default = [] } +locals { + ufw_config = templatefile("${path.module}/scripts/ufw.sh", { + private_interface = var.private_interface + kubernetes_interface = var.kubernetes_interface + vpn_interface = var.vpn_interface + vpn_port = var.vpn_port + additional_rules = join("\nufw ", flatten(["", var.additional_rules])) + }) +} + resource "null_resource" "firewall" { count = var.node_count triggers = { - template = data.null_data_source.ufw.outputs.content + template = local.ufw_config } connection { @@ -39,20 +49,6 @@ resource "null_resource" "firewall" { } provisioner "remote-exec" { - inline = [ - data.null_data_source.ufw.outputs.content - ] - } -} - -data "null_data_source" "ufw" { - inputs = { - content = templatefile("${path.module}/scripts/ufw.sh", { - private_interface = var.private_interface - kubernetes_interface = var.kubernetes_interface - vpn_interface = var.vpn_interface - vpn_port = var.vpn_port - additional_rules = join("\nufw ", flatten(["", var.additional_rules])) - }) + inline = [local.ufw_config] } } diff --git a/service/etcd/main.tf b/service/etcd/main.tf index c76267a..a5c6751 100644 --- a/service/etcd/main.tf +++ b/service/etcd/main.tf @@ -29,7 +29,7 @@ resource "null_resource" "etcd" { count = var.node_count triggers = { - template = join("", data.null_data_source.etcd-service.*.outputs.content) + template = join("", local.etcd_service) } connection { @@ -47,7 +47,7 @@ resource "null_resource" "etcd" { } provisioner "file" { - content = element(data.null_data_source.etcd-service.*.outputs.content, count.index) + content = element(local.etcd_service, count.index) destination = "/etc/systemd/system/etcd.service" } @@ -64,29 +64,20 @@ resource "null_resource" "etcd" { } } -data "null_data_source" "etcd-service" { - count = var.node_count - - inputs = { - content = templatefile("${path.module}/templates/etcd.service", { - hostname = element(local.etcd_hostnames, count.index) +locals { + etcd_service = [ + for n in range(var.node_count) : + templatefile("${path.module}/templates/etcd.service", { + hostname = element(local.etcd_hostnames, n) intial_cluster = "${join(",", formatlist("%s=http://%s:2380", local.etcd_hostnames, local.etcd_vpn_ips))}" - listen_client_urls = "http://${element(local.etcd_vpn_ips, count.index)}:2379" - advertise_client_urls = "http://${element(local.etcd_vpn_ips, count.index)}:2379" - listen_peer_urls = "http://${element(local.etcd_vpn_ips, count.index)}:2380" + listen_client_urls = "http://${element(local.etcd_vpn_ips, n)}:2379" + advertise_client_urls = "http://${element(local.etcd_vpn_ips, n)}:2379" + listen_peer_urls = "http://${element(local.etcd_vpn_ips, n)}:2380" vpn_unit = var.vpn_unit }) - } -} - -data "null_data_source" "endpoints" { - depends_on = [null_resource.etcd] - - inputs = { - list = "${join(",", formatlist("http://%s:2379", local.etcd_vpn_ips))}" - } + ] } output "endpoints" { - value = split(",", data.null_data_source.endpoints.outputs["list"]) + value = formatlist("http://%s:2379", local.etcd_vpn_ips) } From 50add0f72b29ba0ae710282995404416adb978b2 Mon Sep 17 00:00:00 2001 From: Andreas Sommer Date: Sun, 10 Sep 2023 21:49:44 +0200 Subject: [PATCH 3/3] Allow setting extra config in `KubeletConfiguration` --- service/kubernetes/main.tf | 10 ++++++++++ service/kubernetes/templates/master-configuration.yml | 1 + 2 files changed, 11 insertions(+) diff --git a/service/kubernetes/main.tf b/service/kubernetes/main.tf index bd1d855..effac51 100644 --- a/service/kubernetes/main.tf +++ b/service/kubernetes/main.tf @@ -14,6 +14,15 @@ variable "apiserver_extra_volumes" { default = [] } +variable "kubelet_extra_config" { + # `map(any)` together with `yamlencode` might turn boolean values into strings, making the YAML + # invalid, so we instead support verbatim YAML input + type = string + + description = "Extra config appended to KubeletConfiguration. Only applies at cluster creation." + default = "" +} + variable "node_count" {} variable "connections" { @@ -93,6 +102,7 @@ resource "null_resource" "kubernetes" { apiserver_extra_volumes = yamlencode(var.apiserver_extra_volumes) etcd_endpoints = "- ${join("\n - ", var.etcd_endpoints)}" cert_sans = "- ${element(var.connections, 0)}" + kubelet_extra_config = var.kubelet_extra_config }) destination = "/tmp/master-configuration.yml" } diff --git a/service/kubernetes/templates/master-configuration.yml b/service/kubernetes/templates/master-configuration.yml index 1d42dcd..17a0a7b 100644 --- a/service/kubernetes/templates/master-configuration.yml +++ b/service/kubernetes/templates/master-configuration.yml @@ -23,3 +23,4 @@ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration failSwapOn: false cgroupDriver: systemd +${kubelet_extra_config}