update version

sergelogvinov · Aug 14, 2024 · 358858c · 358858c
1 parent e048cf0
commit 358858c
Show file tree

Hide file tree

Showing 16 changed files with 537 additions and 113 deletions.
diff --git a/README.md b/README.md
@@ -25,11 +25,11 @@ Having a single Kubernetes control plane that spans multiple cloud providers can
 | [Azure](azure)         | 1.3.4  | CCM,CSI,Autoscaler | many regions, many zones | &check; | &check; |
 | [Exoscale](exoscale)   | 1.3.0  | CCM,Autoscaler     | many regions | &cross; | |
 | [GCP](gcp-zonal)       | 1.3.4  | CCM,CSI,Autoscaler | one region, many zones | &check; | &check; |
-| [Hetzner](hetzner)     | 1.4.0  | CCM,CSI,Autoscaler | many regions, one network zone | &cross; | &check; |
+| [Hetzner](hetzner)     | 1.8.0  | CCM,CSI,Autoscaler | many regions, one network zone | &cross; | &check; |
 | [Openstack](openstack) | 1.3.4  | CCM,CSI            | many regions, many zones | &check; | &check; |
-| [Oracle](oracle)       | 1.3.4  | CCM,~~CSI~~,Autoscaler | one region, many zones | &check; | &check; |
-| [Proxmox](proxmox)     | 1.3.4  | CCM,CSI            | one region, mny zones | &check; | &check; |
-| [Scaleway](scaleway)   | 1.3.4  | CCM,CSI            | one region | &check; | &check; |
+| [Oracle](oracle)       | 1.3.4  | CCM,CSI,Autoscaler | one region, many zones | &check; | &check; |
+| [Proxmox](proxmox)     | 1.8.0  | CCM,CSI            | one region, mny zones | &check; | &check; |
+| [Scaleway](scaleway)   | 1.8.0  | CCM,CSI            | one region | &check; | &check; |
 
 ## Known issues
 

diff --git a/hetzner/.gitignore b/hetzner/.gitignore
@@ -1,5 +1,10 @@
 _cfgs/
-templates/controlplane.yaml
-controlplane-*.yaml
-worker-*.yaml
-*.patch
+.terraform.lock.hcl
+.terraform.tfstate.lock.info
+/terraform.tfstate
+terraform.tfstate.backup
+terraform.tfvars
+terraform.tfvars.json
+terraform.tfvars.sops.json
+#
+age.key.txt
diff --git a/hetzner/.sops.yaml b/hetzner/.sops.yaml
@@ -0,0 +1,21 @@
+---
+creation_rules:
+  - path_regex: \.env\.yaml$
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
+  - path_regex: terraform\.tfvars\.sops\.json$
+    encrypted_regex: "(token|Secret|ID)"
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
+  - path_regex: _cfgs/controlplane.yaml$
+    encrypted_regex: "(token|key|secret|id)"
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
+  - path_regex: _cfgs/talosconfig$
+    encrypted_regex: "key"
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
diff --git a/hetzner/Makefile b/hetzner/Makefile
@@ -1,33 +1,30 @@
 
 CLUSTERNAME := "talos-k8s-hetzner"
 CPFIRST := ${shell terraform output -raw controlplane_firstnode 2>/dev/null}
-ENDPOINT := ${shell terraform output -raw controlplane_endpoint 2>/dev/null}
-ifneq (,$(findstring Warning,${ENDPOINT}))
-ENDPOINT := api.cluster.local
+ENDPOINT := ${shell terraform output -raw controlplane_firstnode 2>/dev/null}
+ifeq ($(ENDPOINT),)
+ENDPOINT := 127.0.0.1
+else ifneq (,$(findstring Warning,${ENDPOINT}))
+ENDPOINT := 127.0.0.1
 endif
 
 help:
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
 clean: ## Clean all
 	terraform destroy -auto-approve
-	rm -rf _cfgs
-	rm -f kubeconfig terraform.tfvars.json
+	rm -rf .terraform.lock.hcl .terraform/ terraform.tfstate terraform.tfstate.backup
+	rm -f kubeconfig terraform.tfvars.sops.json
 
 prepare:
 	@[ -f ~/.ssh/terraform ] || ssh-keygen -f ~/.ssh/terraform -N '' -t rsa
 
-create-lb: ## Create load balancer
-	terraform init
-	terraform apply -auto-approve -target=hcloud_floating_ip.api -target=hcloud_load_balancer.api
-	terraform refresh
-
 create-config: ## Genereate talos configs
 	talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false ${CLUSTERNAME} https://${ENDPOINT}:6443
 	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
 
 create-templates:
-	@echo 'podSubnets: "10.32.0.0/12,fd40:10:32::/102"'        >  _cfgs/tfstate.vars
+	@echo 'podSubnets: "10.32.0.0/12,fd40:10:32::/96"'         >  _cfgs/tfstate.vars
 	@echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"'  >> _cfgs/tfstate.vars
 	@echo 'apiDomain: api.cluster.local'                       >> _cfgs/tfstate.vars
 	@yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}'       >> _cfgs/tfstate.vars
@@ -39,36 +36,66 @@ create-templates:
 	@yq eval '.cluster.token'  _cfgs/controlplane.yaml            | awk '{ print "token: "$$1}'        >> _cfgs/tfstate.vars
 	@yq eval '.cluster.ca.crt' _cfgs/controlplane.yaml            | awk '{ print "ca: "$$1}'           >> _cfgs/tfstate.vars
 
-	@yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json
-
-create-controlplane-bootstrap:
-	talosctl --talosconfig _cfgs/talosconfig config endpoint ${CPFIRST}
-	talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} bootstrap
+	@yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.sops.json
+	@sops --encrypt -i terraform.tfvars.sops.json
+	@yq eval .ca _cfgs/tfstate.vars | base64 --decode > _cfgs/ca.crt
+	@sops --encrypt --input-type=yaml --output-type=yaml _cfgs/talosconfig       > _cfgs/talosconfig.sops.yaml
+	@sops --encrypt --input-type=yaml --output-type=yaml _cfgs/controlplane.yaml > _cfgs/controlplane.sops.yaml
 
-create-controlplane: ## Bootstrap first controlplane node
-	terraform apply -auto-approve -target=hcloud_server.controlplane -target=null_resource.controlplane
+create-lb: ## Create load balancer
+	terraform init
+	terraform apply -auto-approve -target=hcloud_floating_ip.api -target=hcloud_load_balancer.api
+	terraform refresh
 
 create-infrastructure: ## Bootstrap all nodes
 	terraform apply
 
-create-kubeconfig: ## Prepare kubeconfig
-	talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} kubeconfig .
-	kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://${ENDPOINT}:6443
-	kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system
+bootstrap: ## Bootstrap controlplane
+	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
+	talosctl --talosconfig _cfgs/talosconfig --nodes ${ENDPOINT} bootstrap
 
-create-secrets:
-	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
-	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret
-	rm -f hcloud-csi-secret.secret
+.PHONY: kubeconfig
+kubeconfig: ## Download kubeconfig
+	rm -f kubeconfig
+	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
+	talosctl --talosconfig _cfgs/talosconfig --nodes ${ENDPOINT} kubeconfig .
+	kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://[${ENDPOINT}]:6443
+	kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system
 
 helm-repos: ## add helm repos
 	helm repo add hcloud               https://charts.hetzner.cloud
 	helm repo add autoscaler           https://kubernetes.github.io/autoscaler
 	helm repo update
 
-create-deployments:
+system-static:
+	helm template --namespace=kube-system -f deployments/talos-ccm.yaml \
+		--set useDaemonSet=true \
+		talos-cloud-controller-manager \
+		oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager > deployments/talos-cloud-controller-manager-result.yaml
+
 	helm template --namespace=kube-system -f deployments/hcloud-ccm.yaml \
 		hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml
 
-	helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \
-		autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml
+	# helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \
+	# 	autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml
+
+system:
+	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system --version=1.15.7 -f deployments/cilium.yaml \
+		cilium cilium/cilium
+
+	kubectl --kubeconfig=kubeconfig -n kube-system delete svc cilium-agent
+
+	kubectl --kubeconfig=kubeconfig apply -f ../_deployments/vars/coredns-local.yaml
+
+	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system -f ../_deployments/vars/metrics-server.yaml \
+		metrics-server metrics-server/metrics-server
+
+	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system -f deployments/talos-ccm.yaml \
+		--set useDaemonSet=true \
+		talos-cloud-controller-manager \
+		oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager
+
+deploy-csi:
+	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
+	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret
+	rm -f hcloud-csi-secret.secret
diff --git a/hetzner/common.tf b/hetzner/common.tf
@@ -1,6 +1,6 @@
 
 data "hcloud_image" "talos" {
-  for_each          = toset(["amd64", "arm64"])
+  for_each          = toset(var.arch)
   with_architecture = each.key == "amd64" ? "x86" : "arm"
   with_selector     = "type=infra"
 }

diff --git a/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml b/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml
@@ -61,8 +61,7 @@ spec:
           effect: "NoExecute"
       containers:
         - name: hcloud-cloud-controller-manager
-          command:
-            - "/bin/hcloud-cloud-controller-manager"
+          args:
             - "--allow-untagged-cloud"
             - "--cloud-provider=hcloud"
             - "--route-reconciliation-period=30s"
@@ -74,11 +73,19 @@ spec:
                 secretKeyRef:
                   key: token
                   name: hcloud
-            - name: NODE_NAME
+            - name: ROBOT_PASSWORD
               valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          image: hetznercloud/hcloud-cloud-controller-manager:v1.17.2 # x-release-please-version
+                secretKeyRef:
+                  key: robot-password
+                  name: hcloud
+                  optional: true
+            - name: ROBOT_USER
+              valueFrom:
+                secretKeyRef:
+                  key: robot-user
+                  name: hcloud
+                  optional: true
+          image: docker.io/hetznercloud/hcloud-cloud-controller-manager:v1.20.0 # x-release-please-version
           ports:
             - name: metrics
               containerPort: 8233

diff --git a/hetzner/deployments/talos-ccm.yaml b/hetzner/deployments/talos-ccm.yaml
@@ -0,0 +1,56 @@
+
+image:
+  # repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager
+  tag: edge
+
+service:
+  containerPort: 50258
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/scheme: "https"
+    prometheus.io/port: "50258"
+
+logVerbosityLevel: 4
+
+enabledControllers:
+  - cloud-node
+  # - node-ipam-controller
+
+# extraArgs:
+#   - --allocate-node-cidrs
+#   - --cidr-allocator-type=CloudAllocator
+#   - --node-cidr-mask-size-ipv4=24
+#   - --node-cidr-mask-size-ipv6=80
+
+tolerations:
+  - effect: NoSchedule
+    operator: Exists
+
+transformations:
+  - name: web
+    nodeSelector:
+      - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+              - ^web-.+$
+    labels:
+      node-role.kubernetes.io/web: ""
+  - name: worker
+    nodeSelector:
+      - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+              - ^worker-.+$
+    labels:
+      node-role.kubernetes.io/worker: ""
+  - name: db
+    nodeSelector:
+      - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+              - ^db-.+$
+    labels:
+      node-role.kubernetes.io/db: ""