update version

README.md

@@ -25,11 +25,11 @@ Having a single Kubernetes control plane that spans multiple cloud providers can
 | [Azure](azure)         | 1.3.4  | CCM,CSI,Autoscaler | many regions, many zones | ✓ | ✓ |
 | [Exoscale](exoscale)   | 1.3.0  | CCM,Autoscaler     | many regions | ✗ | |
 | [GCP](gcp-zonal)       | 1.3.4  | CCM,CSI,Autoscaler | one region, many zones | ✓ | ✓ |
-| [Hetzner](hetzner)     | 1.4.0  | CCM,CSI,Autoscaler | many regions, one network zone | ✗ | ✓ |
+| [Hetzner](hetzner)     | 1.8.0  | CCM,CSI,Autoscaler | many regions, one network zone | ✗ | ✓ |
 | [Openstack](openstack) | 1.3.4  | CCM,CSI            | many regions, many zones | ✓ | ✓ |
-| [Oracle](oracle)       | 1.3.4  | CCM,~~CSI~~,Autoscaler | one region, many zones | ✓ | ✓ |
+| [Oracle](oracle)       | 1.3.4  | CCM,CSI,Autoscaler | one region, many zones | ✓ | ✓ |
-| [Proxmox](proxmox)     | 1.3.4  | CCM,CSI            | one region, mny zones | ✓ | ✓ |
+| [Proxmox](proxmox)     | 1.8.0  | CCM,CSI            | one region, mny zones | ✓ | ✓ |
-| [Scaleway](scaleway)   | 1.3.4  | CCM,CSI            | one region | ✓ | ✓ |
+| [Scaleway](scaleway)   | 1.8.0  | CCM,CSI            | one region | ✓ | ✓ |
 
 ## Known issues
 

hetzner/.gitignore

@@ -1,5 +1,10 @@
 _cfgs/
-templates/controlplane.yaml
+.terraform.lock.hcl
-controlplane-*.yaml
+.terraform.tfstate.lock.info
-worker-*.yaml
+/terraform.tfstate
-*.patch
+terraform.tfstate.backup
+terraform.tfvars
+terraform.tfvars.json
+terraform.tfvars.sops.json
+#
+age.key.txt

hetzner/.sops.yaml

@@ -0,0 +1,21 @@
+---
+creation_rules:
+  - path_regex: \.env\.yaml$
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
+  - path_regex: terraform\.tfvars\.sops\.json$
+    encrypted_regex: "(token|Secret|ID)"
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
+  - path_regex: _cfgs/controlplane.yaml$
+    encrypted_regex: "(token|key|secret|id)"
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf
+  - path_regex: _cfgs/talosconfig$
+    encrypted_regex: "key"
+    key_groups:
+      - age:
+          - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf

hetzner/Makefile

@@ -1,9 +1,11 @@
 
 CLUSTERNAME := "talos-k8s-hetzner"
 CPFIRST := ${shell terraform output -raw controlplane_firstnode 2>/dev/null}
-ENDPOINT := ${shell terraform output -raw controlplane_endpoint 2>/dev/null}
+ENDPOINT := ${shell terraform output -raw controlplane_firstnode 2>/dev/null}
-ifneq (,$(findstring Warning,${ENDPOINT}))
+ifeq ($(ENDPOINT),)
-ENDPOINT := api.cluster.local
+ENDPOINT := 127.0.0.1
+else ifneq (,$(findstring Warning,${ENDPOINT}))
+ENDPOINT := 127.0.0.1
 endif
 
 help:
@@ -11,23 +13,18 @@ help:
 
 clean: ## Clean all
 	terraform destroy -auto-approve
-	rm -rf _cfgs
+	rm -rf .terraform.lock.hcl .terraform/ terraform.tfstate terraform.tfstate.backup
-	rm -f kubeconfig terraform.tfvars.json
+	rm -f kubeconfig terraform.tfvars.sops.json
 
 prepare:
 	@[ -f ~/.ssh/terraform ] || ssh-keygen -f ~/.ssh/terraform -N '' -t rsa
 
-create-lb: ## Create load balancer
-	terraform init
-	terraform apply -auto-approve -target=hcloud_floating_ip.api -target=hcloud_load_balancer.api
-	terraform refresh
-
 create-config: ## Genereate talos configs
 	talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false ${CLUSTERNAME} https://${ENDPOINT}:6443
 	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
 
 create-templates:
-	@echo 'podSubnets: "10.32.0.0/12,fd40:10:32::/102"'        >  _cfgs/tfstate.vars
+	@echo 'podSubnets: "10.32.0.0/12,fd40:10:32::/96"'         >  _cfgs/tfstate.vars
 	@echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"'  >> _cfgs/tfstate.vars
 	@echo 'apiDomain: api.cluster.local'                       >> _cfgs/tfstate.vars
 	@yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}'       >> _cfgs/tfstate.vars
@@ -39,36 +36,66 @@ create-templates:
 	@yq eval '.cluster.token'  _cfgs/controlplane.yaml            | awk '{ print "token: "$$1}'        >> _cfgs/tfstate.vars
 	@yq eval '.cluster.ca.crt' _cfgs/controlplane.yaml            | awk '{ print "ca: "$$1}'           >> _cfgs/tfstate.vars
 
-	@yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json
+	@yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.sops.json
+	@sops --encrypt -i terraform.tfvars.sops.json
+	@yq eval .ca _cfgs/tfstate.vars | base64 --decode > _cfgs/ca.crt
+	@sops --encrypt --input-type=yaml --output-type=yaml _cfgs/talosconfig       > _cfgs/talosconfig.sops.yaml
+	@sops --encrypt --input-type=yaml --output-type=yaml _cfgs/controlplane.yaml > _cfgs/controlplane.sops.yaml
 
-create-controlplane-bootstrap:
+create-lb: ## Create load balancer
-	talosctl --talosconfig _cfgs/talosconfig config endpoint ${CPFIRST}
+	terraform init
-	talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} bootstrap
+	terraform apply -auto-approve -target=hcloud_floating_ip.api -target=hcloud_load_balancer.api
-
+	terraform refresh
-create-controlplane: ## Bootstrap first controlplane node
-	terraform apply -auto-approve -target=hcloud_server.controlplane -target=null_resource.controlplane
 
 create-infrastructure: ## Bootstrap all nodes
 	terraform apply
 
-create-kubeconfig: ## Prepare kubeconfig
+bootstrap: ## Bootstrap controlplane
-	talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} kubeconfig .
+	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
-	kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://${ENDPOINT}:6443
+	talosctl --talosconfig _cfgs/talosconfig --nodes ${ENDPOINT} bootstrap
-	kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system
 
-create-secrets:
+.PHONY: kubeconfig
-	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
+kubeconfig: ## Download kubeconfig
-	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret
+	rm -f kubeconfig
-	rm -f hcloud-csi-secret.secret
+	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
+	talosctl --talosconfig _cfgs/talosconfig --nodes ${ENDPOINT} kubeconfig .
+	kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://[${ENDPOINT}]:6443
+	kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system
 
 helm-repos: ## add helm repos
 	helm repo add hcloud               https://charts.hetzner.cloud
 	helm repo add autoscaler           https://kubernetes.github.io/autoscaler
 	helm repo update
 
-create-deployments:
+system-static:
+	helm template --namespace=kube-system -f deployments/talos-ccm.yaml \
+		--set useDaemonSet=true \
+		talos-cloud-controller-manager \
+		oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager > deployments/talos-cloud-controller-manager-result.yaml
+
 	helm template --namespace=kube-system -f deployments/hcloud-ccm.yaml \
 		hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml
 
-	helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \
+	# helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \
-		autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml
+	# 	autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml
+
+system:
+	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system --version=1.15.7 -f deployments/cilium.yaml \
+		cilium cilium/cilium
+
+	kubectl --kubeconfig=kubeconfig -n kube-system delete svc cilium-agent
+
+	kubectl --kubeconfig=kubeconfig apply -f ../_deployments/vars/coredns-local.yaml
+
+	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system -f ../_deployments/vars/metrics-server.yaml \
+		metrics-server metrics-server/metrics-server
+
+	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system -f deployments/talos-ccm.yaml \
+		--set useDaemonSet=true \
+		talos-cloud-controller-manager \
+		oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager
+
+deploy-csi:
+	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
+	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret
+	rm -f hcloud-csi-secret.secret

hetzner/common.tf

@@ -1,6 +1,6 @@
 
 data "hcloud_image" "talos" {
-  for_each          = toset(["amd64", "arm64"])
+  for_each          = toset(var.arch)
   with_architecture = each.key == "amd64" ? "x86" : "arm"
   with_selector     = "type=infra"
 }

hetzner/deployments/hcloud-cloud-controller-manager-result.yaml

@@ -61,8 +61,7 @@ spec:
           effect: "NoExecute"
       containers:
         - name: hcloud-cloud-controller-manager
-          command:
+          args:
-            - "/bin/hcloud-cloud-controller-manager"
             - "--allow-untagged-cloud"
             - "--cloud-provider=hcloud"
             - "--route-reconciliation-period=30s"
@@ -74,11 +73,19 @@ spec:
                 secretKeyRef:
                   key: token
                   name: hcloud
-            - name: NODE_NAME
+            - name: ROBOT_PASSWORD
               valueFrom:
-                fieldRef:
+                secretKeyRef:
-                  fieldPath: spec.nodeName
+                  key: robot-password
-          image: hetznercloud/hcloud-cloud-controller-manager:v1.17.2 # x-release-please-version
+                  name: hcloud
+                  optional: true
+            - name: ROBOT_USER
+              valueFrom:
+                secretKeyRef:
+                  key: robot-user
+                  name: hcloud
+                  optional: true
+          image: docker.io/hetznercloud/hcloud-cloud-controller-manager:v1.20.0 # x-release-please-version
           ports:
             - name: metrics
               containerPort: 8233

hetzner/deployments/talos-ccm.yaml

@@ -0,0 +1,56 @@
+
+image:
+  # repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager
+  tag: edge
+
+service:
+  containerPort: 50258
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/scheme: "https"
+    prometheus.io/port: "50258"
+
+logVerbosityLevel: 4
+
+enabledControllers:
+  - cloud-node
+  # - node-ipam-controller
+
+# extraArgs:
+#   - --allocate-node-cidrs
+#   - --cidr-allocator-type=CloudAllocator
+#   - --node-cidr-mask-size-ipv4=24
+#   - --node-cidr-mask-size-ipv6=80
+
+tolerations:
+  - effect: NoSchedule
+    operator: Exists
+
+transformations:
+  - name: web
+    nodeSelector:
+      - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+              - ^web-.+$
+    labels:
+      node-role.kubernetes.io/web: ""
+  - name: worker
+    nodeSelector:
+      - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+              - ^worker-.+$
+    labels:
+      node-role.kubernetes.io/worker: ""
+  - name: db
+    nodeSelector:
+      - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+              - ^db-.+$
+    labels:
+      node-role.kubernetes.io/db: ""

hetzner/deployments/talos-cloud-controller-manager-result.yaml

@@ -0,0 +1,318 @@
+---
+# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.3.1
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "v1.6.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+---
+# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml
+apiVersion: talos.dev/v1alpha1
+kind: ServiceAccount
+metadata:
+  name: talos-cloud-controller-manager-talos-secrets
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.3.1
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "v1.6.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+spec:
+  roles:
+    - os:reader
+---
+# Source: talos-cloud-controller-manager/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.3.1
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "v1.6.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+data:
+  ccm-config.yaml: |
+    global:
+      approveNodeCSR: true
+    transformations:
+      - labels:
+          node-role.kubernetes.io/web: ""
+        name: web
+        nodeSelector:
+        - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+            - ^web-.+$
+      - labels:
+          node-role.kubernetes.io/worker: ""
+        name: worker
+        nodeSelector:
+        - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+            - ^worker-.+$
+      - labels:
+          node-role.kubernetes.io/db: ""
+        name: db
+        nodeSelector:
+        - matchExpressions:
+          - key: hostname
+            operator: Regexp
+            values:
+            - ^db-.+$
+---
+# Source: talos-cloud-controller-manager/templates/role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.3.1
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "v1.6.0"
+    app.kubernetes.io/managed-by: Helm
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  verbs:
+  - approve
+---
+# Source: talos-cloud-controller-manager/templates/rolebinding.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:talos-cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:talos-cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: talos-cloud-controller-manager
+  namespace: kube-system
+---
+# Source: talos-cloud-controller-manager/templates/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:talos-cloud-controller-manager:extension-apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: talos-cloud-controller-manager
+    namespace: kube-system
+---
+# Source: talos-cloud-controller-manager/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.3.1
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "v1.6.0"
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    prometheus.io/port: "50258"
+    prometheus.io/scheme: https
+    prometheus.io/scrape: "true"
+  namespace: kube-system
+spec:
+  clusterIP: None
+  type: ClusterIP
+  ports:
+    - name: https
+      port: 50258
+      targetPort: 50258
+      protocol: TCP
+  selector:
+        app.kubernetes.io/name: talos-cloud-controller-manager
+        app.kubernetes.io/instance: talos-cloud-controller-manager
+---
+# Source: talos-cloud-controller-manager/templates/deployment.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.3.1
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "v1.6.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: talos-cloud-controller-manager
+      app.kubernetes.io/instance: talos-cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: talos-cloud-controller-manager
+        app.kubernetes.io/instance: talos-cloud-controller-manager
+    spec:
+      serviceAccountName: talos-cloud-controller-manager
+      securityContext:
+        fsGroup: 10258
+        fsGroupChangePolicy: OnRootMismatch
+        runAsGroup: 10258
+        runAsNonRoot: true
+        runAsUser: 10258
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: talos-cloud-controller-manager
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          image: "ghcr.io/siderolabs/talos-cloud-controller-manager:edge"
+          imagePullPolicy: IfNotPresent
+          command: ["/talos-cloud-controller-manager"]
+          args:
+            - --v=4
+            - --cloud-provider=talos
+            - --cloud-config=/etc/talos/ccm-config.yaml
+            - --controllers=cloud-node
+            - --leader-elect-resource-name=cloud-controller-manager-talos
+            - --use-service-account-credentials
+            - --secure-port=50258
+            - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics
+          env:
+            - name: TALOS_ENDPOINTS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KUBERNETES_SERVICE_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KUBERNETES_SERVICE_PORT
+              value: "6443"
+          ports:
+            - containerPort: 50258
+              name: https
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: https
+              scheme: HTTPS
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+          volumeMounts:
+            - name: cloud-config
+              mountPath: /etc/talos
+              readOnly: true
+            - name: talos-secrets
+              mountPath: /var/run/secrets/talos.dev
+              readOnly: true
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/not-ready
+          operator: Exists
+      volumes:
+        - name: cloud-config
+          configMap:
+            name: talos-cloud-controller-manager
+            defaultMode: 416 # 0640
+        - name: talos-secrets
+          secret:
+            secretName: talos-cloud-controller-manager-talos-secrets
+            defaultMode: 416 # 0640

hetzner/images/hetzner.pkr.hcl

@@ -2,8 +2,8 @@
 packer {
   required_plugins {
     hcloud = {
-      version = ">= 1.0.5"
+      version = ">= 1.5.0"
-      source  = "github.com/hashicorp/hcloud"
+      source  = "github.com/hetznercloud/hcloud"
     }
   }
 }
@@ -11,7 +11,7 @@ packer {
 source "hcloud" "talos" {
   token       = var.hcloud_token
   rescue      = "linux64"
-  image       = "debian-11"
+  image       = "debian-12"
   location    = var.hcloud_location
   server_type = var.hcloud_type
 

hetzner/images/variables.pkr.hcl

@@ -12,12 +12,12 @@ variable "hcloud_location" {
 
 variable "hcloud_type" {
   type      = string
-  default   = "cx11" # cx11|cax11 (arm)
+  default   = "cax11" # cx11|cax11 (arm)
 }
 
 variable "talos_version" {
   type    = string
-  default = "v1.4.1"
+  default = "v1.7.6"
 }
 
 locals {

hetzner/instances-controlplane.tf

@@ -35,22 +35,6 @@ resource "hcloud_server" "controlplane" {
     ip         = each.value.ip
   }
 
-  #   user_data = templatefile("${path.module}/templates/controlplane.yaml",
-  #     merge(var.kubernetes, {
-  #       name           = each.value.name
-  #       ipv4_vip       = local.ipv4_vip
-  #       ipv4_local     = each.value.ip
-  #       lbv4_local     = local.lbv4_local
-  #       lbv4           = local.lbv4
-  #       lbv6           = local.lbv6
-  #       hcloud_network = hcloud_network.main.id
-  #       hcloud_token   = var.hcloud_token
-  #       hcloud_image   = data.hcloud_image.talos["amd64"].id
-  #       robot_user     = var.robot_user
-  #       robot_password = var.robot_password
-  #     })
-  #   )
-
   lifecycle {
     ignore_changes = [
       network,
@@ -73,38 +57,34 @@ resource "hcloud_load_balancer_target" "api" {
 # Secure push talos config to the controlplane
 #
 
-resource "local_file" "controlplane" {
+resource "local_sensitive_file" "controlplane" {
   for_each = local.controlplanes
-
   content = templatefile("${path.module}/templates/controlplane.yaml.tpl",
-    {
+    merge(local.kubernetes, try(var.instances["all"], {}), {
       name        = each.value.name
-      apiDomain      = var.kubernetes["apiDomain"]
+      nodeSubnets = hcloud_network_subnet.core.ip_range
-      domain         = var.kubernetes["domain"]
-      podSubnets     = var.kubernetes["podSubnets"]
-      serviceSubnets = var.kubernetes["serviceSubnets"]
       ipv4_vip    = local.ipv4_vip
       ipv4_local  = each.value.ip
       lbv4_local  = local.lbv4_local
       lbv4        = local.lbv4
       lbv6        = local.lbv6
-      nodeSubnets    = hcloud_network_subnet.core.ip_range
+
       hcloud_network = hcloud_network.main.id
       hcloud_token   = var.hcloud_token
       hcloud_image   = data.hcloud_image.talos["amd64"].id
       hcloud_sshkey  = hcloud_ssh_key.infra.id
       robot_user     = var.robot_user
       robot_password = var.robot_password
-    }
+    })
   )
   filename        = "_cfgs/${each.value.name}.yaml"
   file_permission = "0600"
 }
 
-resource "null_resource" "controlplane" {
+locals {
-  for_each = local.controlplanes
+  controlplane_config = { for k, v in local.controlplanes : v.name => "talosctl apply-config --insecure --nodes ${hcloud_server.controlplane[k].ipv4_address} --config-patch @_cfgs/${v.name}.yaml --file _cfgs/controlplane.yaml" }
-  provisioner "local-exec" {
+}
-    command = "sleep 30 && talosctl apply-config --insecure --nodes ${hcloud_server.controlplane[each.key].ipv4_address} --timeout 5m0s --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml"
+
-  }
+output "controlplane_config" {
-  depends_on = [hcloud_load_balancer_target.api, local_file.controlplane]
+  value = local.controlplane_config
 }

hetzner/templates/controlplane.yaml.tpl

@@ -7,6 +7,7 @@ machine:
     - "${ipv4_vip}"
     - "${apiDomain}"
   kubelet:
+    image: ghcr.io/siderolabs/kubelet:${version}
     extraArgs:
       rotate-server-certificates: true
     clusterDNS:
@@ -15,7 +16,7 @@ machine:
     nodeIP:
       validSubnets: ${format("%#v",split(",",nodeSubnets))}
   network:
-    hostname: "${name}"
+    hostname: ${name}
     interfaces:
       - interface: eth0
         dhcp: true
@@ -65,9 +66,11 @@ machine:
         - kube-system
 cluster:
   adminKubeconfig:
-    certLifetime: 8h0m0s
+    certLifetime: 48h0m0s
   controlPlane:
     endpoint: https://${apiDomain}:6443
+  discovery:
+    enabled: false
   network:
     dnsDomain: ${domain}
     podSubnets: ${format("%#v",split(",",podSubnets))}
@@ -79,6 +82,7 @@ cluster:
   proxy:
     disabled: true
   apiServer:
+    image: registry.k8s.io/kube-apiserver:${version}
     certSANs:
       - "${lbv4}"
       - "${lbv6}"
@@ -87,9 +91,12 @@ cluster:
       - "${ipv4_vip}"
       - "${apiDomain}"
   controllerManager:
+    image: registry.k8s.io/kube-controller-manager:${version}
     extraArgs:
-        node-cidr-mask-size-ipv4: 24
+        node-cidr-mask-size-ipv4: "24"
-        node-cidr-mask-size-ipv6: 112
+        node-cidr-mask-size-ipv6: "112"
+  scheduler:
+    image: registry.k8s.io/kube-scheduler:${version}
   etcd:
     advertisedSubnets:
       - ${nodeSubnets}
@@ -114,10 +121,8 @@ cluster:
   externalCloudProvider:
     enabled: true
     manifests:
-      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/talos-cloud-controller-manager-result.yaml
-      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml
-      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-csi.yaml
-      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-ns.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-result.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/coredns-local.yaml

hetzner/variables.tf

@@ -25,19 +25,18 @@ variable "regions" {
   default     = ["nbg1", "fsn1", "hel1"]
 }
 
-variable "kubernetes" {
+variable "arch" {
-  type = map(string)
+  description = "The Talos architecture list"
-  default = {
+  type        = list(string)
-    podSubnets     = "10.32.0.0/12,fd40:10:32::/102"
+  default     = ["amd64", "arm64"]
-    serviceSubnets = "10.200.0.0/22,fd40:10:200::/112"
+}
-    apiDomain      = "api.cluster.local"
+
-    domain         = "cluster.local"
+data "sops_file" "tfvars" {
-    clusterName    = "talos-k8s-hetzner"
+  source_file = "terraform.tfvars.sops.json"
-    tokenMachine   = ""
+}
-    caMachine      = ""
+
-    token          = ""
+locals {
-    ca             = ""
+  kubernetes = jsondecode(data.sops_file.tfvars.raw)["kubernetes"]
-  }
 }
 
 variable "vpc_main_cidr" {
@@ -65,7 +64,6 @@ variable "controlplane" {
     "all" = {
       type_lb = "" # lb11, if "" use floating-ip
     },
-
     "nbg1" = {
       count = 0,
       type  = "cpx11",
@@ -85,6 +83,9 @@ variable "instances" {
   description = "Map of region's properties"
   type        = map(any)
   default = {
+    "all" = {
+      version = "v1.30.2"
+    },
     "nbg1" = {
       web_count    = 0,
       web_type     = "cx11",

hetzner/versions.tf

@@ -2,8 +2,12 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = "~> 1.38.2"
+      version = "~> 1.45"
+    }
+    sops = {
+      source  = "carlpett/sops"
+      version = "1.0.0"
     }
   }
-  required_version = ">= 1.2"
+  required_version = ">= 1.5"
 }

proxmox/variables.tf

@@ -38,7 +38,7 @@ variable "vpc_main_cidr" {
 variable "release" {
   type        = string
   description = "The version of the Talos image"
-  default     = "1.7.4"
+  default     = "1.8.0"
 }
 
 data "sops_file" "tfvars" {
@@ -97,7 +97,7 @@ variable "instances" {
   type        = map(any)
   default = {
     "all" = {
-      version = "v1.30.2"
+      version = "v1.31.0"
     },
     "hvm-1" = {
       enabled         = false,

scaleway/images/variables.pkr.hcl

@@ -29,7 +29,7 @@ variable "scaleway_type" {
 
 variable "talos_version" {
   type    = string
-  default = "v1.7.6"
+  default = "v1.8.0"
 }
 
 locals {