update cloud deploys

.gitignore

@@ -33,6 +33,6 @@ override.tf.json
 talosctl
 talosconfig
 kubeconfig
-
+*.secret
 #
 /talos/

hetzner/Makefile

@@ -60,3 +60,7 @@ create-deployments:
 		cilium/cilium > deployments/cilium-result.yaml
 	helm template --namespace=ingress-nginx --version=4.4.0 -f deployments/ingress.yaml ingress-nginx \
 		ingress-nginx/ingress-nginx > deployments/ingress-result.yaml
+
+create-secrets:
+	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
+	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret

hetzner/deployments/hcloud-cloud-controller-manager.yaml

@@ -38,31 +38,22 @@ spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: cloud-controller-manager
       dnsPolicy: Default
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
       tolerations:
         - key: "node.cloudprovider.kubernetes.io/uninitialized"
           value: "true"
           effect: "NoSchedule"
-        - key: "node-role.kubernetes.io/master"
-          effect: NoSchedule
         - key: "node-role.kubernetes.io/control-plane"
           effect: NoSchedule
-        - key: "node.kubernetes.io/not-ready"
-          effect: "NoSchedule"
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
       containers:
-        # - image: hetznercloud/hcloud-cloud-controller-manager:v1.12.1
-        - image: ghcr.io/sergelogvinov/hetzner-cloud-controller-manager:v1.12.2-dev
+        - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2
+        # - image: ghcr.io/sergelogvinov/hetzner-cloud-controller-manager:v1.12.2-dev
           name: hcloud-cloud-controller-manager
           args:
             - --cloud-provider=hcloud
             - --allow-untagged-cloud
-            - --controllers=cloud-node,cloud-node-lifecycle
+            - --controllers=cloud-node-lifecycle
           resources:
             requests:
               cpu: 100m

hetzner/deployments/hcloud-csi.yaml

@@ -8,6 +8,7 @@ spec:
   podInfoOnMount: true
   volumeLifecycleModes:
     - Persistent
+  fsGroupPolicy: File
 ---
 kind: StorageClass
 apiVersion: storage.k8s.io/v1
@@ -16,82 +17,178 @@ metadata:
   annotations:
     storageclass.kubernetes.io/is-default-class: "false"
 provisioner: csi.hetzner.cloud
+reclaimPolicy: Delete
 volumeBindingMode: WaitForFirstConsumer
 allowVolumeExpansion: true
 ---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: hcloud-volumes-enc
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: csi.hetzner.cloud
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  csi.storage.k8s.io/node-publish-secret-name: hcloud-csi-secret
+  csi.storage.k8s.io/node-publish-secret-namespace: kube-system
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: hcloud-csi
+  name: hcloud-csi-controller
   namespace: kube-system
 ---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: hcloud-csi
+  name: hcloud-csi-controller
 rules:
-  # attacher
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["csi.storage.k8s.io"]
-    resources: ["csinodeinfos"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments/status"]
-    verbs: ["patch"]
-  # provisioner
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "delete", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims", "persistentvolumeclaims/status"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["get", "list"]
-  # resizer
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "list", "watch"]
-  # node
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["get", "list", "watch", "create", "update", "patch"]
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - csi.storage.k8s.io
+  resources:
+  - csinodeinfos
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  - persistentvolumeclaims/status
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
 ---
-kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
 metadata:
-  name: hcloud-csi
-subjects:
-  - kind: ServiceAccount
-    name: hcloud-csi
-    namespace: kube-system
+  name: hcloud-csi-controller
 roleRef:
-  kind: ClusterRole
-  name: hcloud-csi
   apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: hcloud-csi-controller
+subjects:
+- kind: ServiceAccount
+  name: hcloud-csi-controller
+  namespace: kube-system
 ---
 kind: StatefulSet
 apiVersion: apps/v1
@@ -109,19 +206,12 @@ spec:
       labels:
         app: hcloud-csi-controller
     spec:
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
       tolerations:
-        - key: "node-role.kubernetes.io/master"
-          effect: NoSchedule
         - key: "node-role.kubernetes.io/control-plane"
           effect: NoSchedule
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node.kubernetes.io/instance-type
-                    operator: Exists
-      serviceAccount: hcloud-csi
+      serviceAccount: hcloud-csi-controller
       containers:
         - name: csi-attacher
           image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1
@@ -157,8 +247,10 @@ spec:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
         - name: hcloud-csi-driver
-          image: hetznercloud/hcloud-csi-driver:1.6.0
+          image: hetznercloud/hcloud-csi-driver:2.0.0
           imagePullPolicy: Always
+          command:
+            - /bin/hcloud-csi-driver-controller
           env:
             - name: CSI_ENDPOINT
               value: unix:///run/csi/socket
@@ -227,8 +319,6 @@ spec:
       tolerations:
         - effect: NoSchedule
           operator: Exists
-        - key: CriticalAddonsOnly
-          operator: Exists
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -258,8 +348,10 @@ spec:
           securityContext:
             privileged: true
         - name: hcloud-csi-driver
-          image: hetznercloud/hcloud-csi-driver:1.6.0
+          image: hetznercloud/hcloud-csi-driver:2.0.0
           imagePullPolicy: Always
+          command:
+            - /bin/hcloud-csi-driver-node
           env:
             - name: CSI_ENDPOINT
               value: unix:///run/csi/socket
@@ -267,16 +359,6 @@ spec:
               value: 0.0.0.0:9189
             - name: ENABLE_METRICS
               value: "true"
-            - name: HCLOUD_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: hcloud
-                  key: token
-            - name: KUBE_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: spec.nodeName
           volumeMounts:
             - name: kubelet-dir
               mountPath: /var/lib/kubelet

hetzner/deployments/metrics-server.yaml

@@ -136,11 +136,12 @@ spec:
       containers:
       - args:
         - --cert-dir=/tmp
-        - --secure-port=443
+        - --secure-port=6443
         - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
         - --kubelet-use-node-status-port
         - --metric-resolution=15s
-        image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1
+        - --authorization-always-allow-paths=/metrics
+        image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0
         imagePullPolicy: IfNotPresent
         livenessProbe:
           failureThreshold: 3
@@ -151,7 +152,7 @@ spec:
           periodSeconds: 10
         name: metrics-server
         ports:
-        - containerPort: 443
+        - containerPort: 6443
           name: https
           protocol: TCP
         readinessProbe:

hetzner/deployments/test-csi.yaml

@@ -8,25 +8,18 @@ spec:
   resources:
     requests:
       storage: 10Gi
-  storageClassName: hcloud-volumes
+  storageClassName: hcloud-volumes-enc
 ---
 kind: Pod
 apiVersion: v1
 metadata:
   name: csi-app
 spec:
+  nodeSelector:
+    node-role.kubernetes.io/control-plane: ""
   tolerations:
-    - effect: NoExecute
-      operator: Exists
-    - effect: NoSchedule
-      operator: Exists
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-          - matchExpressions:
-              - key: node.kubernetes.io/instance-type
-                operator: Exists
+    - key: "node-role.kubernetes.io/control-plane"
+      effect: NoSchedule
   containers:
     - name: my-frontend
       image: alpine

hetzner/instances-web.tf

@@ -17,6 +17,6 @@ module "web" {
 
   vm_params = merge(var.kubernetes, {
     lbv4   = local.ipv4_vip
-    labels = "project.io/node-pool=web,node.kubernetes.io/disktype=ssd,topology.kubernetes.io/region=${each.key},hcloud/node-group=web-${each.key}"
+    labels = "project.io/node-pool=web,node.kubernetes.io/disktype=ssd,hcloud/node-group=web-${each.key}"
   })
 }

hetzner/instances-workers.tf

@@ -17,6 +17,6 @@ module "worker" {
 
   vm_params = merge(var.kubernetes, {
     lbv4   = local.ipv4_vip
-    labels = "project.io/node-pool=worker,node.kubernetes.io/disktype=ssd,topology.kubernetes.io/region=${each.key},hcloud/node-group=worker-${each.key}"
+    labels = "project.io/node-pool=worker,node.kubernetes.io/disktype=ssd,hcloud/node-group=worker-${each.key}"
   })
 }