hetzner csi/autoscaller

2025-10-29 17:42:47 +00:00 · 2024-08-15 16:41:49 +03:00
parent 71dbd19fdc
commit d33210018b
18 changed files with 490 additions and 1304 deletions
--- a/aws/deployments/cilium.yaml
+++ b/aws/deployments/cilium.yaml
@@ -1,77 +0,0 @@
---
-
-k8sServiceHost: "api.cluster.local"
-k8sServicePort: "6443"
-
-operator:
-  enabled: true
-  rollOutPods: true
-  replicas: 1
-  prometheus:
-    enabled: false
-  nodeSelector:
-    node-role.kubernetes.io/control-plane: ""
-  tolerations:
-    - operator: Exists
-      effect: NoSchedule
-
-identityAllocationMode: crd
-kubeProxyReplacement: strict
-enableK8sEndpointSlice: true
-localRedirectPolicy: true
-
-tunnel: "vxlan"
-autoDirectNodeRoutes: false
-devices: [eth+]
-
-healthChecking: true
-
-cni:
-  install: true
-
-ipam:
-  mode: "kubernetes"
-k8s:
-  requireIPv4PodCIDR: true
-  requireIPv6PodCIDR: true
-
-bpf:
-  masquerade: false
-ipv4:
-  enabled: true
-ipv6:
-  enabled: true
-hostServices:
-  enabled: true
-hostPort:
-  enabled: true
-nodePort:
-  enabled: true
-externalIPs:
-  enabled: true
-hostFirewall:
-  enabled: true
-ingressController:
-  enabled: false
-
-securityContext:
-  privileged: true
-
-hubble:
-  enabled: false
-
-prometheus:
-  enabled: true
-
-cgroup:
-  autoMount:
-    enabled: false
-  hostRoot: /sys/fs/cgroup
-
-resources:
-  limits:
-    cpu: 2
-    memory: 1Gi
-  requests:
-    cpu: 100m
-    memory: 128Mi
--- a/aws/deployments/coredns-local.yaml
+++ b/aws/deployments/coredns-local.yaml
@@ -1,153 +0,0 @@
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns-local
-  namespace: kube-system
-data:
-  empty.db: |
-    @       60      IN      SOA     localnet. root.localnet. (
-                    1       ; serial
-                    60      ; refresh
-                    60      ; retry
-                    60      ; expiry
-                    60 )    ; minimum
-    ;
-    @       IN      NS      localnet.
-
-  hosts: |
-    # static hosts
-    169.254.2.53        dns.local
-
-  Corefile.local: |
-    (empty) {
-        file /etc/coredns/empty.db
-    }
-
-    .:53 {
-        errors
-        bind 169.254.2.53
-
-        health 127.0.0.1:8091 {
-           lameduck 5s
-        }
-
-        hosts /etc/coredns/hosts {
-            reload 60s
-            fallthrough
-        }
-
-        kubernetes cluster.local in-addr.arpa ip6.arpa {
-            endpoint https://api.cluster.local:6443
-            kubeconfig /etc/coredns/kubeconfig.conf coredns
-            pods insecure
-            ttl 60
-        }
-        prometheus :9153
-
-        forward . /etc/resolv.conf {
-            policy sequential
-            expire 30s
-        }
-
-        cache 300
-        loop
-        reload
-        loadbalance
-    }
-  kubeconfig.conf: |-
-    apiVersion: v1
-    kind: Config
-    clusters:
-    - cluster:
-        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        server: https://api.cluster.local:6443
-      name: default
-    contexts:
-    - context:
-        cluster: default
-        namespace: kube-system
-        user: coredns
-      name: coredns
-    current-context: coredns
-    users:
-    - name: coredns
-      user:
-        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
---
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: coredns-local
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns-local
-    kubernetes.io/name: CoreDNS
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  minReadySeconds: 15
-  selector:
-    matchLabels:
-      k8s-app: kube-dns-local
-      kubernetes.io/name: CoreDNS
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-dns-local
-        kubernetes.io/name: CoreDNS
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "9153"
-    spec:
-      priorityClassName: system-node-critical
-      serviceAccount: coredns
-      serviceAccountName: coredns
-      enableServiceLinks: false
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/control-plane
-        operator: Exists
-      - effect: NoSchedule
-        key: node.cloudprovider.kubernetes.io/uninitialized
-        operator: Exists
-      hostNetwork: true
-      containers:
-      - name: coredns
-        image: coredns/coredns:1.9.4
-        imagePullPolicy: IfNotPresent
-        resources:
-          limits:
-            cpu: 100m
-            memory: 128Mi
-          requests:
-            cpu: 50m
-            memory: 64Mi
-        args: [ "-conf", "/etc/coredns/Corefile.local" ]
-        volumeMounts:
-        - name: config-volume
-          mountPath: /etc/coredns
-          readOnly: true
-        livenessProbe:
-          httpGet:
-            host: 127.0.0.1
-            path: /health
-            port: 8091
-            scheme: HTTP
-          initialDelaySeconds: 60
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 5
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-          readOnlyRootFilesystem: true
-      dnsPolicy: Default
-      volumes:
-        - name: config-volume
-          configMap:
-            name: coredns-local
--- a/aws/deployments/ingress-ns.yaml
+++ b/aws/deployments/ingress-ns.yaml
@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: ingress-nginx
--- a/aws/deployments/ingress.yaml
+++ b/aws/deployments/ingress.yaml
@@ -1,116 +0,0 @@
-
-controller:
-  kind: DaemonSet
-
-  hostNetwork: true
-  hostPort:
-    enabled: false
-    ports:
-      http: 80
-      https: 443
-
-  dnsPolicy: ClusterFirstWithHostNet
-
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate
-
-  publishService:
-    enabled: false
-
-  config:
-    worker-processes: "auto"
-    worker-cpu-affinity: "auto"
-    error-log-level: "error"
-
-    server-tokens: "false"
-    http-redirect-code: "301"
-
-    use-gzip: "true"
-    use-geoip: "false"
-    use-geoip2: "false"
-
-    use-forwarded-headers: "true"
-    # curl https://www.cloudflare.com/ips-v4 2>/dev/null | tr '\n' ','
-    proxy-real-ip-cidr: "173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,172.64.0.0/13,131.0.72.0/22,104.16.0.0/13,104.24.0.0/14,172.16.0.0/12"
-
-    enable-access-log-for-default-backend: "true"
-    log-format-escape-json: "true"
-    log-format-upstream: '{"ip":"$remote_addr", "ssl":"$ssl_protocol", "method":"$request_method", "proto":"$scheme", "host":"$host", "uri":"$request_uri", "status":$status, "size":$bytes_sent, "agent":"$http_user_agent", "referer":"$http_referer", "namespace":"$namespace"}'
-
-    upstream-keepalive-connections: "32"
-    proxy-connect-timeout: "10"
-    proxy-read-timeout: "60"
-    proxy-send-timeout: "60"
-
-    ssl-protocols: "TLSv1.3"
-    hsts: "true"
-    hsts-max-age: "31536000"
-    hsts-include-subdomains: "true"
-    hsts-preload: "true"
-    proxy-hide-headers: "strict-transport-security"
-    proxy-headers-hash-bucket-size: "128"
-
-    server-name-hash-bucket-size: "64"
-    server-name-hash-max-size: "512"
-
-    limit-req-status-code: "429"
-
-    client-header-timeout: "30"
-    client-body-timeout: "30"
-
-  minReadySeconds: 15
-
-  podAnnotations:
-    prometheus.io/scrape: "true"
-    prometheus.io/port: "10254"
-
-  extraEnvs:
-    - name: NODE_NAME
-      valueFrom:
-        fieldRef:
-          fieldPath: metadata.name
-
-  livenessProbe:
-    initialDelaySeconds: 15
-    periodSeconds: 30
-  readinessProbe:
-    periodSeconds: 30
-
-  resources:
-    limits:
-      cpu: 1
-      memory: 1Gi
-    requests:
-      cpu: 100m
-      memory: 128Mi
-
-  affinity:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-          - matchExpressions:
-              - key: project.io/node-pool
-                operator: In
-                values:
-                  - web
-
-  service:
-    enabled: true
-    type: ClusterIP
-    clusterIP: None
-    ipFamilyPolicy: "RequireDualStack"
-    ipFamilies:
-      - IPv4
-      - IPv6
-
-  admissionWebhooks:
-    enabled: false
-  metrics:
-    enabled: false
-
-revisionHistoryLimit: 2
-
-defaultBackend:
-  enabled: false
--- a/aws/deployments/kubelet-serving-cert-approver.yaml
+++ b/aws/deployments/kubelet-serving-cert-approver.yaml
@@ -1,250 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: certificates:kubelet-serving-cert-approver
-rules:
- apiGroups:
-  - certificates.k8s.io
-  resources:
-  - certificatesigningrequests
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - certificates.k8s.io
-  resources:
-  - certificatesigningrequests/approval
-  verbs:
-  - update
- apiGroups:
-  - authorization.k8s.io
-  resources:
-  - subjectaccessreviews
-  verbs:
-  - create
- apiGroups:
-  - certificates.k8s.io
-  resourceNames:
-  - kubernetes.io/kubelet-serving
-  resources:
-  - signers
-  verbs:
-  - approve
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: events:kubelet-serving-cert-approver
-rules:
- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: psp:kubelet-serving-cert-approver
-rules:
- apiGroups:
-  - policy
-  resourceNames:
-  - kubelet-serving-cert-approver
-  resources:
-  - podsecuritypolicies
-  verbs:
-  - use
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: events:kubelet-serving-cert-approver
-  namespace: default
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: events:kubelet-serving-cert-approver
-subjects:
- kind: ServiceAccount
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: psp:kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: psp:kubelet-serving-cert-approver
-subjects:
- kind: ServiceAccount
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: certificates:kubelet-serving-cert-approver
-subjects:
- kind: ServiceAccount
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
-spec:
-  ports:
-  - name: metrics
-    port: 9090
-    protocol: TCP
-    targetPort: metrics
-  selector:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: kubelet-serving-cert-approver
-      app.kubernetes.io/name: kubelet-serving-cert-approver
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: kubelet-serving-cert-approver
-        app.kubernetes.io/name: kubelet-serving-cert-approver
-    spec:
-      tolerations:
-        - key: "node.cloudprovider.kubernetes.io/uninitialized"
-          value: "true"
-          effect: NoSchedule
-        - key: "CriticalAddonsOnly"
-          operator: Exists
-        - key: "node-role.kubernetes.io/master"
-          effect: NoSchedule
-      affinity:
-        nodeAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - preference:
-              matchExpressions:
-              - key: node-role.kubernetes.io/master
-                operator: DoesNotExist
-              - key: node-role.kubernetes.io/control-plane
-                operator: DoesNotExist
-            weight: 100
-      containers:
-      - args:
-        - serve
-        env:
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        image: ghcr.io/alex1989hu/kubelet-serving-cert-approver:main
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: health
-          initialDelaySeconds: 6
-        name: cert-approver
-        ports:
-        - containerPort: 8080
-          name: health
-        - containerPort: 9090
-          name: metrics
-        readinessProbe:
-          httpGet:
-            path: /readyz
-            port: health
-          initialDelaySeconds: 3
-        resources:
-          limits:
-            cpu: 250m
-            memory: 32Mi
-          requests:
-            cpu: 10m
-            memory: 16Mi
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          readOnlyRootFilesystem: true
-          runAsNonRoot: true
-      priorityClassName: system-cluster-critical
-      securityContext:
-        fsGroup: 65534
-        runAsGroup: 65534
-        runAsUser: 65534
-      serviceAccountName: kubelet-serving-cert-approver
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-        operator: Exists
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/control-plane
-        operator: Exists
-      - effect: NoSchedule
-        key: node.cloudprovider.kubernetes.io/uninitialized
-        operator: Exists
--- a/aws/deployments/metrics-server.yaml
+++ b/aws/deployments/metrics-server.yaml
@@ -1,197 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: metrics-server
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    k8s-app: metrics-server
-    rbac.authorization.k8s.io/aggregate-to-admin: "true"
-    rbac.authorization.k8s.io/aggregate-to-edit: "true"
-    rbac.authorization.k8s.io/aggregate-to-view: "true"
-  name: system:aggregated-metrics-reader
-rules:
- apiGroups:
-  - metrics.k8s.io
-  resources:
-  - pods
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: system:metrics-server
-rules:
- apiGroups:
-  - ""
-  resources:
-  - pods
-  - nodes
-  - nodes/stats
-  - namespaces
-  - configmaps
-  verbs:
-  - get
-  - list
-  - watch
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: metrics-server-auth-reader
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: extension-apiserver-authentication-reader
-subjects:
- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: metrics-server:system:auth-delegator
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:auth-delegator
-subjects:
- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: system:metrics-server
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:metrics-server
-subjects:
- kind: ServiceAccount
-  name: metrics-server
-  namespace: kube-system
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: metrics-server
-  namespace: kube-system
-spec:
-  ports:
-  - name: https
-    port: 443
-    protocol: TCP
-    targetPort: https
-  selector:
-    k8s-app: metrics-server
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: metrics-server
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: metrics-server
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 0
-  template:
-    metadata:
-      labels:
-        k8s-app: metrics-server
-    spec:
-      nodeSelector:
-        kubernetes.io/os: linux
-        node-role.kubernetes.io/control-plane: ""
-      tolerations:
-        - key: "node-role.kubernetes.io/control-plane"
-          effect: NoSchedule
-      containers:
-      - args:
-        - --cert-dir=/tmp
-        - --secure-port=6443
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-        - --kubelet-use-node-status-port
-        - --metric-resolution=15s
-        - --authorization-always-allow-paths=/metrics
-        image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: /livez
-            port: https
-            scheme: HTTPS
-          periodSeconds: 10
-        name: metrics-server
-        ports:
-        - containerPort: 6443
-          name: https
-          protocol: TCP
-        readinessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: /readyz
-            port: https
-            scheme: HTTPS
-          initialDelaySeconds: 20
-          periodSeconds: 10
-        resources:
-          requests:
-            cpu: 100m
-            memory: 200Mi
-        securityContext:
-          readOnlyRootFilesystem: true
-          runAsNonRoot: true
-          runAsUser: 1000
-        volumeMounts:
-        - mountPath: /tmp
-          name: tmp-dir
-      priorityClassName: system-cluster-critical
-      serviceAccountName: metrics-server
-      volumes:
-      - emptyDir: {}
-        name: tmp-dir
---
-apiVersion: apiregistration.k8s.io/v1
-kind: APIService
-metadata:
-  labels:
-    k8s-app: metrics-server
-  name: v1beta1.metrics.k8s.io
-spec:
-  group: metrics.k8s.io
-  groupPriorityMinimum: 100
-  insecureSkipTLSVerify: true
-  service:
-    name: metrics-server
-    namespace: kube-system
-  version: v1beta1
-  versionPriority: 100
--- a/aws/deployments/test-as.yaml
+++ b/aws/deployments/test-as.yaml
@@ -1,48 +0,0 @@
-apiVersion: scheduling.k8s.io/v1
-kind: PriorityClass
-metadata:
-  name: overprovisioning
-value: -1
-globalDefault: false
-description: "Priority class used by overprovisioning."
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: overprovisioning
-  namespace: default
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      run: overprovisioning
-  template:
-    metadata:
-      labels:
-        run: overprovisioning
-    spec:
-      nodeSelector:
-        project.io/node-pool: web
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node.kubernetes.io/instance-type
-                    operator: Exists
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - topologyKey: kubernetes.io/hostname
-              labelSelector:
-                matchExpressions:
-                  - key: run
-                    operator: In
-                    values:
-                      - overprovisioning
-      priorityClassName: overprovisioning
-      containers:
-      - name: reserve-resources
-        image: k8s.gcr.io/pause
-        resources:
-          requests:
-            cpu: "700m"
--- a/hetzner/Makefile
+++ b/hetzner/Makefile
@@ -76,8 +76,11 @@ system-static:
 	helm template --namespace=kube-system -f deployments/hcloud-ccm.yaml \
 		hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml

-	# helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \
-	# 	autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml
+	helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \
+		autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml
+
+	helm template --namespace=kube-system -f deployments/hcloud-csi.yaml hcloud-csi \
+		hcloud/hcloud-csi > deployments/hcloud-csi-result.yaml

 system:
 	helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system --version=1.15.7 -f deployments/cilium.yaml \
@@ -95,7 +98,12 @@ system:
 		talos-cloud-controller-manager \
 		oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager

+	base64 -i _cfgs/worker-as.yaml > _cfgs/worker-as.yaml.base64
+	kubectl --kubeconfig=kubeconfig -n kube-system create secret generic hcloud-init --from-file=worker=_cfgs/worker-as.yaml.base64
+
 deploy-csi:
 	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
 	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret
 	rm -f hcloud-csi-secret.secret
+
+	kubectl --kubeconfig=kubeconfig apply -f deployments/hcloud-csi-result.yaml
--- a/hetzner/deployments/cluster-autoscaler-hcloud-result.yaml
+++ b/hetzner/deployments/cluster-autoscaler-hcloud-result.yaml
@@ -7,7 +7,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
  namespace: kube-system
 spec:
@@ -26,7 +26,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
  namespace: kube-system
 automountServiceAccountToken: true
@@ -39,7 +39,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
 rules:
  - apiGroups:
@@ -78,6 +78,8 @@ rules:
    verbs:
    - watch
    - list
+    - create
+    - delete
    - get
    - update
  - apiGroups:
@@ -156,6 +158,7 @@ rules:
    verbs:
      - list
      - watch
+      - get
  - apiGroups:
    - coordination.k8s.io
    resources:
@@ -180,7 +183,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
 roleRef:
  apiGroup: rbac.authorization.k8s.io
@@ -199,7 +202,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
  namespace: kube-system
 rules:
@@ -228,7 +231,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
  namespace: kube-system
 roleRef:
@@ -248,7 +251,7 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
  namespace: kube-system
 spec:
@@ -272,11 +275,12 @@ metadata:
    app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
    app.kubernetes.io/name: "hetzner-cluster-autoscaler"
    app.kubernetes.io/managed-by: "Helm"
-    helm.sh/chart: "cluster-autoscaler-9.29.3"
+    helm.sh/chart: "cluster-autoscaler-9.37.0"
  name: cluster-autoscaler-hcloud
  namespace: kube-system
 spec:
  replicas: 1
+  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/instance: "cluster-autoscaler-hcloud"
@@ -291,15 +295,13 @@ spec:
      dnsPolicy: "ClusterFirst"
      containers:
        - name: hetzner-cluster-autoscaler
-          image: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.27.3"
+          image: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.0"
          imagePullPolicy: "IfNotPresent"
          command:
            - ./cluster-autoscaler
            - --cloud-provider=hetzner
            - --namespace=kube-system
-            - --nodes=0:2:CPX31:NBG1:worker-nbg1
            - --nodes=0:2:CPX31:FSN1:worker-fsn1
-            - --nodes=0:2:CPX31:HEL1:worker-hel1
            - --logtostderr=true
            - --node-deletion-delay-timeout=10m0s
            - --regional=true
@@ -307,10 +309,18 @@ spec:
            - --stderrthreshold=info
            - --v=4
          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SERVICE_ACCOUNT
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
            - name: HCLOUD_CLOUD_INIT
              valueFrom:
                secretKeyRef:
-                  name: hcloud
+                  name: hcloud-init
                  key: worker
            - name: HCLOUD_IMAGE
              valueFrom:
--- a/hetzner/deployments/hcloud-autoscaler.yaml
+++ b/hetzner/deployments/hcloud-autoscaler.yaml
@@ -1,20 +1,26 @@

 fullnameOverride: cluster-autoscaler-hcloud
-image:
-  tag: v1.27.3
+# image:
+#   tag: v1.27.3

 cloudProvider: hetzner

 autoscalingGroups:
-  - name: CPX31:NBG1:worker-nbg1
-    maxSize: 2
-    minSize: 0
-  - name: CPX31:FSN1:worker-fsn1
-    maxSize: 2
-    minSize: 0
-  - name: CPX31:HEL1:worker-hel1
+  # - name: worker-nbg1
+  #   maxSize: 2
+  #   minSize: 0
+  #   instanceType: CPX31
+  #   region: FSN1
+  - name: worker-fsn1
    maxSize: 2
    minSize: 0
+    instanceType: CPX31
+    region: FSN1
+  # - name: worker-hel1
+  #   maxSize: 2
+  #   minSize: 0
+  #   instanceType: CPX31
+  #   region: FSN1

 extraEnvSecrets:
  HCLOUD_TOKEN:
@@ -31,7 +37,7 @@ extraEnvSecrets:
    key: image
  HCLOUD_CLOUD_INIT:
    name: hcloud
-    key: worker
+    key: init

 containerSecurityContext:
  allowPrivilegeEscalation: false
--- a/hetzner/deployments/hcloud-csi-result.yaml
+++ b/hetzner/deployments/hcloud-csi-result.yaml
@@ -0,0 +1,380 @@
+---
+# Source: hcloud-csi/templates/controller/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hcloud-csi-controller
+  namespace: "kube-system"
+  labels:
+    app.kubernetes.io/name: hcloud-csi
+    helm.sh/chart: hcloud-csi-2.9.0
+    app.kubernetes.io/instance: hcloud-csi
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+automountServiceAccountToken: true
+---
+# Source: hcloud-csi/templates/core/storageclass.yaml
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: hcloud-volumes
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: csi.hetzner.cloud
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+reclaimPolicy: "Delete"
+---
+# Source: hcloud-csi/templates/controller/clusterrole.yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: hcloud-csi-controller
+  labels:
+    app.kubernetes.io/name: hcloud-csi
+    helm.sh/chart: hcloud-csi-2.9.0
+    app.kubernetes.io/instance: hcloud-csi
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+rules:
+# attacher
+- apiGroups: [""]
+  resources: [persistentvolumes]
+  verbs: [get, list, watch, update, patch]
+- apiGroups: [""]
+  resources: [nodes]
+  verbs: [get, list, watch]
+- apiGroups: [csi.storage.k8s.io]
+  resources: [csinodeinfos]
+  verbs: [get, list, watch]
+- apiGroups: [storage.k8s.io]
+  resources: [csinodes]
+  verbs: [get, list, watch]
+- apiGroups: [storage.k8s.io]
+  resources: [volumeattachments]
+  verbs: [get, list, watch, update, patch]
+- apiGroups: [storage.k8s.io]
+  resources: [volumeattachments/status]
+  verbs: [patch]
+# provisioner
+- apiGroups: [""]
+  resources: [secrets]
+  verbs: [get, list]
+- apiGroups: [""]
+  resources: [persistentvolumes]
+  verbs: [get, list, watch, create, delete, patch]
+- apiGroups: [""]
+  resources: [persistentvolumeclaims, persistentvolumeclaims/status]
+  verbs: [get, list, watch, update, patch]
+- apiGroups: [storage.k8s.io]
+  resources: [storageclasses]
+  verbs: [get, list, watch]
+- apiGroups: [""]
+  resources: [events]
+  verbs: [list, watch, create, update, patch]
+- apiGroups: [snapshot.storage.k8s.io]
+  resources: [volumesnapshots]
+  verbs: [get, list]
+- apiGroups: [snapshot.storage.k8s.io]
+  resources: [volumesnapshotcontents]
+  verbs: [get, list]
+# resizer
+- apiGroups: [""]
+  resources: [pods]
+  verbs: [get, list, watch]
+# node
+- apiGroups: [""]
+  resources: [events]
+  verbs: [get, list, watch, create, update, patch]
+---
+# Source: hcloud-csi/templates/controller/clusterrolebinding.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: hcloud-csi-controller
+  labels:
+    app.kubernetes.io/name: hcloud-csi
+    helm.sh/chart: hcloud-csi-2.9.0
+    app.kubernetes.io/instance: hcloud-csi
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: hcloud-csi-controller
+subjects:
+  - kind: ServiceAccount
+    name: hcloud-csi-controller
+    namespace: "kube-system"
+---
+# Source: hcloud-csi/templates/node/daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: hcloud-csi-node
+  namespace: "kube-system"
+  labels:
+    app.kubernetes.io/name: hcloud-csi
+    helm.sh/chart: hcloud-csi-2.9.0
+    app.kubernetes.io/instance: hcloud-csi
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: node
+    app: hcloud-csi 
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: hcloud-csi
+      app.kubernetes.io/instance: hcloud-csi
+      app.kubernetes.io/component: node
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: hcloud-csi
+        helm.sh/chart: hcloud-csi-2.9.0
+        app.kubernetes.io/instance: hcloud-csi
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/component: node
+    spec:
+      
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: instance.hetzner.cloud/is-root-server
+                operator: NotIn
+                values:
+                - "true"
+      nodeSelector:
+        node.cloudprovider.kubernetes.io/platform: hcloud
+      tolerations:
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+        - key: CriticalAddonsOnly
+          operator: Exists
+      priorityClassName: "system-node-critical"
+      securityContext:
+        fsGroup: 1001
+      initContainers:
+      containers:
+        - name: csi-node-driver-registrar
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /run/csi
+            - name: registration-dir
+              mountPath: /registration
+          resources:
+            limits: {}
+            requests: {}
+        - name: liveness-probe
+          image: registry.k8s.io/sig-storage/livenessprobe:v2.13.1
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+          - mountPath: /run/csi
+            name: plugin-dir
+          resources:
+            limits: {}
+            requests: {}
+        - name: hcloud-csi-driver
+          image: docker.io/hetznercloud/hcloud-csi-driver:v2.9.0 # x-release-please-version
+          imagePullPolicy: IfNotPresent
+          command: [/bin/hcloud-csi-driver-node]
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /run/csi
+            - name: device-dir
+              mountPath: /dev
+          securityContext:
+            privileged: true
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///run/csi/socket
+            - name: ENABLE_METRICS
+              value: "false"
+          ports:
+            - name: healthz
+              protocol: TCP
+              containerPort: 9808
+          resources:
+            limits: {}
+            requests: {}
+          livenessProbe:
+            failureThreshold: 5
+            initialDelaySeconds: 10
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 3
+            httpGet:
+              path: /healthz
+              port: healthz
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.hetzner.cloud/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory
+---
+# Source: hcloud-csi/templates/controller/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hcloud-csi-controller
+  namespace: "kube-system"
+  labels:
+    app.kubernetes.io/name: hcloud-csi
+    helm.sh/chart: hcloud-csi-2.9.0
+    app.kubernetes.io/instance: hcloud-csi
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+    app: hcloud-csi-controller 
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: hcloud-csi
+      app.kubernetes.io/instance: hcloud-csi
+      app.kubernetes.io/component: controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: hcloud-csi
+        helm.sh/chart: hcloud-csi-2.9.0
+        app.kubernetes.io/instance: hcloud-csi
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/component: controller
+    spec:
+      serviceAccountName: hcloud-csi-controller
+      
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+      priorityClassName: "system-cluster-critical"
+      securityContext:
+        fsGroup: 1001
+      initContainers:
+      containers:
+        - name: csi-attacher
+          image: registry.k8s.io/sig-storage/csi-attacher:v4.6.1
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits: {}
+            requests: {}
+          args:
+            - --default-fstype=ext4
+          volumeMounts:
+          - name: socket-dir
+            mountPath: /run/csi
+
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer:v1.11.2
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits: {}
+            requests: {}
+          volumeMounts:
+          - name: socket-dir
+            mountPath: /run/csi
+
+        - name: csi-provisioner
+          image: registry.k8s.io/sig-storage/csi-provisioner:v5.0.2
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits: {}
+            requests: {}
+          args:
+            - --feature-gates=Topology=true
+            - --default-fstype=ext4
+          volumeMounts:
+          - name: socket-dir
+            mountPath: /run/csi
+
+        - name: liveness-probe
+          image: registry.k8s.io/sig-storage/livenessprobe:v2.13.1
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits: {}
+            requests: {}
+          volumeMounts:
+          - mountPath: /run/csi
+            name: socket-dir
+
+        - name: hcloud-csi-driver
+          image: docker.io/hetznercloud/hcloud-csi-driver:v2.9.0 # x-release-please-version
+          imagePullPolicy: IfNotPresent
+          command: [/bin/hcloud-csi-driver-controller]
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///run/csi/socket
+            - name: ENABLE_METRICS
+              value: "false"
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: HCLOUD_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: hcloud
+                  key: token
+          resources:
+            limits: {}
+            requests: {}
+          ports:
+            - name: healthz
+              protocol: TCP
+              containerPort: 9808
+          livenessProbe:
+            failureThreshold: 5
+            initialDelaySeconds: 10
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 3
+            httpGet:
+              path: /healthz
+              port: healthz
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /run/csi
+
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+---
+# Source: hcloud-csi/templates/core/csidriver.yaml
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: csi.hetzner.cloud
+spec:
+  attachRequired: true
+  fsGroupPolicy: File
+  podInfoOnMount: true
+  volumeLifecycleModes:
+  - Persistent
--- a/hetzner/deployments/hcloud-csi.yaml
+++ b/hetzner/deployments/hcloud-csi.yaml
@@ -1,411 +1,24 @@
---
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: hcloud-volumes
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "false"
-provisioner: csi.hetzner.cloud
-reclaimPolicy: Delete
-volumeBindingMode: WaitForFirstConsumer
-allowVolumeExpansion: true
---
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: hcloud-volumes-enc
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "false"
-provisioner: csi.hetzner.cloud
-reclaimPolicy: Delete
-volumeBindingMode: WaitForFirstConsumer
-allowVolumeExpansion: true
-parameters:
-  csi.storage.k8s.io/node-publish-secret-name: hcloud-csi-secret
-  csi.storage.k8s.io/node-publish-secret-namespace: kube-system
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: hcloud-csi-controller
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: hcloud-csi-controller
-rules:
- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - csi.storage.k8s.io
-  resources:
-  - csinodeinfos
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - csinodes
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments/status
-  verbs:
-  - patch
- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-  - patch
- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  - persistentvolumeclaims/status
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
- apiGroups:
-  - storage.k8s.io
-  resources:
-  - storageclasses
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - list
-  - watch
-  - create
-  - update
-  - patch
- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshots
-  verbs:
-  - get
-  - list
- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotcontents
-  verbs:
-  - get
-  - list
- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: hcloud-csi-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: hcloud-csi-controller
-subjects:
- kind: ServiceAccount
-  name: hcloud-csi-controller
-  namespace: kube-system
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: hcloud-csi-controller
-  name: hcloud-csi-controller-metrics
-  namespace: kube-system
-spec:
-  ports:
-  - name: metrics
-    port: 9189
-    targetPort: metrics
-  selector:
-    app: hcloud-csi-controller
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: hcloud-csi
-  name: hcloud-csi-node-metrics
-  namespace: kube-system
-spec:
-  ports:
-  - name: metrics
-    port: 9189
-    targetPort: metrics
-  selector:
-    app: hcloud-csi
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hcloud-csi-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: hcloud-csi-controller
-  template:
-    metadata:
-      labels:
-        app: hcloud-csi-controller
-    spec:
-      nodeSelector:
-        node-role.kubernetes.io/control-plane: ""
-        node.cloudprovider.kubernetes.io/platform: hcloud
-      tolerations:
-        - key: "node-role.kubernetes.io/control-plane"
-          effect: NoSchedule
-      containers:
-      - args:
-        - --default-fstype=ext4
-        image: registry.k8s.io/sig-storage/csi-attacher:v4.1.0
-        name: csi-attacher
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0
-        name: csi-resizer
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - args:
-        - --feature-gates=Topology=true
-        - --default-fstype=ext4
-        image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0
-        name: csi-provisioner
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - command:
-        - /bin/hcloud-csi-driver-controller
-        env:
-        - name: CSI_ENDPOINT
-          value: unix:///run/csi/socket
-        - name: METRICS_ENDPOINT
-          value: 0.0.0.0:9189
-        - name: ENABLE_METRICS
-          value: "true"
-        - name: KUBE_NODE_NAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: HCLOUD_TOKEN
-          valueFrom:
-            secretKeyRef:
-              key: token
-              name: hcloud
-        image: hetznercloud/hcloud-csi-driver:v2.3.2
-        imagePullPolicy: Always
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 2
-          timeoutSeconds: 3
-        name: hcloud-csi-driver
-        ports:
-        - containerPort: 9189
-          name: metrics
-        - containerPort: 9808
-          name: healthz
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      - image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0
-        imagePullPolicy: Always
-        name: liveness-probe
-        volumeMounts:
-        - mountPath: /run/csi
-          name: socket-dir
-      serviceAccountName: hcloud-csi-controller
-      volumes:
-      - emptyDir: {}
-        name: socket-dir
---
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  labels:
-    app: hcloud-csi
-  name: hcloud-csi-node
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      app: hcloud-csi
-  template:
-    metadata:
-      labels:
-        app: hcloud-csi
-    spec:
-      nodeSelector:
-        node.cloudprovider.kubernetes.io/platform: hcloud
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-      containers:
-      - args:
-        - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket
-        image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0
-        name: csi-node-driver-registrar
-        volumeMounts:
-        - mountPath: /run/csi
-          name: plugin-dir
-        - mountPath: /registration
-          name: registration-dir
-      - command:
-        - /bin/hcloud-csi-driver-node
-        env:
-        - name: CSI_ENDPOINT
-          value: unix:///run/csi/socket
-        - name: METRICS_ENDPOINT
-          value: 0.0.0.0:9189
-        - name: ENABLE_METRICS
-          value: "true"
-        image: hetznercloud/hcloud-csi-driver:v2.3.2
-        imagePullPolicy: Always
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 2
-          timeoutSeconds: 3
-        name: hcloud-csi-driver
-        ports:
-        - containerPort: 9189
-          name: metrics
-        - containerPort: 9808
-          name: healthz
-          protocol: TCP
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /var/lib/kubelet
-          mountPropagation: Bidirectional
-          name: kubelet-dir
-        - mountPath: /run/csi
-          name: plugin-dir
-        - mountPath: /dev
-          name: device-dir
-      - image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0
-        imagePullPolicy: Always
-        name: liveness-probe
-        volumeMounts:
-        - mountPath: /run/csi
-          name: plugin-dir
-      tolerations:
-      - effect: NoExecute
-        operator: Exists
-      - effect: NoSchedule
-        operator: Exists
-      - key: CriticalAddonsOnly
-        operator: Exists
-      volumes:
-      - hostPath:
-          path: /var/lib/kubelet
-          type: Directory
-        name: kubelet-dir
-      - hostPath:
-          path: /var/lib/kubelet/plugins/csi.hetzner.cloud/
-          type: DirectoryOrCreate
-        name: plugin-dir
-      - hostPath:
-          path: /var/lib/kubelet/plugins_registry/
-          type: Directory
-        name: registration-dir
-      - hostPath:
-          path: /dev
-          type: Directory
-        name: device-dir
---
-apiVersion: storage.k8s.io/v1
-kind: CSIDriver
-metadata:
-  name: csi.hetzner.cloud
-spec:
-  attachRequired: true
-  fsGroupPolicy: File
-  podInfoOnMount: true
-  volumeLifecycleModes:
-  - Persistent
+
+controller:
+  hcloudToken:
+    existingSecret:
+      name: hcloud
+      key: token
+
+  priorityClassName: system-cluster-critical
+
+  nodeSelector:
+    node-role.kubernetes.io/control-plane: ""
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      effect: NoSchedule
+
+node:
+  priorityClassName: system-node-critical
+  nodeSelector:
+    node.cloudprovider.kubernetes.io/platform: hcloud
+
+storageClasses:
+  - name: hcloud-volumes
+    defaultStorageClass: false
+    reclaimPolicy: Delete
--- a/hetzner/deployments/test-as.yaml
+++ b/hetzner/deployments/test-as.yaml
@@ -24,18 +24,18 @@ spec:
      # nodeSelector:
      #   node.cloudprovider.kubernetes.io/platform: hcloud
      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  # - key: node.kubernetes.io/instance-type
-                  #   operator: Exists
-                  # - key: instance.hetzner.cloud/is-root-server
-                  #   operator: NotIn
-                  #   values:
-                  #     - "true"
-                  - key: hcloud/node-group
-                    operator: Exists
+        # nodeAffinity:
+        #   requiredDuringSchedulingIgnoredDuringExecution:
+        #     nodeSelectorTerms:
+        #       - matchExpressions:
+        #           # - key: node.kubernetes.io/instance-type
+        #           #   operator: Exists
+        #           # - key: instance.hetzner.cloud/is-root-server
+        #           #   operator: NotIn
+        #           #   values:
+        #           #     - "true"
+        #           - key: hcloud/node-group
+        #             operator: Exists
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - topologyKey: kubernetes.io/hostname
--- a/hetzner/deployments/test-csi.yaml
+++ b/hetzner/deployments/test-csi.yaml
@@ -15,8 +15,8 @@ apiVersion: v1
 metadata:
  name: csi-app
 spec:
-  nodeSelector:
-    node-role.kubernetes.io/control-plane: ""
+  # nodeSelector:
+  #   node-role.kubernetes.io/control-plane: ""
  tolerations:
    - key: "node-role.kubernetes.io/control-plane"
      effect: NoSchedule
--- a/hetzner/instances-controlplane.tf
+++ b/hetzner/instances-controlplane.tf
@@ -74,6 +74,13 @@ resource "local_sensitive_file" "controlplane" {
      hcloud_token   = var.hcloud_token
      hcloud_image   = data.hcloud_image.talos["amd64"].id
      hcloud_sshkey  = hcloud_ssh_key.infra.id
+      hcloud_init = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
+        merge(local.kubernetes, try(var.instances["all"], {}), {
+          lbv4        = local.ipv4_vip
+          nodeSubnets = var.vpc_main_cidr
+          labels      = "${local.worker_labels},hcloud/node-group=worker-as"
+        })
+      ))
      robot_user     = var.robot_user
      robot_password = var.robot_password
    })
--- a/hetzner/network-lb.tf
+++ b/hetzner/network-lb.tf
@@ -18,11 +18,17 @@ resource "hcloud_floating_ip" "api" {
  labels        = merge(var.tags, { type = "infra" })
 }

-# resource "hcloud_floating_ip_assignment" "api" {
-#   count          = local.lb_enable ? 0 : 1
-#   floating_ip_id = hcloud_floating_ip.api[0].id
-#   server_id      = one(hcloud_server.controlplane).id
-# }
+resource "hcloud_floating_ip_assignment" "api" {
+  count          = local.lb_enable && length(local.controlplanes) > 0 ? 0 : 1
+  floating_ip_id = hcloud_floating_ip.api[0].id
+  server_id      = hcloud_server.controlplane[keys(local.controlplanes)[0]].id
+
+  lifecycle {
+    ignore_changes = [
+      server_id,
+    ]
+  }
+}

 resource "hcloud_load_balancer" "api" {
  count              = local.lb_enable ? 1 : 0
--- a/hetzner/templates/controlplane.yaml.tpl
+++ b/hetzner/templates/controlplane.yaml.tpl
@@ -116,8 +116,9 @@ cluster:
          token: ${base64encode(hcloud_token)}
          user: ${base64encode(robot_user)}
          password: ${base64encode(robot_password)}
-          image: ${base64encode(hcloud_image)}
          sshkey: ${base64encode(hcloud_sshkey)}
+          image: ${base64encode(hcloud_image)}
+          init: ${base64encode(hcloud_init)}
  externalCloudProvider:
    enabled: true
    manifests:
--- a/hetzner/variables.tf
+++ b/hetzner/variables.tf
@@ -66,15 +66,15 @@ variable "controlplane" {
    },
    "nbg1" = {
      count = 0,
-      type  = "cpx11",
+      type  = "cax21",
    },
    "fsn1" = {
-      count = 0,
-      type  = "cpx11",
+      count = 1,
+      type  = "cax21",
    },
    "hel1" = {
      count = 0,
-      type  = "cax11",
+      type  = "cax21",
    }
  }
 }