diff --git a/.gitignore b/.gitignore index 023f810..605e01c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,6 @@ override.tf.json talosctl talosconfig kubeconfig - +*.secret # /talos/ diff --git a/hetzner/Makefile b/hetzner/Makefile index 6e8fcae..e36bdde 100644 --- a/hetzner/Makefile +++ b/hetzner/Makefile @@ -60,3 +60,7 @@ create-deployments: cilium/cilium > deployments/cilium-result.yaml helm template --namespace=ingress-nginx --version=4.4.0 -f deployments/ingress.yaml ingress-nginx \ ingress-nginx/ingress-nginx > deployments/ingress-result.yaml + +create-secrets: + dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret + kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret diff --git a/hetzner/deployments/hcloud-cloud-controller-manager.yaml b/hetzner/deployments/hcloud-cloud-controller-manager.yaml index 4dbafb1..cd2794e 100644 --- a/hetzner/deployments/hcloud-cloud-controller-manager.yaml +++ b/hetzner/deployments/hcloud-cloud-controller-manager.yaml @@ -38,31 +38,22 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: cloud-controller-manager dnsPolicy: Default + nodeSelector: + node-role.kubernetes.io/control-plane: "" tolerations: - key: "node.cloudprovider.kubernetes.io/uninitialized" value: "true" effect: "NoSchedule" - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule - - key: "node.kubernetes.io/not-ready" - effect: "NoSchedule" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists containers: - # - image: hetznercloud/hcloud-cloud-controller-manager:v1.12.1 - - image: ghcr.io/sergelogvinov/hetzner-cloud-controller-manager:v1.12.2-dev + - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2 + # - image: ghcr.io/sergelogvinov/hetzner-cloud-controller-manager:v1.12.2-dev name: hcloud-cloud-controller-manager args: - --cloud-provider=hcloud - --allow-untagged-cloud - - --controllers=cloud-node,cloud-node-lifecycle + - --controllers=cloud-node-lifecycle resources: requests: cpu: 100m diff --git a/hetzner/deployments/hcloud-csi.yaml b/hetzner/deployments/hcloud-csi.yaml index ee20247..e78e1e3 100644 --- a/hetzner/deployments/hcloud-csi.yaml +++ b/hetzner/deployments/hcloud-csi.yaml @@ -8,6 +8,7 @@ spec: podInfoOnMount: true volumeLifecycleModes: - Persistent + fsGroupPolicy: File --- kind: StorageClass apiVersion: storage.k8s.io/v1 @@ -16,82 +17,178 @@ metadata: annotations: storageclass.kubernetes.io/is-default-class: "false" provisioner: csi.hetzner.cloud +reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer allowVolumeExpansion: true --- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: hcloud-volumes-enc + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: csi.hetzner.cloud +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +parameters: + csi.storage.k8s.io/node-publish-secret-name: hcloud-csi-secret + csi.storage.k8s.io/node-publish-secret-namespace: kube-system +--- apiVersion: v1 kind: ServiceAccount metadata: - name: hcloud-csi + name: hcloud-csi-controller namespace: kube-system --- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: hcloud-csi + name: hcloud-csi-controller rules: - # attacher - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - # provisioner - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims", "persistentvolumeclaims/status"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] - # resizer - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - # node - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch --- -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: hcloud-csi -subjects: - - kind: ServiceAccount - name: hcloud-csi - namespace: kube-system + name: hcloud-csi-controller roleRef: - kind: ClusterRole - name: hcloud-csi apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hcloud-csi-controller +subjects: +- kind: ServiceAccount + name: hcloud-csi-controller + namespace: kube-system --- kind: StatefulSet apiVersion: apps/v1 @@ -109,19 +206,12 @@ spec: labels: app: hcloud-csi-controller spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" tolerations: - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/instance-type - operator: Exists - serviceAccount: hcloud-csi + serviceAccount: hcloud-csi-controller containers: - name: csi-attacher image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 @@ -157,8 +247,10 @@ spec: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:1.6.0 + image: hetznercloud/hcloud-csi-driver:2.0.0 imagePullPolicy: Always + command: + - /bin/hcloud-csi-driver-controller env: - name: CSI_ENDPOINT value: unix:///run/csi/socket @@ -227,8 +319,6 @@ spec: tolerations: - effect: NoSchedule operator: Exists - - key: CriticalAddonsOnly - operator: Exists affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -258,8 +348,10 @@ spec: securityContext: privileged: true - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:1.6.0 + image: hetznercloud/hcloud-csi-driver:2.0.0 imagePullPolicy: Always + command: + - /bin/hcloud-csi-driver-node env: - name: CSI_ENDPOINT value: unix:///run/csi/socket @@ -267,16 +359,6 @@ spec: value: 0.0.0.0:9189 - name: ENABLE_METRICS value: "true" - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - name: hcloud - key: token - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet diff --git a/hetzner/deployments/metrics-server.yaml b/hetzner/deployments/metrics-server.yaml index a06028b..f259001 100644 --- a/hetzner/deployments/metrics-server.yaml +++ b/hetzner/deployments/metrics-server.yaml @@ -136,11 +136,12 @@ spec: containers: - args: - --cert-dir=/tmp - - --secure-port=443 + - --secure-port=6443 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --kubelet-use-node-status-port - --metric-resolution=15s - image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1 + - --authorization-always-allow-paths=/metrics + image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -151,7 +152,7 @@ spec: periodSeconds: 10 name: metrics-server ports: - - containerPort: 443 + - containerPort: 6443 name: https protocol: TCP readinessProbe: diff --git a/hetzner/deployments/test-csi.yaml b/hetzner/deployments/test-csi.yaml index 7bf7d17..ba60543 100644 --- a/hetzner/deployments/test-csi.yaml +++ b/hetzner/deployments/test-csi.yaml @@ -8,25 +8,18 @@ spec: resources: requests: storage: 10Gi - storageClassName: hcloud-volumes + storageClassName: hcloud-volumes-enc --- kind: Pod apiVersion: v1 metadata: name: csi-app spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" tolerations: - - effect: NoExecute - operator: Exists - - effect: NoSchedule - operator: Exists - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/instance-type - operator: Exists + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule containers: - name: my-frontend image: alpine diff --git a/hetzner/instances-web.tf b/hetzner/instances-web.tf index 96702d8..4bf7630 100644 --- a/hetzner/instances-web.tf +++ b/hetzner/instances-web.tf @@ -17,6 +17,6 @@ module "web" { vm_params = merge(var.kubernetes, { lbv4 = local.ipv4_vip - labels = "project.io/node-pool=web,node.kubernetes.io/disktype=ssd,topology.kubernetes.io/region=${each.key},hcloud/node-group=web-${each.key}" + labels = "project.io/node-pool=web,node.kubernetes.io/disktype=ssd,hcloud/node-group=web-${each.key}" }) } diff --git a/hetzner/instances-workers.tf b/hetzner/instances-workers.tf index 1580267..b50c61f 100644 --- a/hetzner/instances-workers.tf +++ b/hetzner/instances-workers.tf @@ -17,6 +17,6 @@ module "worker" { vm_params = merge(var.kubernetes, { lbv4 = local.ipv4_vip - labels = "project.io/node-pool=worker,node.kubernetes.io/disktype=ssd,topology.kubernetes.io/region=${each.key},hcloud/node-group=worker-${each.key}" + labels = "project.io/node-pool=worker,node.kubernetes.io/disktype=ssd,hcloud/node-group=worker-${each.key}" }) }