diff --git a/hetzner/deployments/hcloud-cloud-controller-manager.yaml b/hetzner/deployments/hcloud-cloud-controller-manager.yaml index 8d35cd7..eb0269f 100644 --- a/hetzner/deployments/hcloud-cloud-controller-manager.yaml +++ b/hetzner/deployments/hcloud-cloud-controller-manager.yaml @@ -1,5 +1,4 @@ # NOTE: this release was tested against kubernetes v1.18.x - --- apiVersion: v1 kind: ServiceAccount @@ -55,8 +54,15 @@ spec: effect: NoSchedule - key: "node.kubernetes.io/not-ready" effect: "NoSchedule" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists containers: - - image: hetznercloud/hcloud-cloud-controller-manager:v1.11.1 + - image: hetznercloud/hcloud-cloud-controller-manager:v1.12.0 name: hcloud-cloud-controller-manager command: - "/bin/hcloud-cloud-controller-manager" diff --git a/hetzner/deployments/hcloud-csi.yml b/hetzner/deployments/hcloud-csi.yml new file mode 100644 index 0000000..ec1926f --- /dev/null +++ b/hetzner/deployments/hcloud-csi.yml @@ -0,0 +1,362 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.hetzner.cloud +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: + - Persistent +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + namespace: kube-system + name: hcloud-volumes + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: csi.hetzner.cloud +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hcloud-csi + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hcloud-csi +rules: + # attacher + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + # provisioner + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims", "persistentvolumeclaims/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + # resizer + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + # node + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hcloud-csi +subjects: + - kind: ServiceAccount + name: hcloud-csi + namespace: kube-system +roleRef: + kind: ClusterRole + name: hcloud-csi + apiGroup: rbac.authorization.k8s.io +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: hcloud-csi-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: hcloud-csi-controller + serviceName: hcloud-csi-controller + replicas: 1 + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/instance-type + operator: Exists + serviceAccount: hcloud-csi + containers: + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1 + volumeMounts: + - name: socket-dir + mountPath: /run/csi + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + - name: csi-resizer + image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0 + volumeMounts: + - name: socket-dir + mountPath: /run/csi + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + - name: csi-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2 + args: + - --feature-gates=Topology=true + - --default-fstype=ext4 + volumeMounts: + - name: socket-dir + mountPath: /run/csi + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:1.6.0 + imagePullPolicy: Always + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + volumeMounts: + - name: socket-dir + mountPath: /run/csi + ports: + - containerPort: 9189 + name: metrics + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + - name: liveness-probe + imagePullPolicy: Always + image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + volumeMounts: + - mountPath: /run/csi + name: socket-dir + volumes: + - name: socket-dir + emptyDir: {} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi +spec: + selector: + matchLabels: + app: hcloud-csi + template: + metadata: + labels: + app: hcloud-csi + spec: + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/instance-type + operator: Exists + - key: "instance.hetzner.cloud/is-root-server" + operator: NotIn + values: + - "true" + serviceAccount: hcloud-csi + containers: + - name: csi-node-driver-registrar + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 + args: + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /run/csi + - name: registration-dir + mountPath: /registration + securityContext: + privileged: true + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:1.6.0 + imagePullPolicy: Always + env: + - name: CSI_ENDPOINT + value: unix:///run/csi/socket + - name: METRICS_ENDPOINT + value: 0.0.0.0:9189 + - name: ENABLE_METRICS + value: "true" + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /run/csi + - name: device-dir + mountPath: /dev + securityContext: + privileged: true + ports: + - containerPort: 9189 + name: metrics + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + - name: liveness-probe + imagePullPolicy: Always + image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 + volumeMounts: + - mountPath: /run/csi + name: plugin-dir + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.hetzner.cloud/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory +--- +apiVersion: v1 +kind: Service +metadata: + name: hcloud-csi-controller-metrics + namespace: kube-system + labels: + app: hcloud-csi +spec: + selector: + app: hcloud-csi-controller + ports: + - port: 9189 + name: metrics + targetPort: metrics + +--- +apiVersion: v1 +kind: Service +metadata: + name: hcloud-csi-node-metrics + namespace: kube-system + labels: + app: hcloud-csi +spec: + selector: + app: hcloud-csi + ports: + - port: 9189 + name: metrics + targetPort: metrics