From 6b2cda0b2b2c3f1fc1c46fc49777330c1dbb19ff Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Thu, 23 Jun 2022 20:20:56 +0300 Subject: [PATCH] Add scaleway csi --- README.md | 6 +- scaleway/deployments/scaleway-csi-node.yaml | 132 +++++++++ scaleway/deployments/scaleway-csi.yaml | 303 ++++++++++++++++++++ scaleway/deployments/scaleway-storage.yaml | 22 ++ scaleway/deployments/test-pv.yaml | 41 +++ scaleway/instances-master.tf | 1 + scaleway/templates/controlplane.yaml.tpl | 1 + 7 files changed, 503 insertions(+), 3 deletions(-) create mode 100644 scaleway/deployments/scaleway-csi-node.yaml create mode 100644 scaleway/deployments/scaleway-csi.yaml create mode 100644 scaleway/deployments/scaleway-storage.yaml create mode 100644 scaleway/deployments/test-pv.yaml diff --git a/README.md b/README.md index 15d781a..1a1a8c4 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ Well tested with talos 1.1.x. Platform addons: CCM,CSI * [Oracle](oracle) - many regions, many zones. Well tested with talos 1.0.0. -* [Scaleway](scaleway) - many regions. -Well tested with talos 1.0.0. -Platform addons: CCM +* [Scaleway](scaleway) - one region. +Well tested with talos 1.1.0. +Platform addons: CCM,CSI ## Common diff --git a/scaleway/deployments/scaleway-csi-node.yaml b/scaleway/deployments/scaleway-csi-node.yaml new file mode 100644 index 0000000..60b1586 --- /dev/null +++ b/scaleway/deployments/scaleway-csi-node.yaml @@ -0,0 +1,132 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: scaleway-csi-node + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-node-driver-registrar + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-node-driver-registrar +subjects: + - kind: ServiceAccount + name: scaleway-csi-node + namespace: kube-system +roleRef: + kind: ClusterRole + name: scaleway-csi-node-driver-registrar + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: scaleway-csi-node + namespace: kube-system + labels: +spec: + selector: + matchLabels: + app: scaleway-csi-node + template: + metadata: + labels: + app: scaleway-csi-node + role: csi + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccount: scaleway-csi-node + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + containers: + - name: scaleway-csi-plugin + image: scaleway/scaleway-csi:v0.1.7 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=4" + - "--mode=node" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + securityContext: + privileged: true + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: device-dir + mountPath: /dev + - name: csi-node-driver-registrar + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1 + args: + - "--v=2" + - "--csi-address=$(CSI_ADDRESS)" + - "--kubelet-registration-path=$(KUBELET_REGISTRATION_PATH)" + env: + - name: CSI_ADDRESS + value: /csi/csi.sock + - name: KUBELET_REGISTRATION_PATH + value: /var/lib/kubelet/plugins/csi.scaleway.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - name: registration-dir + mountPath: /registration/ + - name: liveness-probe + image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0 + args: + - "--csi-address=$(CSI_ADDRESS)" + env: + - name: CSI_ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.scaleway.com + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev diff --git a/scaleway/deployments/scaleway-csi.yaml b/scaleway/deployments/scaleway-csi.yaml new file mode 100644 index 0000000..7307615 --- /dev/null +++ b/scaleway/deployments/scaleway-csi.yaml @@ -0,0 +1,303 @@ +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: scaleway-csi-controller + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-provisioner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-controller +subjects: + - kind: ServiceAccount + name: scaleway-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: scaleway-csi-provisioner + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-attacher +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-attacher +subjects: + - kind: ServiceAccount + name: scaleway-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: scaleway-csi-attacher + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-snapshotter +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: scaleway-csi-snapshotter +subjects: + - kind: ServiceAccount + name: scaleway-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: scaleway-csi-snapshotter + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-resizer +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +subjects: + - kind: ServiceAccount + name: scaleway-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-resizer + apiGroup: rbac.authorization.k8s.io +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: scaleway-csi-controller + namespace: kube-system +spec: + selector: + matchLabels: + app: scaleway-csi-controller + replicas: 1 + template: + metadata: + labels: + app: scaleway-csi-controller + spec: + serviceAccount: scaleway-csi-controller + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: scaleway-csi-plugin + image: scaleway/scaleway-csi:v0.1.8 + args : + - "--endpoint=$(CSI_ENDPOINT)" + - "--mode=controller" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + envFrom: + - secretRef: + name: scaleway-secret + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + - name: csi-provisioner + image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + - "--feature-gates=Topology=true" + - "--default-fstype=xfs" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.3.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-resizer + image: k8s.gcr.io/sig-storage/csi-resizer:v1.3.0 + args: + - "--v=5" + - "--csi-address=$(CSI_ADDRESS)" + - "--leader-election" + env: + - name: CSI_ADDRESS + value: /var/lib/csi/sockets/pluginproxy/mock.socket + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: liveness-probe + image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/scaleway/deployments/scaleway-storage.yaml b/scaleway/deployments/scaleway-storage.yaml new file mode 100644 index 0000000..dd9e6f0 --- /dev/null +++ b/scaleway/deployments/scaleway-storage.yaml @@ -0,0 +1,22 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.scaleway.com +spec: + attachRequired: true + podInfoOnMount: true + fsGroupPolicy: File +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + name: scw-bssd +provisioner: csi.scaleway.com +# parameters: +# kind: Managed +# fsType: xfs +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true diff --git a/scaleway/deployments/test-pv.yaml b/scaleway/deployments/test-pv.yaml new file mode 100644 index 0000000..8b2add2 --- /dev/null +++ b/scaleway/deployments/test-pv.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: statefulset-scaleway + namespace: default + labels: + app: nginx +spec: + podManagementPolicy: Parallel # default is OrderedReady + serviceName: statefulset-scaleway + replicas: 1 + template: + metadata: + labels: + app: nginx + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + containers: + - name: statefulset-scaleway + image: ubuntu + command: ["sleep","6000"] + volumeMounts: + - name: persistent-storage + mountPath: /mnt/scaleway + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: nginx + volumeClaimTemplates: + - metadata: + name: persistent-storage + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + storageClassName: scw-bssd diff --git a/scaleway/instances-master.tf b/scaleway/instances-master.tf index 7b0b2a4..269600d 100644 --- a/scaleway/instances-master.tf +++ b/scaleway/instances-master.tf @@ -35,6 +35,7 @@ resource "scaleway_instance_server" "controlplane" { secret = var.scaleway_secret project_id = var.scaleway_project_id region = "fr-par" + zone = var.regions[0] vpc_id = split("/", scaleway_vpc_private_network.main.id)[1] }) ) diff --git a/scaleway/templates/controlplane.yaml.tpl b/scaleway/templates/controlplane.yaml.tpl index 88a8ddb..915d02e 100644 --- a/scaleway/templates/controlplane.yaml.tpl +++ b/scaleway/templates/controlplane.yaml.tpl @@ -103,6 +103,7 @@ cluster: SCW_ACCESS_KEY: ${base64encode(access)} SCW_SECRET_KEY: ${base64encode(secret)} SCW_DEFAULT_PROJECT_ID: ${base64encode(project_id)} + SCW_DEFAULT_ZONE: ${base64encode(zone)} SCW_DEFAULT_REGION: ${base64encode(region)} SCW_VPC_ID: ${base64encode(vpc_id)} externalCloudProvider: