From 7666a5a4199ced4bf8697c08a5a550b576adae35 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 5 Feb 2024 18:42:52 +0100 Subject: [PATCH] csi work --- TODO | 2 + .../templates/helmreleases/cilium.yaml | 1 + packages/core/installer/images/installer.json | 6 +- .../kubevirt-csi-node/templates/deploy.yaml | 278 ++++++++++++++++++ .../kubevirt/templates/kubevirt-cr.yaml | 3 +- 5 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 packages/system/kubevirt-csi-node/templates/deploy.yaml diff --git a/TODO b/TODO index fd9261ef..f16e9167 100644 --- a/TODO +++ b/TODO @@ -25,3 +25,5 @@ nginx-ingress has no values update all applications to be managed by operators fullnameOverride kamaji-etcd specify nip.io domain by default +automatically delete provisioned services with the cluster +replicated storageclass diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml index 04dd2fd1..96c759fd 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml @@ -37,6 +37,7 @@ spec: cni: chainingMode: ~ customConf: false + configMap: "" routingMode: native enableIPv4Masquerade: true ipv4NativeRoutingCIDR: "10.244.0.0/16" diff --git a/packages/core/installer/images/installer.json b/packages/core/installer/images/installer.json index f5e22783..96494dc1 100644 --- a/packages/core/installer/images/installer.json +++ b/packages/core/installer/images/installer.json @@ -1,14 +1,14 @@ { - "containerimage.config.digest": "sha256:e8905096d13b4a13e02ed95d1aeeb828f4a6d0a63041cb83f4f5a0e33a5eb146", + "containerimage.config.digest": "sha256:3fb39acdc610ab4e09bc4d7f2f0e471dce7e4779b824461c8d3a3862ba6400d8", "containerimage.descriptor": { "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "digest": "sha256:afd955e5d9ec6d6c3368d07e44110fdde1eb3819cd3e78eb51c99f81beb7431f", + "digest": "sha256:79a28f49a495400ee395d0cff098f9b789ca6e8c48084727ea5fb4395e2ac510", "size": 2074, "platform": { "architecture": "amd64", "os": "linux" } }, - "containerimage.digest": "sha256:afd955e5d9ec6d6c3368d07e44110fdde1eb3819cd3e78eb51c99f81beb7431f", + "containerimage.digest": "sha256:79a28f49a495400ee395d0cff098f9b789ca6e8c48084727ea5fb4395e2ac510", "image.name": "ghcr.io/aenix-io/cozystack/installer:latest" } \ No newline at end of file diff --git a/packages/system/kubevirt-csi-node/templates/deploy.yaml b/packages/system/kubevirt-csi-node/templates/deploy.yaml new file mode 100644 index 00000000..290d38c1 --- /dev/null +++ b/packages/system/kubevirt-csi-node/templates/deploy.yaml @@ -0,0 +1,278 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.kubevirt.io +spec: + attachRequired: true + podInfoOnMount: true + fsGroupPolicy: ReadWriteOnceWithFSType +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubevirt-csi-controller-sa + namespace: kubevirt-csi-driver +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubevirt-csi-controller-cr +rules: + - apiGroups: [''] + resources: ['persistentvolumes'] + verbs: ['create', 'delete', 'get', 'list', 'watch', 'update', 'patch'] + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get', 'list'] + - apiGroups: [''] + resources: ['persistentvolumeclaims'] + verbs: ['get', 'list', 'watch', 'update'] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [''] + resources: ['nodes'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['storage.k8s.io'] + resources: ['volumeattachments'] + verbs: ['get', 'list', 'watch', 'update', 'patch'] + - apiGroups: ['storage.k8s.io'] + resources: ['storageclasses'] + verbs: ['get', 'list', 'watch'] + - apiGroups: ['csi.storage.k8s.io'] + resources: ['csidrivers'] + verbs: ['get', 'list', 'watch', 'update', 'create'] + - apiGroups: [''] + resources: ['events'] + verbs: ['list', 'watch', 'create', 'update', 'patch'] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubevirt-csi-controller-binding +subjects: + - kind: ServiceAccount + name: kubevirt-csi-controller-sa + namespace: kubevirt-csi-driver +roleRef: + kind: ClusterRole + name: kubevirt-csi-controller-cr + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubevirt-csi-node-sa + namespace: kubevirt-csi-driver +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubevirt-csi-node-cr +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + verbs: ["use"] + resourceNames: ["privileged"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubevirt-csi-node-binding +subjects: + - kind: ServiceAccount + name: kubevirt-csi-node-sa + namespace: kubevirt-csi-driver +roleRef: + kind: ClusterRole + name: kubevirt-csi-node-cr + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kubevirt-csi-node + namespace: kubevirt-csi-driver +spec: + selector: + matchLabels: + app: kubevirt-csi-driver + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: kubevirt-csi-driver + spec: + serviceAccount: kubevirt-csi-node-sa + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: csi-driver + securityContext: + privileged: true + allowPrivilegeEscalation: true + imagePullPolicy: Always + image: ghcr.io/kvaps/test:kubevirt-csi-driver + args: + - "--endpoint=unix:/csi/csi.sock" + - "--node-name=$(KUBE_NODE_NAME)" + - "--v=5" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: udev + mountPath: /run/udev + ports: + - name: healthz + containerPort: 10300 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + resources: + requests: + memory: 50Mi + cpu: 10m + - name: csi-node-driver-registrar + image: quay.io/openshift/origin-csi-node-driver-registrar:latest + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - "--v=5" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.kubevirt.io-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.kubevirt.io/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + requests: + memory: 20Mi + cpu: 5m + - name: csi-liveness-probe + image: quay.io/openshift/origin-csi-livenessprobe:latest + args: + - "--csi-address=/csi/csi.sock" + - "--probe-timeout=3s" + - "--health-port=10300" + volumeMounts: + - name: plugin-dir + mountPath: /csi + resources: + requests: + memory: 20Mi + cpu: 5m + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.kubevirt.io/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: udev + hostPath: + path: /run/udev +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: driver-config + namespace: kubevirt-csi-driver +data: + infraClusterNamespace: + infraClusterLabels: csi-driver/cluster=tenant +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kubevirt + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: csi.kubevirt.io +parameters: + infraStorageClassName: "" + bus: scsi diff --git a/packages/system/kubevirt/templates/kubevirt-cr.yaml b/packages/system/kubevirt/templates/kubevirt-cr.yaml index 4d41e149..f8b70ebd 100644 --- a/packages/system/kubevirt/templates/kubevirt-cr.yaml +++ b/packages/system/kubevirt/templates/kubevirt-cr.yaml @@ -8,7 +8,8 @@ spec: certificateRotateStrategy: {} configuration: developerConfiguration: - featureGates: [] + featureGates: + - HotplugVolumes customizeComponents: {} imagePullPolicy: IfNotPresent workloadUpdateStrategy: {}