fix(rook-ceph-cluster/biohazard): add CephFS size=2 pool, MDS priorityClass, tolerate Cilium taint

kube/deploy/core/storage/rook-ceph/cluster/biohazard/hr.yaml

@@ -105,6 +105,8 @@ spec:
           tolerations:
             - key: "node-role.kubernetes.io/control-plane"
               operator: "Exists"
+            - key: "node.cilium.io/agent-not-ready" # only RGW uses CNI
+              operator: "Exists"
             - key: "node-role.kubernetes.io/unschedulable"
               operator: "Exists"
       resources: # defaults are very aggressive
@@ -271,6 +273,15 @@ spec:
                 min_size: "2"
                 compression_mode: "aggressive"
                 compression_algorithm: "zstd"
+            - name: "replicas-2"
+              failureDomain: "host"
+              deviceClass: "ssd"
+              replicated:
+                size: 2
+              parameters:
+                min_size: "2"
+                compression_mode: "aggressive"
+                compression_algorithm: "lz4"
             - name: "ec-2-1-min-2"
               failureDomain: "host"
               deviceClass: "ssd"
@@ -284,6 +295,7 @@ spec:
           metadataServer: &mds
             activeCount: 3 # 3 active 3 standby
             activeStandby: true
+            priorityClassName: system-cluster-critical
             placement:
               topologySpreadConstraints:
                 - maxSkew: 2 # at least 2 distinct nodes should have active MDSes
@@ -298,7 +310,7 @@ spec:
                 memory: 2Gi
               limits:
                 cpu: 2000m # 2 cores
-                memory: 4Gi
+                memory: 2Gi
         storageClass: &cephfsSC
           enabled: true
           isDefault: false

kube/deploy/core/storage/rook-ceph/cluster/biohazard/storageclass-file-ec-2-1.yaml

@@ -1,22 +0,0 @@
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: file-ec-2-1
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "false"
-provisioner: rook-ceph.cephfs.csi.ceph.com
-reclaimPolicy: Delete
-allowVolumeExpansion: true
-volumeBindingMode: Immediate
-parameters:
-  clusterID: rook-ceph
-  fsName: biohazard-fs
-  pool: biohazard-fs-ec-2-1-min-2
-  csi.storage.k8s.io/fstype: ext4
-  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
-  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
-  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
-  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
-  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
-  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph

kube/deploy/core/storage/rook-ceph/cluster/biohazard/storageclass.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: file-ec-2-1
+  annotations:
+    description: "Generally used for non-important mass storage."
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: rook-ceph.cephfs.csi.ceph.com
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+volumeBindingMode: Immediate
+parameters:
+  clusterID: rook-ceph
+  fsName: biohazard-fs
+  pool: biohazard-fs-ec-2-1-min-2
+  csi.storage.k8s.io/fstype: ext4
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: file-size-2
+  annotations:
+    description: "Generally used for important mass storage (e.g. Immich library), reclaimPolicy=Retain to avoid accidentally deleting mass storage data."
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: rook-ceph.cephfs.csi.ceph.com
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+volumeBindingMode: Immediate
+parameters:
+  clusterID: rook-ceph
+  fsName: biohazard-fs
+  pool: biohazard-fs-replicas-2
+  csi.storage.k8s.io/fstype: ext4
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph