feat(mTLS): Enable Cilium mTLS

apps/plex/service.yaml

@@ -5,6 +5,7 @@ metadata:
 spec:
   selector:
     app: plex
+  type: LoadBalancer
   ports:
     - name: web
       port: 32400

infra/cilium/announce.yaml

@@ -2,6 +2,7 @@ apiVersion: cilium.io/v2alpha1
 kind: CiliumL2AnnouncementPolicy
 metadata:
   name: default-l2-announcement-policy
+  namespace: kube-system
 spec:
   interfaces:
     - enp0s25

infra/cilium/ip-pool.yaml

@@ -2,6 +2,7 @@ apiVersion: cilium.io/v2alpha1
 kind: CiliumLoadBalancerIPPool
 metadata:
   name: default-pool
+  namespace: kube-system
 spec:
   cidrs:
     - cidr: 192.168.1.128/25

infra/cilium/kustomization.yaml

@@ -1,20 +1,17 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace: kube-system
 
 resources:
   - traefik-forward-auth
   - ingress-route.yaml
   - ip-pool.yaml
   - announce.yaml
-
-#patchesStrategicMerge:
-#  # peer-service: "hubble-peer.default.svc.cluster.local:443"
-#  - patches/hubble-relay-config-peer-service-patch.yaml
+  - pv-cilium-spire-config.yaml
 
 helmCharts:
   - name: cilium
     repo: https://helm.cilium.io
     version: 1.14.4
     releaseName: "cilium"
+    namespace: kube-system
     valuesFile: values.yaml

infra/cilium/patches/hubble-relay-config-peer-service-patch.yaml

@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hubble-relay-config
-data:
-  # default set to 'default' ns which is wrong
-  config.yaml: |
-    cluster-name: kubernetes
-    peer-service: "hubble-peer.kube-system.svc.cluster.local:443"
-    listen-address: :4245
-    dial-timeout:
-    retry-timeout:
-    sort-buffer-len-max:
-    sort-buffer-drain-timeout:
-    tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
-    tls-client-key-file: /var/lib/hubble-relay/tls/client.key
-    tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt
-    disable-server-tls: true
-

infra/cilium/pv-cilium-spire-config.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: cilium-spire-pv
+spec:
+  capacity:
+    storage: 1Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: cilium-spire-sc
+  local:
+    path: /disk/etc/cilium-spire
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - gauss

infra/cilium/traefik-forward-auth/kustomization.yaml

@@ -1,5 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: kube-system
 commonLabels:
   app: traefik-forward-auth
 

infra/cilium/values.yaml

@@ -1,50 +1,37 @@
 cluster:
+  name: gauss
   id: 0
-  name: kubernetes
 
-debug:
-  enabled: true
+#debug:
+#  enabled: true
 
-encryption:
-  nodeEncryption: false
+k8sServiceHost: "192.168.1.12"
+k8sServicePort: "6443"
 
-k8sServiceHost: 192.168.1.12
-k8sServicePort: 6443
+# Roll out cilium agent pods automatically when ConfigMap is updated.
+rollOutCiliumPods: true
 
-kubeProxyReplacement: strict
+# Increase rate limit when doing L2 announcements
+k8sClientRateLimit:
+  qps: 50
+  burst: 100
 
-bgpControlPlane:
-  enabled: false
+kubeProxyReplacement: true
 
 l2announcements:
   enabled: true
+
 externalIPs:
   enabled: true
 
-# -- Roll out cilium agent pods automatically when configmap is updated.
-rollOutCiliumPods: false
-
-containerRuntime:
-  integration: containerd
-
 enableCiliumEndpointSlice: true
 
 operator:
+  # Can't have more replicas than nodes
   replicas: 1
 
-serviceAccounts:
-  cilium:
-    name: cilium
-  operator:
-    name: cilium-operator
-
-tunnel: vxlan
-
 hubble:
   enabled: true
-  peerService:
-    enabled: true
-    clusterDomain: cluster.local
   relay:
     enabled: true
     rollOutPods: true
@@ -59,4 +46,15 @@ hubble:
     auto:
       enabled: true
       method: helm
-      certValidityDuration: 1095
+      certValidityDuration: 1095
+
+# mTLS
+authentication:
+  enabled: true
+  mutual:
+    spire:
+      enabled: true
+      install:
+        server:
+          dataStorage:
+            storageClass: cilium-spire-sc

infra/project.yaml

@@ -16,6 +16,8 @@ spec:
       server: '*'
     - namespace: 'cloudflared'
       server: '*'
+    - namespace: 'cilium-spire'
+      server: '*'
     - namespace: 'gpu-operator'
       server: '*'
     - namespace: 'kubernetes-dashboard'