feat(lb): Replace MetalLB with Cilium LB IPAM + BGP Control Plane

README.md

@@ -92,14 +92,19 @@ Validate install
 cilium status
 ```
 
-## MetalLB
+## Cilium LB IPAM
 
-For load balancing
+For [Cilium to act as a load balancer](https://docs.cilium.io/en/stable/network/lb-ipam/) and start assigning IPs
-
+to `LoadBalancer` services create a viable IP pool, e.g. `192.168.1.128/5`, by creating a `CiliumLoadBalancerIPPool`
-https://metallb.universe.tf/installation/
 
 ```shell
-kubectl apply -k infra/metallb
+kubectl apply infra/cilium/ip-pool.yaml
+```
+
+and announce using a `CiliumL2AnnouncementPolicy`
+
+```shell
+kubectl apply infra/cilium/announce.yaml
 ```
 
 # Sealed Secrets
@@ -111,6 +116,7 @@ kubectl apply -k infra/sealed-secrets
 ```
 
 Be sure to store the generated sealed secret key in a safa place!
+
 ```shell
 kubectl -n kube-system get secrets
 ```
@@ -166,6 +172,7 @@ kubectl apply -k infra/argocd
 ```
 
 Get ArgoCD initial secret
+
 ```shell
 kubectl -n argocd get secrets argocd-initial-admin-secret -o json | jq -r .data.password | base64 -d
 ```
@@ -181,6 +188,7 @@ kubectl apply -k infra/dashboard
 ```
 
 Create a token
+
 ```shell
 kubectl -n kubernetes-dashboard create token admin-user
 ```
@@ -209,7 +217,7 @@ version `v1alpha2` ([link](https://kubernetes.io/blog/2022/11/18/upcoming-change
 
 Make sure that `runc` is properly configured in containerd.
 
-NB: Make sure the correct `containerd` daemon is running. 
+NB: Make sure the correct `containerd` daemon is running.
 (Check the loaded `containerd` service definition as reported by `systemctl status containerd`)
 Follow https://github.com/containerd/containerd/blob/main/docs/getting-started.md for further instructions.
 

apps/plex/service.yaml

@@ -5,7 +5,6 @@ metadata:
 spec:
   selector:
     app: plex
-  type: LoadBalancer
   ports:
     - name: web
       port: 32400

infra/cilium/announce.yaml

@@ -0,0 +1,9 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: default-l2-announcement-policy
+spec:
+  interfaces:
+    - enp0s25
+  externalIPs: true
+  loadBalancerIPs: true

infra/cilium/ip-pool.yaml

@@ -0,0 +1,7 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default-pool
+spec:
+  cidrs:
+    - cidr: 192.168.1.128/25

infra/cilium/kustomization.yaml

@@ -5,10 +5,12 @@ namespace: kube-system
 resources:
   - traefik-forward-auth
   - ingress-route.yaml
+  - ip-pool.yaml
+  - announce.yaml
 
-patchesStrategicMerge:
+#patchesStrategicMerge:
-  # peer-service: "hubble-peer.default.svc.cluster.local:443"
+#  # peer-service: "hubble-peer.default.svc.cluster.local:443"
-  - patches/hubble-relay-config-peer-service-patch.yaml
+#  - patches/hubble-relay-config-peer-service-patch.yaml
 
 helmCharts:
   - name: cilium

infra/cilium/values.yaml

@@ -13,6 +13,14 @@ k8sServicePort: 6443
 
 kubeProxyReplacement: strict
 
+bgpControlPlane:
+  enabled: false
+
+l2announcements:
+  enabled: true
+externalIPs:
+  enabled: true
+
 # -- Roll out cilium agent pods automatically when configmap is updated.
 rollOutCiliumPods: false
 

infra/cloudflared/deployment.yaml

@@ -17,7 +17,7 @@ spec:
       dnsConfig:
         # Use Pi-Hole DNS
         nameservers:
-          - 192.168.1.131
+          - 192.168.1.154
       containers:
         - name: cloudflared
           image: cloudflare/cloudflared:2023.10.0

infra/metallb/configuration.yaml

@@ -1,17 +0,0 @@
-# Configure IP-pool
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
-  name: first-pool
-  namespace: metallb-system
-spec:
-  addresses:
-    - 192.168.1.128/25
-
----
-# Advertise
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: example
-  namespace: metallb-system

infra/metallb/kustomization.yaml

@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: metallb-system
-
-resources:
-  - github.com/metallb/metallb/config/native?ref=v0.13.9
-  - configuration.yaml

infra/net-aux/config/pi-hole/02-custom.conf

@@ -1,2 +1,2 @@
-address=/stonegarden.dev/192.168.1.128
+address=/stonegarden.dev/192.168.1.142
 edns-packet-max=1232

infra/net-aux/service.yaml

@@ -4,8 +4,8 @@ metadata:
   name: net-aux
   namespace: net-aux
   annotations:
-    metallb.universe.tf/loadBalancerIPs: 192.168.1.153
+    io.cilium/lb-ipam-ips: 192.168.1.153
-    metallb.universe.tf/allow-shared-ip: net-aux
+    io.cilium/lb-ipam-sharing-key: net-aux
 spec:
   selector:
     app: net-aux
@@ -28,9 +28,9 @@ apiVersion: v1
 metadata:
   name: net-aux-udp
   namespace: net-aux
-  annotations:
+#  annotations:
-    metallb.universe.tf/loadBalancerIPs: 192.168.1.153
+#    io.cilium/lb-ipam-ips: 192.168.1.153
-    metallb.universe.tf/allow-shared-ip: net-aux
+#    io.cilium/lb-ipam-sharing-key: net-aux
 spec:
   selector:
     app: net-aux

infra/pi-hole/config/02-custom.conf

@@ -1,2 +1,2 @@
-address=/stonegarden.dev/192.168.1.128
+address=/stonegarden.dev/192.168.1.142
 edns-packet-max=1232

infra/pi-hole/service.yaml

@@ -4,8 +4,8 @@ metadata:
   name: pi-hole
   namespace: pi-hole
   annotations:
-    metallb.universe.tf/loadBalancerIPs: 192.168.1.154
+    io.cilium/lb-ipam-ips: 192.168.1.154
-    metallb.universe.tf/allow-shared-ip: pi-hole
+    io.cilium/lb-ipam-sharing-key: pi-hole
 spec:
   selector:
     app: pi-hole
@@ -24,9 +24,9 @@ apiVersion: v1
 metadata:
   name: pi-hole-udp
   namespace: pi-hole
-  annotations:
+#  annotations:
-    metallb.universe.tf/loadBalancerIPs: 192.168.1.154
+#    io.cilium/lb-ipam-ips: 192.168.1.154
-    metallb.universe.tf/allow-shared-ip: pi-hole
+#    io.cilium/lb-ipam-sharing-key: pi-hole
 spec:
   selector:
     app: pi-hole

infra/traefik/values.yaml

@@ -9,9 +9,9 @@ deployment:
         - name: data
           mountPath: /data
 
-services:
+service:
   annotations:
-    metallb.universe.tf/loadBalancerIPs: 192.168.1.128
+    io.cilium/lb-ipam-ips: 192.168.1.142
 
 providers:
   kubernetesCRD: