feat(biohazard)!: Talos network config for new Fortinet stack, add Multus NetworkAttachments

- Multus: switch HASS & KubeVirt VMs to use new NetworkAttachments - Talos: FortiSwitch can only do tagged VLANs on its direct 1G ports, 10G unmanaged switch (TL-ST1008F) is downstream of FortiSwitch and packets with VLAN tags from TL-ST1008F are mysteriously dropped before it gets to other hosts or the upstream FortiSwitch including ARP (but tagged VLANs worked when TL-ST1008F's upstream was PVE OVS bridge with OPNsense VM and VLANs? lolwut)
2026-01-28 10:18:26 +00:00 · 2024-06-22 02:33:30 +08:00
parent fb3a8f87c2
commit 28875fce99
9 changed files with 208 additions and 139 deletions
--- a/kube/clusters/biohazard/talos/talconfig.yaml
+++ b/kube/clusters/biohazard/talos/talconfig.yaml
@@ -33,59 +33,72 @@ nodes:
    installDiskSelector:
      size: "<= 600GB"
      type: "nvme"
-    nameservers: ["${IP_HOME_DNS}"]
+    nameservers: ["${IP_ROUTER_VLAN_K8S}"]
    disableSearchDomain: true
    networkInterfaces:
      - &m720q-net
-        mtu: 9000 # TODO: switch to 9000 once M720q risers arrive
+        interface: bond0
+        mtu: 9000
        dhcp: false
-        deviceSelector:
-          # onboard I219-V NIC
-          # driver: "e1000e"
-          # hardwareAddr: "*:9a"
-          # TODO: switch to Mellanox once M720q risers arrive
-          driver: "mlx4_core"
-          hardwareAddr: "*:80"
+        bond: &bond0
+          mode: active-backup
+          miimom: 100
+          #primary: enp1s0 # TODO: wen eta Talos bond primary device selector
+          #primary_reselect: better
+          deviceSelectors:
+            # Mellanox ConnectX (enp1s0)
+            - driver: "mlx4_core"
+              hardwareAddr: "*:80"
+        addresses: ["${IP_ROUTER_VLAN_K8S_PREFIX}1/28"]
+        routes:
+          - network: "${IP_ROUTER_VLAN_K8S_CIDR}"
+            metric: 1
+          - network: "0.0.0.0/0"
+            gateway: "${IP_ROUTER_VLAN_K8S}"
+        #vip:
+        #  ip: "${IP_CLUSTER_VIP}"
+      - &m720q-bond1
+        interface: bond1
+        bond:
+          mode: active-backup
+          miimon: 100
+          deviceSelectors:
+            - driver: e1000e
+      - &m720q-br0
+        interface: br0
+        mtu: 1500
+        dhcp: true # native/untagged VLAN will be either undefined or an empty VLAN with no DHCP in prod, this is for emergency/recovery use
+        bridge:
+          interfaces: [bond1]
+          stp: {enabled: true}
        vlans:
-          - &m720q-v58
-            vlanId: 58
-            mtu: 9000
+          - &vlan-active-directory
+            vlanId: 888
+            mtu: 1500
            dhcp: false
-            addresses: ["${IP_ROUTER_VLAN_K8S_PREFIX}1/28"]
-            routes:
-              - network: "${IP_ROUTER_VLAN_K8S_CIDR}"
-                metric: 1
-              - network: "0.0.0.0/0"
-                gateway: "${IP_ROUTER_VLAN_K8S}"
-            #vip:
-            #  ip: "${IP_CLUSTER_VIP}"
-          - &vlan-ceph
-            vlanId: 678
-            mtu: 9000
+          - &vlan-iot
+            vlanId: 227
+            mtu: 1500
            dhcp: false
-            addresses: ["${IP_VLAN_CEPH_PREFIX}1/28"]
-            routes:
-              - network: "${IP_VLAN_CEPH_CIDR}"
-                metric: 1
-    machineFiles:
-      - &ts
-        op: create
-        path: /var/etc/tailscale/auth.env
-        permissions: 0o600
-        content: |
-          TS_KUBE_SECRET=""
-          TS_STATE_DIR=/var/lib/tailscale
-          TS_USERSPACE=false
-          TS_ACCEPT_DNS=false
-      - &tsNode
-        op: append
-        path: /var/etc/tailscale/auth.env
-        permissions: 0o600
-        content: |
-          TS_HOSTNAME=ange-talos
-          TS_EXTRA_ARGS=--accept-routes=false --netfilter-mode=off --snat-subnet-routes=false --advertise-tags=tag:talos,tag:ange
-          TS_AUTHKEY=${SECRET_TAILSCALE_TALOS_AUTHKEY_ANGE}
-          TS_ROUTES=${IP_ROUTER_VLAN_K8S_PREFIX}1/32
+#    machineFiles:
+#      - &ts
+#        op: create
+#        path: /var/etc/tailscale/auth.env
+#        permissions: 0o600
+#        content: |
+#          TS_KUBE_SECRET=""
+#          TS_STATE_DIR=/var/lib/tailscale
+#          TS_USERSPACE=false
+#          TS_ACCEPT_DNS=false
+#      - &tsNode
+#        op: append
+#        path: /var/etc/tailscale/auth.env
+#        permissions: 0o600
+#        content: |
+#          TS_HOSTNAME=ange-talos
+#          TS_EXTRA_ARGS=--accept-routes=false --netfilter-mode=off --snat-subnet-routes=false --advertise-tags=tag:talos,tag:ange
+#          TS_AUTHKEY=${SECRET_TAILSCALE_TALOS_AUTHKEY_ANGE}
+#          TS_ROUTES=${IP_ROUTER_VLAN_K8S_PREFIX}1/32
    schematic:
      customization:
        systemExtensions:
@@ -93,7 +106,7 @@ nodes:
            - siderolabs/i915-ucode
            - siderolabs/intel-ucode
            - siderolabs/iscsi-tools
-            - siderolabs/tailscale
+            #- siderolabs/tailscale
    inlinePatch:
      machine:
        sysfs:
@@ -105,44 +118,46 @@ nodes:
    ipAddress: "${IP_ROUTER_VLAN_K8S_PREFIX}2"
    networkInterfaces:
      - <<: *m720q-net
-        deviceSelector:
-          driver: "mlx4_core"
-          hardwareAddr: "*:6a"
-        vlans:
-          - <<: *m720q-v58
-            addresses: ["${IP_ROUTER_VLAN_K8S_PREFIX}2/28"]
-          - <<: *vlan-ceph
-            addresses: ["${IP_VLAN_CEPH_PREFIX}2/28"]
-    machineFiles:
-      - *ts
-      - <<: *tsNode
-        content: |
-          TS_HOSTNAME=charlotte-talos
-          TS_EXTRA_ARGS=--accept-routes=false --netfilter-mode=off --snat-subnet-routes=false --advertise-tags=tag:talos,tag:charlotte
-          TS_AUTHKEY=${SECRET_TAILSCALE_TALOS_AUTHKEY_CHARLOTTE}
-          TS_ROUTES=${IP_ROUTER_VLAN_K8S_PREFIX}2/32
+        addresses: ["${IP_ROUTER_VLAN_K8S_PREFIX}2/28"]
+        bond:
+          <<: *bond0
+          deviceSelectors:
+            # Mellanox ConnectX (enp1s0)
+            - driver: "mlx4_core"
+              hardwareAddr: "*:6a"
+      - *m720q-bond0
+      - *m720q-br0
+#    machineFiles:
+#      - *ts
+#      - <<: *tsNode
+#        content: |
+#          TS_HOSTNAME=charlotte-talos
+#          TS_EXTRA_ARGS=--accept-routes=false --netfilter-mode=off --snat-subnet-routes=false --advertise-tags=tag:talos,tag:charlotte
+#          TS_AUTHKEY=${SECRET_TAILSCALE_TALOS_AUTHKEY_CHARLOTTE}
+#          TS_ROUTES=${IP_ROUTER_VLAN_K8S_PREFIX}2/32

  - <<: *m720q # TODO: this is cp2 but bare metal, only apply after PVE converted to Talos
    hostname: "chise.${DNS_CLUSTER}" # M720q, i3-8100T 4C4T, 32GB RAM, 512GB OS NVMe
    ipAddress: "${IP_ROUTER_VLAN_K8S_PREFIX}3"
    networkInterfaces:
      - <<: *m720q-net
-        deviceSelector:
-          driver: "mlx4_core"
-          hardwareAddr: "*:40"
-        vlans:
-          - <<: *m720q-v58
-            addresses: ["${IP_ROUTER_VLAN_K8S_PREFIX}3/28"]
-          - <<: *vlan-ceph
-            addresses: ["${IP_VLAN_CEPH_PREFIX}3/28"]
-    machineFiles:
-      - *ts
-      - <<: *tsNode
-        content: |
-          TS_HOSTNAME=chise-talos
-          TS_EXTRA_ARGS=--accept-routes=false --netfilter-mode=off --snat-subnet-routes=false --advertise-tags=tag:talos,tag:chise
-          TS_AUTHKEY=${SECRET_TAILSCALE_TALOS_AUTHKEY_CHISE}
-          TS_ROUTES=${IP_ROUTER_VLAN_K8S_PREFIX}3/32
+        addresses: ["${IP_ROUTER_VLAN_K8S_PREFIX}3/28"]
+        bond:
+          <<: *bond0
+          deviceSelectors:
+            # Mellanox ConnectX (enp1s0)
+            - driver: "mlx4_core"
+              hardwareAddr: "*:40"
+      - *m720q-bond0
+      - *m720q-br0
+#    machineFiles:
+#      - *ts
+#      - <<: *tsNode
+#        content: |
+#          TS_HOSTNAME=chise-talos
+#          TS_EXTRA_ARGS=--accept-routes=false --netfilter-mode=off --snat-subnet-routes=false --advertise-tags=tag:talos,tag:chise
+#          TS_AUTHKEY=${SECRET_TAILSCALE_TALOS_AUTHKEY_CHISE}
+#          TS_ROUTES=${IP_ROUTER_VLAN_K8S_PREFIX}3/32
    inlinePatch:
      machine:
        sysfs:
@@ -161,12 +176,6 @@ patches:
          #- SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660"
          - SUBSYSTEM=="drm", GROUP="44", MODE="0660"

-  - &kubeletExtraArgs |-
-    - op: add
-      path: /machine/kubelet/extraArgs
-      value:
-        feature-gates: GracefulNodeShutdown=true,ServerSideApply=true
-
  - &machinePatch |-
    machine:
      install:
@@ -189,7 +198,7 @@ patches:
            aliases: ["nas.${DNS_MAIN}"]
      time:
        disabled: false
-        servers: ["${IP_ROUTER_LAN}"]
+        servers: ["${IP_ROUTER_VLAN_K8S}"]
        bootTimeout: 2m0s
 #      kernel:
 #        modules:
@@ -246,13 +255,6 @@ patches:

 controlPlane:
  patches:
-
-    - &apiServerExtraArgs |-
-      - op: add
-        path: /cluster/apiServer/extraArgs
-        value:
-          feature-gates: GracefulNodeShutdown=true,ServerSideApply=true
-
    - &apiServerResources |-
      cluster:
        apiServer:
@@ -289,36 +291,25 @@ controlPlane:
          extraArgs:
            listen-metrics-urls: "http://0.0.0.0:2381"

-    # - &scheduler |-
-    #   cluster:
-    #     scheduler:
-    #       extraArgs:
-    #         config: "/custom/etc/kube-scheduler/config.yaml"
-    #       extraVolumes:
-    #         - hostPath: "/var/etc/kube-scheduler"
-    #           mountPath: "/custom/etc/kube-scheduler"
-    #           readonly: true
-    #   machine:
-    #     files:
-    #       - op: create
-    #         path: "/var/etc/kube-scheduler/config.yaml"
-    #         permissions: 0o400
-    #         content: |
-    #           apiVersion: kubescheduler.config.k8s.io/v1
-    #           kind: KubeSchedulerConfiguration
-    #           profiles:
-    #             - schedulerName: default-scheduler
-    #               pluginConfig:
-    #                 - name: PodTopologySpread
-    #                   args:
-    #                     defaultingType: List
-    #                     defaultConstraints:
-    #                       - maxSkew: 1
-    #                         topologyKey: "kubernetes.io/hostname"
-    #                         whenUnsatisfiable: ScheduleAnyway
-    #                       - maxSkew: 5
-    #                         topologyKey: "topology.kubernetes.io/zone"
-    #                         whenUnsatisfiable: ScheduleAnyway
+    - &scheduler |-
+      cluster:
+        scheduler:
+          config:
+            apiVersion: kubescheduler.config.k8s.io/v1
+            kind: KubeSchedulerConfiguration
+            profiles:
+              - schedulerName: default-scheduler
+                pluginConfig:
+                  - name: PodTopologySpread
+                    args:
+                      defaultingType: List
+                      defaultConstraints:
+                        - maxSkew: 1
+                          topologyKey: "kubernetes.io/hostname"
+                          whenUnsatisfiable: DoNotSchedule
+                        - maxSkew: 3
+                          topologyKey: "topology.kubernetes.io/zone"
+                          whenUnsatisfiable: ScheduleAnyway

    # TODO: can't create files outside /var, maybe open FR to whitelist this?
    # - &nfsMountOptions |-
--- a/kube/deploy/apps/home-assistant/app/hr.yaml
+++ b/kube/deploy/apps/home-assistant/app/hr.yaml
@@ -23,6 +23,15 @@ spec:
          labels:
            ingress.home.arpa/nginx-internal: "allow"
            egress.home.arpa/iot: "allow"
+          annotations:
+            k8s.v1.cni.cncf.io/networks: |
+              [{
+                "name":"iot",
+                "namespace": "home-assistant",
+                "ips": ["${APP_IP_HOME_ASSISTANT_IOT}"],
+                "mac": "${APP_MAC_HOME_ASSISTANT_IOT}",
+                "gateway": "${IP_ROUTER_VLAN_IOT}"
+              }]
        containers:
          main:
            image:
--- a/kube/deploy/apps/home-assistant/app/multus.yaml
+++ b/kube/deploy/apps/home-assistant/app/multus.yaml
@@ -0,0 +1,32 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: iot
+  namespace: home-assistant
+spec:
+  config: |-
+    {
+      "cniVersion": "0.3.1",
+      "name": "iot",
+      "plugins": [
+        {
+          "type": "macvlan",
+          "master": "br0.227",
+          "mode": "bridge",
+          "capabilities": {
+            "ips": true
+          },
+          "ipam": {
+            "type": "static",
+            "routes": [
+              { "dst": "${IP_VLAN_IOT_CIDR}", "gw": "${IP_ROUTER_VLAN_IOT}" }
+            ]
+          }
+        },
+        {
+          "capabilities": { "mac": true },
+          "type": "tuning"
+        }
+      ]
+    }
--- a/kube/deploy/apps/home-assistant/ks.yaml
+++ b/kube/deploy/apps/home-assistant/ks.yaml
@@ -13,6 +13,7 @@ spec:
  targetNamespace: "home-assistant"
  dependsOn:
    - name: home-assistant-pvc
+    - name: 1-core-1-networking-multus-app
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
--- a/kube/deploy/core/_networking/cilium/app/config/biohazard/helm-values.yaml
+++ b/kube/deploy/core/_networking/cilium/app/config/biohazard/helm-values.yaml
@@ -34,7 +34,7 @@ cluster:

 ## NOTE: Cilium's routing modes for inter-nodes pod traffic
 routingMode: native
-devices: 'enp1s0' # use specific VLAN # TODO: figure out how to regex to match all interfaces with VLAN 58
+devices: 'enp1s0,br0' # use specific VLAN # TODO: figure out how to regex to match all interfaces with VLAN 58
 autoDirectNodeRoutes: true
 ipv4NativeRoutingCIDR: "${IP_POD_CIDR_V4}"
 loadBalancer:
@@ -58,6 +58,10 @@ k8sServiceHost: "127.0.0.1"
 k8sServicePort: "7445"
 kubeProxyReplacementHealthzBindAddr: "0.0.0.0:10256"

+## Multus compatibility
+cni:
+  exclusive: false
+
 ## NOTE: Cilium can automatically kill and respawn pods upon ConfigMap updates or other resource changes
 rollOutCiliumPods: true
 operator:
--- a/kube/deploy/core/_networking/multus/app/hr.yaml
+++ b/kube/deploy/core/_networking/multus/app/hr.yaml
@@ -16,11 +16,11 @@ spec:
  values:
    image:
      repository: ghcr.io/k8snetworkplumbingwg/multus-cni
-      tag: v4.0.2-thick
+      tag: v4.0.2-thick@sha256:3fbcc32bd4e4d15bd93c96def784a229cd84cca27942bf4858b581f31c97ee02
    cni:
      image:
        repository: ghcr.io/angelnu/cni-plugins
-        tag: 1.4.0@sha256:cd2b132f3725c303a809f253aac07d11e696910a6053e7535e609118151aa9fe
+        tag: 1.5.1@sha256:caac0fccffb23811386919e3f8cc216de7e1a907648f19bc671d9e495fb77069
      paths:
        config: /etc/cni/net.d
        bin: /opt/cni/bin
--- a/kube/deploy/vm/ad/_deps/multus.yaml
+++ b/kube/deploy/vm/ad/_deps/multus.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: vm-ad
+  namespace: vm-ad
+spec:
+  config: |-
+    {
+      "cniVersion": "0.3.1",
+      "name": "vm-ad",
+      "plugins": [
+        {
+          "type": "bridge",
+          "bridge": "br0",
+          "vlan": 888,
+          "capabilities": { "mac": true },
+          "disableContainerInterface": true,
+          "macspoofchk": true
+        }
+      ]
+    }
--- a/kube/deploy/vm/ad/ks.yaml
+++ b/kube/deploy/vm/ad/ks.yaml
@@ -56,6 +56,7 @@ spec:
  path: ./kube/deploy/vm/ad/template-dc
  targetNamespace: "vm-ad"
  dependsOn:
+    - name: 1-core-1-networking-multus-app
    - name: zz-vm-1-kubevirt-app
    - name: zz-vm-ad-1-deps
    - name: zz-vm-ad-dc1-pvc
@@ -63,6 +64,7 @@ spec:
    substitute:
      NUM: "1"
      UUID: "${VM_UUID_AD_DC1}"
+      MAC: "${VM_MAC_AD_DC1}"
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
@@ -110,6 +112,7 @@ spec:
  path: ./kube/deploy/vm/ad/template-dc
  targetNamespace: "vm-ad"
  dependsOn:
+    - name: 1-core-1-networking-multus-app
    - name: zz-vm-1-kubevirt-app
    - name: zz-vm-ad-1-deps
    - name: 1-core-storage-rook-ceph-cluster
@@ -117,3 +120,4 @@ spec:
    substitute:
      NUM: "2"
      UUID: "${VM_UUID_AD_DC2}"
+      MAC: "${VM_MAC_AD_DC2}"
--- a/kube/deploy/vm/ad/template-dc/vm.yaml
+++ b/kube/deploy/vm/ad/template-dc/vm.yaml
@@ -20,9 +20,12 @@ spec:
    spec:
      hostname: "thunder-dc${NUM}"
      networks:
-        - name: "main"
-          pod:
-            vmNetworkCIDR: "${IP_KUBEVIRT_AD_CIDR_V4}"
+        #- name: "main"
+        #  pod:
+        #    vmNetworkCIDR: "${IP_KUBEVIRT_AD_CIDR_V4}"
+        - name: "vlan"
+          multus:
+            networkName: "vm-ad"
      volumes:
        - name: "c-drive"
          persistentVolumeClaim:
@@ -33,15 +36,18 @@ spec:
            - name: "c-drive"
              disk: {}
          interfaces:
-            - name: "main"
-              masquerade: {}
-              ports:
-                - name: "wireguard"
-                  port: 45678
-                  protocol: "UDP"
-                - name: "tailscale"
-                  port: 41641
-                  protocol: "UDP"
+            #- name: "main"
+            #  masquerade: {}
+            #  ports:
+            #    - name: "wireguard"
+            #      port: 45678
+            #      protocol: "UDP"
+            #    - name: "tailscale"
+            #      port: 41641
+            #      protocol: "UDP"
+            - name: "vlan"
+              bridge: {}
+              macAddress: "${MAC}"
          autoattachInputDevice: false
          inputs: []
        firmware:
@@ -55,4 +61,4 @@ spec:
          whenUnsatisfiable: "DoNotSchedule"
          labelSelector:
            matchLabels:
-              vm.home.arpa/windows: "ad"
+              vm.home.arpa/windows: "ad"