fix(netbird-agent): allow read root fs and add sysctl

The agent seems to assume it's running with full root privileges,
something we don't want. Accommodate by changing manually.

k8s/infra/vpn/netbird/agent/daemon-set.yaml

@@ -15,23 +15,30 @@ spec:
       securityContext:
         seccompProfile:
           type: RuntimeDefault
+        sysctls:
+          # Considered unsafe, need to enable with kubelet argument.
+          # see https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls
+          - name: net.ipv4.conf.all.src_valid_mark
+            value: "1"
       containers:
         - name: agent
           image: docker.io/netbirdio/netbird:0.35.2 # renovate: docker=docker.io/netbirdio/netbird
           imagePullPolicy: IfNotPresent
           securityContext:
             allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
+            readOnlyRootFilesystem: false
+            runAsNonRoot: false
             capabilities:
-              add: [ NET_ADMIN, PERFMON, BPF ]
+              drop: [ ALL ]
+              add: [ NET_ADMIN, NET_RAW, PERFMON, BPF ]
           envFrom:
             - configMapRef:
                 name: agent-config
             - secretRef:
                 name: agent-setup-key
           volumeMounts:
-            - name: config
+            - name: netbird
-              mountPath: /config
+              mountPath: /var/lib/netbird
           resources:
             requests:
               memory: 192Mi
@@ -40,5 +47,5 @@ spec:
               memory: 256Mi
               cpu: 4000m
       volumes:
-        - name: config
+        - name: netbird
           emptyDir: { }

k8s/infra/vpn/netbird/agent/kustomization.yaml

@@ -9,7 +9,7 @@ configMapGenerator:
       - NB_LOG_LEVEL="warn"
       - NB_MANAGEMENT_URL="https://netbird.stonegarden.dev"
       - NB_ADMIN_URL="https://netbird.stonegarden.dev"
-      - NB_CONFIG="/config/config.json"
+      - NB_CONFIG="/var/lib/netbird/config.json"
 
 resources:
   - agent-setup-key.yaml

tofu/kubernetes/talos/machine-config/control-plane.yaml.tftpl

@@ -1,4 +1,8 @@
 machine:
+  kubelet:
+    extraArgs:
+      # Needed for Netbird agent https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#enabling-unsafe-sysctls
+      allowed-unsafe-sysctls: net.ipv4.conf.all.src_valid_mark
   network:
     hostname: ${hostname}
   nodeLabels: