cilium: enforce device detection and enable image building (#151)

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
2026-01-27 10:18:39 +00:00 · 2024-05-27 19:40:57 +02:00
parent b5f8006f3c
commit 6211f9d876
10 changed files with 306 additions and 11 deletions
--- a/1
+++ b/1
@@ -3,6 +3,7 @@
 build:
 	make -C packages/apps/http-cache image
 	make -C packages/apps/kubernetes image
+	make -C packages/system/cilium image
 	make -C packages/system/kubeovn image
 	make -C packages/system/dashboard image
 	make -C packages/core/installer image
--- a/packages/system/cilium/Makefile
+++ b/packages/system/cilium/Makefile
@@ -1,6 +1,9 @@
+CILIUM_TAG=$(shell awk '$$1 == "version:" {print $$2}' charts/cilium/Chart.yaml)
+
 NAME=cilium
 NAMESPACE=cozy-$(NAME)

+include ../../../scripts/common-envs.mk
 include ../../../scripts/package-system.mk

 update:
@@ -8,5 +11,20 @@ update:
 	helm repo add cilium https://helm.cilium.io/
 	helm repo update cilium
 	helm pull cilium/cilium --untar --untardir charts --version 1.15
+	ln -s ../../images charts/cilium/images
+	sed -i 's/include "cilium.image" .Values.image/include "cilium.image" ./g' charts/cilium/templates/cilium-agent/daemonset.yaml
 	sed -i -e '/Used in iptables/d' -e '/SYS_MODULE/d' charts/cilium/values.yaml
-	patch -p3 --no-backup-if-mismatch < patches/fix-cgroups.patch
+	version=$$(awk '$$1 == "version:" {print $$2}' charts/cilium/Chart.yaml) && \
+	sed -i "s/ARG VERSION=.*/ARG VERSION=v$${version}/" images/cilium/Dockerfile
+
+image:
+	docker buildx build images/cilium \
+		--provenance false \
+		--tag $(REGISTRY)/cilium:$(call settag,$(CILIUM_TAG)) \
+		--tag $(REGISTRY)/cilium:$(call settag,$(CILIUM_TAG)-$(TAG)) \
+		--cache-from type=registry,ref=$(REGISTRY)/cilium:latest \
+		--cache-to type=inline \
+		--metadata-file images/cilium.json \
+		--push=$(PUSH) \
+		--load=$(LOAD)
+	echo "$(REGISTRY)/cilium:$(call settag,$(TAG))" > images/cilium.tag
--- a/packages/system/cilium/charts/cilium/images
+++ b/packages/system/cilium/charts/cilium/images
@@ -0,0 +1 @@
+../../images
--- a/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml
+++ b/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml
@@ -94,7 +94,7 @@ spec:
      {{- end }}
      containers:
      - name: cilium-agent
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        {{- if .Values.sleepAfterInit }}
        command:
@@ -398,7 +398,7 @@ spec:
        {{- end }}
      {{- if .Values.monitor.enabled }}
      - name: cilium-monitor
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        command:
        - /bin/bash
@@ -430,7 +430,7 @@ spec:
      {{- end }}
      initContainers:
      - name: config
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        command:
        - cilium-dbg
@@ -485,7 +485,7 @@ spec:
      # Required to mount cgroup2 filesystem on the underlying Kubernetes node.
      # We use nsenter command with host's cgroup and mount namespaces enabled.
      - name: mount-cgroup
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        env:
        - name: CGROUP_ROOT
@@ -531,7 +531,7 @@ spec:
              - ALL
          {{- end}}
      - name: apply-sysctl-overwrites
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        {{- with .Values.initResources }}
        resources:
@@ -580,7 +580,7 @@ spec:
      # from a privileged container because the mount propagation bidirectional
      # only works from privileged containers.
      - name: mount-bpf-fs
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        {{- with .Values.initResources }}
        resources:
@@ -605,7 +605,7 @@ spec:
      {{- end }}
      {{- if and .Values.nodeinit.enabled .Values.nodeinit.bootstrapFile }}
      - name: wait-for-node-init
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        {{- with .Values.initResources }}
        resources:
@@ -625,7 +625,7 @@ spec:
          mountPath: "/tmp/cilium-bootstrap.d"
      {{- end }}
      - name: clean-cilium-state
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        command:
        - /init-container.sh
@@ -697,7 +697,7 @@ spec:
        {{- end }}
      {{- if and .Values.waitForKubeProxy (and (ne (toString $kubeProxyReplacement) "strict") (ne (toString $kubeProxyReplacement) "true"))  }}
      - name: wait-for-kube-proxy
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        {{- with .Values.initResources }}
        resources:
@@ -735,7 +735,7 @@ spec:
      {{- if .Values.cni.install }}
      # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent
      - name: install-cni-binaries
-        image: {{ include "cilium.image" .Values.image | quote }}
+        image: {{ include "cilium.image" . | quote }}
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        command:
          - "/install-plugin.sh"
--- a/packages/system/cilium/images/cilium.json
+++ b/packages/system/cilium/images/cilium.json
@@ -0,0 +1,4 @@
+{
+  "containerimage.config.digest": "sha256:5d7a65f2d5c41bd53cccaa55d4f5d28933c08f5294e732b9a00427d091c1d78f",
+  "containerimage.digest": "sha256:f9f46b6c57cbe9ccb2686be7e58236e3bfae0942c4be687f0bf16270832f09ab"
+}
--- a/packages/system/cilium/images/cilium.tag
+++ b/packages/system/cilium/images/cilium.tag
@@ -0,0 +1 @@
+ghcr.io/aenix-io/cozystack/cilium:latest
--- a/packages/system/cilium/images/cilium/Dockerfile
+++ b/packages/system/cilium/images/cilium/Dockerfile
@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:experimental
+
+ARG VERSION=v1.15.5
+
+FROM golang:1.22-bookworm as builder
+
+WORKDIR /source
+
+COPY enforce-device-detection.diff /enforce-device-detection.diff
+
+RUN wget -O- https://github.com/cilium/cilium/archive/refs/tags/v1.15.5.tar.gz | tar xzf - --strip-components=1
+RUN git apply /enforce-device-detection.diff
+RUN make build-agent
+
+FROM quay.io/cilium/cilium:${VERSION}
+COPY --from=builder /source/daemon/cilium-agent /usr/bin/cilium-agent
--- a/packages/system/cilium/images/cilium/enforce-device-detection.diff
+++ b/packages/system/cilium/images/cilium/enforce-device-detection.diff
@@ -0,0 +1,244 @@
+diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go
+index 9435b2570b..416c34ed7a 100644
+--- a/daemon/cmd/daemon_main.go
+++ b/daemon/cmd/daemon_main.go
+@@ -222,6 +222,9 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
+ 	flags.StringSlice(option.Devices, []string{}, "List of devices facing cluster/external network (used for BPF NodePort, BPF masquerading and host firewall); supports '+' as wildcard in device name, e.g. 'eth+'")
+ 	option.BindEnv(vp, option.Devices)
+ 
+	flags.Bool(option.EnforceDeviceDetection, false, "Enforces the auto-detection of devices, even if specific devices are explicitly listed")
+	option.BindEnv(vp, option.EnforceDeviceDetection)
+
+ 	flags.String(option.DirectRoutingDevice, "", "Device name used to connect nodes in direct routing mode (used by BPF NodePort, BPF host routing; if empty, automatically set to a device with k8s InternalIP/ExternalIP or with a default route)")
+ 	option.BindEnv(vp, option.DirectRoutingDevice)
+ 
+diff --git a/install/kubernetes/cilium/README.md b/install/kubernetes/cilium/README.md
+index 6a92a83040..bf728ce32e 100644
+--- a/install/kubernetes/cilium/README.md
+++ b/install/kubernetes/cilium/README.md
+@@ -309,6 +309,7 @@ contributors across the globe, there is almost always someone available to help.
+ | endpointHealthChecking.enabled | bool | `true` | Enable connectivity health checking between virtual endpoints. |
+ | endpointRoutes.enabled | bool | `false` | Enable use of per endpoint routes instead of routing via the cilium_host interface. |
+ | endpointStatus | object | `{"enabled":false,"status":""}` | Enable endpoint status. Status can be: policy, health, controllers, log and / or state. For 2 or more options use a space. |
+| enforceDeviceDetection | bool | `false` | Enforces the auto-detection of devices, even if specific devices are explicitly listed |
+ | eni.awsEnablePrefixDelegation | bool | `false` | Enable ENI prefix delegation |
+ | eni.awsReleaseExcessIPs | bool | `false` | Release IPs not used from the ENI |
+ | eni.ec2APIEndpoint | string | `""` | EC2 API endpoint to use |
+diff --git a/install/kubernetes/cilium/templates/cilium-configmap.yaml b/install/kubernetes/cilium/templates/cilium-configmap.yaml
+index 9531ca18a8..1ce677f8ce 100644
+--- a/install/kubernetes/cilium/templates/cilium-configmap.yaml
+++ b/install/kubernetes/cilium/templates/cilium-configmap.yaml
+@@ -688,6 +688,10 @@ data:
+   enable-runtime-device-detection: "true"
+ {{- end }}
+ 
+{{- if .Values.enforceDeviceDetection }}
+  enforce-device-detection: "true"
+{{- end }}
+
+   kube-proxy-replacement: {{ $kubeProxyReplacement | quote }}
+ 
+ {{- if ne $kubeProxyReplacement "disabled" }}
+diff --git a/install/kubernetes/cilium/values.yaml b/install/kubernetes/cilium/values.yaml
+index e81d3ebffd..2196bbb814 100644
+--- a/install/kubernetes/cilium/values.yaml
+++ b/install/kubernetes/cilium/values.yaml
+@@ -690,6 +690,9 @@ daemon:
+ # be considered.
+ enableRuntimeDeviceDetection: false
+ 
+# -- Enforces the auto-detection of devices, even if specific devices are explicitly listed
+enforceDeviceDetection: false
+
+ # -- Chains to ignore when installing feeder rules.
+ # disableIptablesFeederRules: ""
+ 
+diff --git a/install/kubernetes/cilium/values.yaml.tmpl b/install/kubernetes/cilium/values.yaml.tmpl
+index 679b4cac07..3ed0b8f88b 100644
+--- a/install/kubernetes/cilium/values.yaml.tmpl
+++ b/install/kubernetes/cilium/values.yaml.tmpl
+@@ -687,6 +687,9 @@ daemon:
+ # be considered.
+ enableRuntimeDeviceDetection: false
+ 
+# -- Enforces the auto-detection of devices, even if specific devices are explicitly listed
+enforceDeviceDetection: false
+
+ # -- Chains to ignore when installing feeder rules.
+ # disableIptablesFeederRules: ""
+ 
+diff --git a/pkg/datapath/cells.go b/pkg/datapath/cells.go
+index 69d6d71836..c0f4947d73 100644
+--- a/pkg/datapath/cells.go
+++ b/pkg/datapath/cells.go
+@@ -114,7 +114,8 @@ var Cell = cell.Module(
+ 		// This is temporary until DevicesController takes ownership of the
+ 		// device-related configuration options.
+ 		return linuxdatapath.DevicesConfig{
+-			Devices: cfg.GetDevices(),
+			Devices:                cfg.GetDevices(),
+			EnforceDeviceDetection: option.Config.EnforceDeviceDetection,
+ 		}
+ 	}),
+ 
+diff --git a/pkg/datapath/linux/devices_controller.go b/pkg/datapath/linux/devices_controller.go
+index fdfde8d4ec..cac3e44a32 100644
+--- a/pkg/datapath/linux/devices_controller.go
+++ b/pkg/datapath/linux/devices_controller.go
+@@ -84,6 +84,9 @@ type DevicesConfig struct {
+ 	// If empty the devices are auto-detected according to rules defined
+ 	// by isSelectedDevice().
+ 	Devices []string
+	// EnforceDeviceDetection forces the auto-detection of devices,
+	// even if user-specific devices are explicitly listed.
+	EnforceDeviceDetection bool
+ }
+ 
+ type devicesControllerParams struct {
+@@ -103,9 +106,10 @@ type devicesController struct {
+ 	params devicesControllerParams
+ 	log    logrus.FieldLogger
+ 
+-	initialized    chan struct{}
+-	filter         deviceFilter
+-	l3DevSupported bool
+	initialized          chan struct{}
+	filter               deviceFilter
+	enforceAutoDetection bool
+	l3DevSupported       bool
+ 
+ 	// deadLinkIndexes tracks the set of links that have been deleted. This is needed
+ 	// to avoid processing route or address updates after a link delete as they may
+@@ -121,11 +125,12 @@ func newDevicesController(lc cell.Lifecycle, p devicesControllerParams) (*device
+ 		p.RouteTable,
+ 	)
+ 	dc := &devicesController{
+-		params:          p,
+-		initialized:     make(chan struct{}),
+-		filter:          deviceFilter(p.Config.Devices),
+-		log:             p.Log,
+-		deadLinkIndexes: sets.New[int](),
+		params:               p,
+		initialized:          make(chan struct{}),
+		filter:               deviceFilter(p.Config.Devices),
+		enforceAutoDetection: p.Config.EnforceDeviceDetection,
+		log:                  p.Log,
+		deadLinkIndexes:      sets.New[int](),
+ 	}
+ 	lc.Append(dc)
+ 	return dc, p.DeviceTable, p.RouteTable
+@@ -529,12 +534,15 @@ func (dc *devicesController) isSelectedDevice(d *tables.Device, txn statedb.Writ
+ 	}
+ 
+ 	// If user specified devices or wildcards, then skip the device if it doesn't match.
+-	// If the device does match, then skip further checks.
+	// If the device does match and user not requested auto detection, then skip further checks.
+	// If the device does match and user requested auto detection, then continue to further checks.
+ 	if dc.filter.nonEmpty() {
+ 		if dc.filter.match(d.Name) {
+ 			return true, ""
+ 		}
+-		return false, fmt.Sprintf("not matching user filter %v", dc.filter)
+		if !dc.enforceAutoDetection {
+			return false, fmt.Sprintf("not matching user filter %v", dc.filter)
+		}
+ 	}
+ 
+ 	// Skip devices that have an excluded interface flag set.
+diff --git a/pkg/datapath/linux/devices_test.go b/pkg/datapath/linux/devices_test.go
+index cd9fdcc7ad..031894498f 100644
+--- a/pkg/datapath/linux/devices_test.go
+++ b/pkg/datapath/linux/devices_test.go
+@@ -254,7 +254,36 @@ func (s *DevicesSuite) TestDetect(c *C) {
+ 		c.Assert(devices, checker.DeepEquals, []string{"bond0", "dummy0", "dummy1", "dummy_v6", "veth0"})
+ 		option.Config.SetDevices([]string{})
+ 		dm.Stop()
+
+		// EnforceDeviceDetection enabled with specific devices
+		option.Config.SetDevices([]string{"dummy1"})
+		option.Config.EnforceDeviceDetection = true
+		c.Assert(createDummy("dummy0", "192.168.0.1/24", false), IsNil)
+		c.Assert(createDummy("dummy1", "192.168.1.1/24", false), IsNil)
+
+		dm, err = newDeviceManagerForTests()
+		c.Assert(err, IsNil)
+		devices, err = dm.Detect(true)
+		c.Assert(err, IsNil)
+		c.Assert(devices, checker.DeepEquals, []string{"dummy0", "dummy1"})
+		option.Config.SetDevices([]string{})
+		option.Config.DirectRoutingDevice = ""
+		dm.Stop()
+
+		// EnforceDeviceDetection disabled with specific devices
+		option.Config.SetDevices([]string{"dummy1"})
+		option.Config.EnforceDeviceDetection = false
+
+		dm, err = newDeviceManagerForTests()
+		c.Assert(err, IsNil)
+		devices, err = dm.Detect(true)
+		c.Assert(err, IsNil)
+		c.Assert(devices, checker.DeepEquals, []string{"dummy1"})
+		option.Config.SetDevices([]string{})
+		option.Config.DirectRoutingDevice = ""
+		dm.Stop()
+ 	})
+
+ }
+ 
+ func (s *DevicesSuite) TestExpandDevices(c *C) {
+diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go
+index fe34caa624..f095ff9a42 100644
+--- a/pkg/defaults/defaults.go
+++ b/pkg/defaults/defaults.go
+@@ -554,6 +554,10 @@ const (
+ 	// identity in a numeric identity. Values > 255 will decrease the number of
+ 	// allocatable identities.
+ 	MaxConnectedClusters = 255
+
+	// EnforceDevicesDetection enforces the auto-detection of devices,
+	// even if specific devices are explicitly listed
+	EnforceDeviceDetection = false
+ )
+ 
+ var (
+diff --git a/pkg/option/config.go b/pkg/option/config.go
+index 27c697d9e3..9ba4367b2e 100644
+--- a/pkg/option/config.go
+++ b/pkg/option/config.go
+@@ -123,6 +123,9 @@ const (
+ 	// Devices facing cluster/external network for attaching bpf_host
+ 	Devices = "devices"
+ 
+	// Enforces the auto-detection of devices, even if specific devices are explicitly listed
+	EnforceDeviceDetection = "enforce-device-detection"
+
+ 	// DirectRoutingDevice is the name of a device used to connect nodes in
+ 	// direct routing mode (only required by BPF NodePort)
+ 	DirectRoutingDevice = "direct-routing-device"
+@@ -2447,6 +2450,10 @@ type DaemonConfig struct {
+ 
+ 	// ServiceNoBackendResponse determines how we handle traffic to a service with no backends.
+ 	ServiceNoBackendResponse string
+
+	// EnforceDeviceDetection forces the auto-detection of devices,
+	// even if specific devices are explicitly listed
+	EnforceDeviceDetection bool
+ }
+ 
+ var (
+@@ -2490,6 +2497,7 @@ var (
+ 		AllocatorListTimeout:            defaults.AllocatorListTimeout,
+ 		EnableICMPRules:                 defaults.EnableICMPRules,
+ 		UseCiliumInternalIPForIPsec:     defaults.UseCiliumInternalIPForIPsec,
+		EnforceDeviceDetection:          defaults.EnforceDeviceDetection,
+ 
+ 		K8sEnableLeasesFallbackDiscovery: defaults.K8sEnableLeasesFallbackDiscovery,
+ 
+@@ -3563,6 +3571,7 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) {
+ 	c.UseCiliumInternalIPForIPsec = vp.GetBool(UseCiliumInternalIPForIPsec)
+ 	c.BypassIPAvailabilityUponRestore = vp.GetBool(BypassIPAvailabilityUponRestore)
+ 	c.EnableK8sTerminatingEndpoint = vp.GetBool(EnableK8sTerminatingEndpoint)
+	c.EnforceDeviceDetection = vp.GetBool(EnforceDeviceDetection)
+ 
+ 	// Disable Envoy version check if L7 proxy is disabled.
+ 	c.DisableEnvoyVersionCheck = vp.GetBool(DisableEnvoyVersionCheck)
--- a/packages/system/cilium/templates/_helpers.tpl
+++ b/packages/system/cilium/templates/_helpers.tpl
@@ -0,0 +1,3 @@
+{{- define "cilium.image" -}}
+{{ .Files.Get "images/cilium.tag" | trim }}@{{ index (.Files.Get "images/cilium.json" | fromJson) "containerimage.digest" }}
+{{- end -}}
--- a/packages/system/cilium/values.yaml
+++ b/packages/system/cilium/values.yaml
@@ -24,4 +24,11 @@ cilium:
    configMap: cni-configuration
  routingMode: native
  enableIPv4Masquerade: false
+  enableIPv6Masquerade: false
  enableIdentityMark: false
+  enableRuntimeDeviceDetection: true
+  #enforceDeviceDetection: true
+  devices: ovn0
+  extraEnv:
+  - name: CILIUM_ENFORCE_DEVICE_DETECTION
+    value: "true"