Update cilium

This commit is contained in:
Serge Logvinov
2021-12-25 16:56:17 +02:00
parent 495aaed683
commit df9cddf2b1
4 changed files with 126 additions and 146 deletions

View File

@@ -38,3 +38,6 @@ create-infrastructure: ## Bootstrap all nodes
create-kubeconfig: ## Prepare kubeconfig create-kubeconfig: ## Prepare kubeconfig
talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.11 kubeconfig talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.11 kubeconfig
create-deployments:
helm template --namespace=kube-system --version=1.11.0 -f deployments/cilium.yaml cilium cilium/cilium > deployments/cilium_result.yaml

View File

@@ -13,24 +13,26 @@ operator:
enabled: false enabled: false
identityAllocationMode: crd identityAllocationMode: crd
kubeProxyReplacement: strict
enableK8sEndpointSlice: true
localRedirectPolicy: true
bpf: healthChecking: true
masquerade: false
tunnel: "vxlan"
autoDirectNodeRoutes: false
cni: cni:
install: true install: true
ipam: ipam:
mode: "kubernetes" mode: "kubernetes"
k8s:
requireIPv4PodCIDR: true
requireIPv6PodCIDR: true
tunnel: "vxlan" bpf:
autoDirectNodeRoutes: false masquerade: false
hostFirewall: true
kubeProxyReplacement: strict
healthChecking: true
ipv4: ipv4:
enabled: true enabled: true
ipv6: ipv6:
@@ -43,17 +45,15 @@ nodePort:
enabled: false enabled: false
externalIPs: externalIPs:
enabled: true enabled: true
hostFirewall:
enabled: true
k8s: hubble:
requireIPv4PodCIDR: true enabled: false
requireIPv6PodCIDR: true
prometheus: prometheus:
enabled: true enabled: true
encryption:
enabled: false
cgroup: cgroup:
autoMount: autoMount:
enabled: false enabled: false

View File

@@ -1,12 +1,12 @@
--- ---
# Source: cilium/templates/cilium-agent-serviceaccount.yaml # Source: cilium/templates/cilium-agent/serviceaccount.yaml
apiVersion: v1 apiVersion: v1
kind: ServiceAccount kind: ServiceAccount
metadata: metadata:
name: "cilium" name: "cilium"
namespace: kube-system namespace: kube-system
--- ---
# Source: cilium/templates/cilium-operator-serviceaccount.yaml # Source: cilium/templates/cilium-operator/serviceaccount.yaml
apiVersion: v1 apiVersion: v1
kind: ServiceAccount kind: ServiceAccount
metadata: metadata:
@@ -26,13 +26,15 @@ data:
# - "crd" stores identities in kubernetes as CRDs (custom resource definition). # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
# These can be queried with: # These can be queried with:
# kubectl get ciliumid # kubectl get ciliumid
# - "kvstore" stores identities in a kvstore, etcd or consul, that is # - "kvstore" stores identities in an etcd kvstore, that is
# configured below. Cilium versions before 1.6 supported only the kvstore # configured below. Cilium versions before 1.6 supported only the kvstore
# backend. Upgrades from these older cilium versions should continue using # backend. Upgrades from these older cilium versions should continue using
# the kvstore by commenting out the identity-allocation-mode below, or # the kvstore by commenting out the identity-allocation-mode below, or
# setting it to "kvstore". # setting it to "kvstore".
identity-allocation-mode: crd identity-allocation-mode: crd
cilium-endpoint-gc-interval: "5m0s" cilium-endpoint-gc-interval: "5m0s"
# Disable the usage of CiliumEndpoint CRD
disable-endpoint-crd: "false"
# If you want to run cilium in debug mode change this value to true # If you want to run cilium in debug mode change this value to true
debug: "false" debug: "false"
@@ -137,7 +139,7 @@ data:
auto-direct-node-routes: "false" auto-direct-node-routes: "false"
enable-bandwidth-manager: "false" enable-bandwidth-manager: "false"
enable-local-redirect-policy: "false" enable-local-redirect-policy: "true"
enable-host-firewall: "true" enable-host-firewall: "true"
kube-proxy-replacement: "strict" kube-proxy-replacement: "strict"
@@ -146,6 +148,7 @@ data:
node-port-bind-protection: "true" node-port-bind-protection: "true"
enable-auto-protect-node-port-range: "true" enable-auto-protect-node-port-range: "true"
enable-session-affinity: "true" enable-session-affinity: "true"
enable-l2-neigh-discovery: "true"
k8s-require-ipv4-pod-cidr: "true" k8s-require-ipv4-pod-cidr: "true"
k8s-require-ipv6-pod-cidr: "true" k8s-require-ipv6-pod-cidr: "true"
enable-endpoint-health-checking: "true" enable-endpoint-health-checking: "true"
@@ -153,21 +156,13 @@ data:
enable-well-known-identities: "false" enable-well-known-identities: "false"
enable-remote-node-identity: "true" enable-remote-node-identity: "true"
operator-api-serve-addr: "127.0.0.1:9234" operator-api-serve-addr: "127.0.0.1:9234"
# Enable Hubble gRPC service.
enable-hubble: "true"
# UNIX domain socket for Hubble server to listen to.
hubble-socket-path: "/var/run/cilium/hubble.sock"
# An additional address for Hubble server to listen to (e.g. ":4244").
hubble-listen-address: ":4244"
hubble-disable-tls: "false"
hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt
hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key
hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt
ipam: "kubernetes" ipam: "kubernetes"
disable-cnp-status-updates: "true" disable-cnp-status-updates: "true"
enable-k8s-endpoint-slice: "true"
cgroup-root: "/sys/fs/cgroup" cgroup-root: "/sys/fs/cgroup"
enable-k8s-terminating-endpoint: "true"
--- ---
# Source: cilium/templates/cilium-agent-clusterrole.yaml # Source: cilium/templates/cilium-agent/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole kind: ClusterRole
metadata: metadata:
@@ -263,10 +258,11 @@ rules:
- ciliumlocalredirectpolicies/status - ciliumlocalredirectpolicies/status
- ciliumlocalredirectpolicies/finalizers - ciliumlocalredirectpolicies/finalizers
- ciliumegressnatpolicies - ciliumegressnatpolicies
- ciliumendpointslices
verbs: verbs:
- '*' - '*'
--- ---
# Source: cilium/templates/cilium-operator-clusterrole.yaml # Source: cilium/templates/cilium-operator/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole kind: ClusterRole
metadata: metadata:
@@ -334,6 +330,7 @@ rules:
- ciliumnodes/status - ciliumnodes/status
- ciliumnodes/finalizers - ciliumnodes/finalizers
- ciliumidentities - ciliumidentities
- ciliumendpointslices
- ciliumidentities/status - ciliumidentities/status
- ciliumidentities/finalizers - ciliumidentities/finalizers
- ciliumlocalredirectpolicies - ciliumlocalredirectpolicies
@@ -366,7 +363,7 @@ rules:
- get - get
- update - update
--- ---
# Source: cilium/templates/cilium-agent-clusterrolebinding.yaml # Source: cilium/templates/cilium-agent/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding kind: ClusterRoleBinding
metadata: metadata:
@@ -380,7 +377,7 @@ subjects:
name: "cilium" name: "cilium"
namespace: kube-system namespace: kube-system
--- ---
# Source: cilium/templates/cilium-operator-clusterrolebinding.yaml # Source: cilium/templates/cilium-operator/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding kind: ClusterRoleBinding
metadata: metadata:
@@ -394,36 +391,36 @@ subjects:
name: "cilium-operator" name: "cilium-operator"
namespace: kube-system namespace: kube-system
--- ---
# Source: cilium/templates/cilium-agent-service.yaml # Source: cilium/templates/cilium-agent/service.yaml
kind: Service
apiVersion: v1 apiVersion: v1
kind: Service
metadata: metadata:
name: cilium-agent name: cilium-agent
namespace: kube-system namespace: kube-system
annotations: annotations:
prometheus.io/scrape: 'true' prometheus.io/scrape: "true"
prometheus.io/port: "9095" prometheus.io/port: "9095"
labels: labels:
k8s-app: cilium k8s-app: cilium
spec: spec:
clusterIP: None clusterIP: None
type: ClusterIP type: ClusterIP
selector:
k8s-app: cilium
ports: ports:
- name: envoy-metrics - name: envoy-metrics
port: 9095 port: 9095
protocol: TCP protocol: TCP
targetPort: envoy-metrics targetPort: envoy-metrics
selector:
k8s-app: cilium
--- ---
# Source: cilium/templates/cilium-agent-daemonset.yaml # Source: cilium/templates/cilium-agent/daemonset.yaml
apiVersion: apps/v1 apiVersion: apps/v1
kind: DaemonSet kind: DaemonSet
metadata: metadata:
labels:
k8s-app: cilium
name: cilium name: cilium
namespace: kube-system namespace: kube-system
labels:
k8s-app: cilium
spec: spec:
selector: selector:
matchLabels: matchLabels:
@@ -469,13 +466,16 @@ spec:
- cilium - cilium
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
containers: containers:
- args: - name: cilium-agent
- --config-dir=/tmp/cilium/config-map image: "quay.io/cilium/cilium:v1.11.0@sha256:ea677508010800214b0b5497055f38ed3bff57963fa2399bcb1c69cf9476453a"
imagePullPolicy: IfNotPresent
command: command:
- cilium-agent - cilium-agent
args:
- --config-dir=/tmp/cilium/config-map
startupProbe: startupProbe:
httpGet: httpGet:
host: '127.0.0.1' host: "127.0.0.1"
path: /healthz path: /healthz
port: 9876 port: 9876
scheme: HTTP scheme: HTTP
@@ -487,29 +487,29 @@ spec:
successThreshold: 1 successThreshold: 1
livenessProbe: livenessProbe:
httpGet: httpGet:
host: '127.0.0.1' host: "127.0.0.1"
path: /healthz path: /healthz
port: 9876 port: 9876
scheme: HTTP scheme: HTTP
httpHeaders: httpHeaders:
- name: "brief" - name: "brief"
value: "true" value: "true"
failureThreshold: 10
periodSeconds: 30 periodSeconds: 30
successThreshold: 1 successThreshold: 1
failureThreshold: 10
timeoutSeconds: 5 timeoutSeconds: 5
readinessProbe: readinessProbe:
httpGet: httpGet:
host: '127.0.0.1' host: "127.0.0.1"
path: /healthz path: /healthz
port: 9876 port: 9876
scheme: HTTP scheme: HTTP
httpHeaders: httpHeaders:
- name: "brief" - name: "brief"
value: "true" value: "true"
failureThreshold: 3
periodSeconds: 30 periodSeconds: 30
successThreshold: 1 successThreshold: 1
failureThreshold: 3
timeoutSeconds: 5 timeoutSeconds: 5
env: env:
- name: K8S_NODE_NAME - name: K8S_NODE_NAME
@@ -527,21 +527,19 @@ spec:
- name: CILIUM_CNI_CHAINING_MODE - name: CILIUM_CNI_CHAINING_MODE
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
key: cni-chaining-mode
name: cilium-config name: cilium-config
key: cni-chaining-mode
optional: true optional: true
- name: CILIUM_CUSTOM_CNI_CONF - name: CILIUM_CUSTOM_CNI_CONF
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
key: custom-cni-conf
name: cilium-config name: cilium-config
key: custom-cni-conf
optional: true optional: true
- name: KUBERNETES_SERVICE_HOST - name: KUBERNETES_SERVICE_HOST
value: "172.16.0.10" value: "172.16.0.10"
- name: KUBERNETES_SERVICE_PORT - name: KUBERNETES_SERVICE_PORT
value: "6443" value: "6443"
image: "quay.io/cilium/cilium:v1.10.4@sha256:7d354052ccf2a7445101d78cebd14444c7c40129ce7889f2f04b89374dbf8a1d"
imagePullPolicy: IfNotPresent
lifecycle: lifecycle:
postStart: postStart:
exec: exec:
@@ -557,87 +555,77 @@ spec:
requests: requests:
cpu: 100m cpu: 100m
memory: 128Mi memory: 128Mi
name: cilium-agent
ports: ports:
- containerPort: 9090 - name: prometheus
containerPort: 9090
hostPort: 9090 hostPort: 9090
name: prometheus protocol: TCP
protocol: TCP - name: envoy-metrics
- containerPort: 9095 containerPort: 9095
hostPort: 9095 hostPort: 9095
name: envoy-metrics
protocol: TCP protocol: TCP
securityContext: securityContext:
capabilities:
add:
- NET_ADMIN
- SYS_MODULE
privileged: true privileged: true
volumeMounts: volumeMounts:
- mountPath: /sys/fs/bpf - name: bpf-maps
name: bpf-maps mountPath: /sys/fs/bpf
mountPropagation: Bidirectional
# Check for duplicate mounts before mounting # Check for duplicate mounts before mounting
- mountPath: /sys/fs/cgroup - name: cilium-cgroup
name: cilium-cgroup mountPath: /sys/fs/cgroup
- mountPath: /var/run/cilium - name: cilium-run
name: cilium-run mountPath: /var/run/cilium
- mountPath: /host/opt/cni/bin - name: cni-path
name: cni-path mountPath: /host/opt/cni/bin
- mountPath: /host/etc/cni/net.d - name: etc-cni-netd
name: etc-cni-netd mountPath: /host/etc/cni/net.d
- mountPath: /var/lib/cilium/clustermesh - name: clustermesh-secrets
name: clustermesh-secrets mountPath: /var/lib/cilium/clustermesh
readOnly: true readOnly: true
- mountPath: /tmp/cilium/config-map - name: cilium-config-path
name: cilium-config-path mountPath: /tmp/cilium/config-map
readOnly: true readOnly: true
# Needed to be able to load kernel modules # Needed to be able to load kernel modules
- mountPath: /lib/modules - name: lib-modules
name: lib-modules mountPath: /lib/modules
readOnly: true
- mountPath: /run/xtables.lock
name: xtables-lock
- mountPath: /var/lib/cilium/tls/hubble
name: hubble-tls
readOnly: true readOnly: true
- name: xtables-lock
mountPath: /run/xtables.lock
hostNetwork: true hostNetwork: true
initContainers: initContainers:
- command: - name: clean-cilium-state
image: "quay.io/cilium/cilium:v1.11.0@sha256:ea677508010800214b0b5497055f38ed3bff57963fa2399bcb1c69cf9476453a"
imagePullPolicy: IfNotPresent
command:
- /init-container.sh - /init-container.sh
env: env:
- name: CILIUM_ALL_STATE - name: CILIUM_ALL_STATE
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
key: clean-cilium-state
name: cilium-config name: cilium-config
key: clean-cilium-state
optional: true optional: true
- name: CILIUM_BPF_STATE - name: CILIUM_BPF_STATE
valueFrom: valueFrom:
configMapKeyRef: configMapKeyRef:
key: clean-cilium-bpf-state
name: cilium-config name: cilium-config
key: clean-cilium-bpf-state
optional: true optional: true
- name: KUBERNETES_SERVICE_HOST - name: KUBERNETES_SERVICE_HOST
value: "172.16.0.10" value: "172.16.0.10"
- name: KUBERNETES_SERVICE_PORT - name: KUBERNETES_SERVICE_PORT
value: "6443" value: "6443"
image: "quay.io/cilium/cilium:v1.10.4@sha256:7d354052ccf2a7445101d78cebd14444c7c40129ce7889f2f04b89374dbf8a1d"
imagePullPolicy: IfNotPresent
name: clean-cilium-state
securityContext: securityContext:
capabilities:
add:
- NET_ADMIN
privileged: true privileged: true
volumeMounts: volumeMounts:
- mountPath: /sys/fs/bpf - name: bpf-maps
name: bpf-maps mountPath: /sys/fs/bpf
# Required to mount cgroup filesystem from the host to cilium agent pod # Required to mount cgroup filesystem from the host to cilium agent pod
- mountPath: /sys/fs/cgroup - name: cilium-cgroup
name: cilium-cgroup mountPath: /sys/fs/cgroup
mountPropagation: HostToContainer mountPropagation: HostToContainer
- mountPath: /var/run/cilium - name: cilium-run
name: cilium-run mountPath: /var/run/cilium
resources: resources:
requests: requests:
cpu: 100m cpu: 100m
@@ -648,75 +636,63 @@ spec:
serviceAccountName: "cilium" serviceAccountName: "cilium"
terminationGracePeriodSeconds: 1 terminationGracePeriodSeconds: 1
tolerations: tolerations:
- operator: Exists - operator: Exists
volumes: volumes:
# To keep state between restarts / upgrades # To keep state between restarts / upgrades
- hostPath: - name: cilium-run
hostPath:
path: /var/run/cilium path: /var/run/cilium
type: DirectoryOrCreate type: DirectoryOrCreate
name: cilium-run
# To keep state between restarts / upgrades for bpf maps # To keep state between restarts / upgrades for bpf maps
- hostPath: - name: bpf-maps
hostPath:
path: /sys/fs/bpf path: /sys/fs/bpf
type: DirectoryOrCreate type: DirectoryOrCreate
name: bpf-maps
# To keep state between restarts / upgrades for cgroup2 filesystem # To keep state between restarts / upgrades for cgroup2 filesystem
- hostPath: - name: cilium-cgroup
hostPath:
path: /sys/fs/cgroup path: /sys/fs/cgroup
type: DirectoryOrCreate type: DirectoryOrCreate
name: cilium-cgroup
# To install cilium cni plugin in the host # To install cilium cni plugin in the host
- hostPath: - name: cni-path
hostPath:
path: /opt/cni/bin path: /opt/cni/bin
type: DirectoryOrCreate type: DirectoryOrCreate
name: cni-path
# To install cilium cni configuration in the host # To install cilium cni configuration in the host
- hostPath: - name: etc-cni-netd
hostPath:
path: /etc/cni/net.d path: /etc/cni/net.d
type: DirectoryOrCreate type: DirectoryOrCreate
name: etc-cni-netd
# To be able to load kernel modules # To be able to load kernel modules
- hostPath: - name: lib-modules
hostPath:
path: /lib/modules path: /lib/modules
name: lib-modules
# To access iptables concurrently with other processes (e.g. kube-proxy) # To access iptables concurrently with other processes (e.g. kube-proxy)
- hostPath: - name: xtables-lock
hostPath:
path: /run/xtables.lock path: /run/xtables.lock
type: FileOrCreate type: FileOrCreate
name: xtables-lock
# To read the clustermesh configuration # To read the clustermesh configuration
- name: clustermesh-secrets - name: clustermesh-secrets
secret: secret:
defaultMode: 420
optional: true
secretName: cilium-clustermesh secretName: cilium-clustermesh
# note: the leading zero means this number is in octal representation: do not remove it
defaultMode: 0400
optional: true
# To read the configuration from the config map # To read the configuration from the config map
- configMap: - name: cilium-config-path
configMap:
name: cilium-config name: cilium-config
name: cilium-config-path
- name: hubble-tls
projected:
sources:
- secret:
name: hubble-server-certs
items:
- key: ca.crt
path: client-ca.crt
- key: tls.crt
path: server.crt
- key: tls.key
path: server.key
optional: true
--- ---
# Source: cilium/templates/cilium-operator-deployment.yaml # Source: cilium/templates/cilium-operator/deployment.yaml
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
name: cilium-operator
namespace: kube-system
labels: labels:
io.cilium/app: operator io.cilium/app: operator
name: cilium-operator name: cilium-operator
name: cilium-operator
namespace: kube-system
spec: spec:
# See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go
# for more details. # for more details.
@@ -750,11 +726,14 @@ spec:
- operator - operator
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
containers: containers:
- args: - name: cilium-operator
- --config-dir=/tmp/cilium/config-map image: quay.io/cilium/operator-generic:v1.11.0@sha256:b522279577d0d5f1ad7cadaacb7321d1b172d8ae8c8bc816e503c897b420cfe3
- --debug=$(CILIUM_DEBUG) imagePullPolicy: IfNotPresent
command: command:
- cilium-operator-generic - cilium-operator-generic
args:
- --config-dir=/tmp/cilium/config-map
- --debug=$(CILIUM_DEBUG)
env: env:
- name: K8S_NODE_NAME - name: K8S_NODE_NAME
valueFrom: valueFrom:
@@ -776,12 +755,9 @@ spec:
value: "172.16.0.10" value: "172.16.0.10"
- name: KUBERNETES_SERVICE_PORT - name: KUBERNETES_SERVICE_PORT
value: "6443" value: "6443"
image: "quay.io/cilium/operator-generic:v1.10.4@sha256:c49a14e34634ff1a494c84b718641f27267fb3a0291ce3d74352b44f8a8d2f93"
imagePullPolicy: IfNotPresent
name: cilium-operator
livenessProbe: livenessProbe:
httpGet: httpGet:
host: '127.0.0.1' host: "127.0.0.1"
path: /healthz path: /healthz
port: 9234 port: 9234
scheme: HTTP scheme: HTTP
@@ -789,8 +765,8 @@ spec:
periodSeconds: 10 periodSeconds: 10
timeoutSeconds: 3 timeoutSeconds: 3
volumeMounts: volumeMounts:
- mountPath: /tmp/cilium/config-map - name: cilium-config-path
name: cilium-config-path mountPath: /tmp/cilium/config-map
readOnly: true readOnly: true
hostNetwork: true hostNetwork: true
restartPolicy: Always restartPolicy: Always
@@ -798,9 +774,9 @@ spec:
serviceAccount: "cilium-operator" serviceAccount: "cilium-operator"
serviceAccountName: "cilium-operator" serviceAccountName: "cilium-operator"
tolerations: tolerations:
- operator: Exists - operator: Exists
volumes: volumes:
# To read the configuration from the config map # To read the configuration from the config map
- configMap: - name: cilium-config-path
configMap:
name: cilium-config name: cilium-config
name: cilium-config-path

View File

@@ -101,6 +101,7 @@ cluster:
externalCloudProvider: externalCloudProvider:
enabled: true enabled: true
manifests: manifests:
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/coredns-local.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/kubelet-serving-cert-approver.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/kubelet-serving-cert-approver.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/metrics-server.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/metrics-server.yaml