Update network

openstack/Makefile

@@ -25,7 +25,7 @@ create-templates:
 	@yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json
 
 create-deployments:
-	helm template --namespace=kube-system   --version=1.11.4 -f deployments/cilium.yaml cilium \
+	helm template --namespace=kube-system   --version=1.11.5 -f deployments/cilium.yaml cilium \
 		cilium/cilium > deployments/cilium-result.yaml
 	helm template --namespace=ingress-nginx --version=4.1.1 -f deployments/ingress.yaml ingress-nginx \
 		ingress-nginx/ingress-nginx > deployments/ingress-result.yaml

openstack/deployments/cilium-result.yaml

@@ -33,6 +33,7 @@ data:
   #   setting it to "kvstore".
   identity-allocation-mode: crd
   cilium-endpoint-gc-interval: "5m0s"
+  nodes-gc-interval: "5m0s"
   # Disable the usage of CiliumEndpoint CRD
   disable-endpoint-crd: "false"
 
@@ -166,6 +167,10 @@ data:
   enable-k8s-endpoint-slice: "true"
   cgroup-root: "/sys/fs/cgroup"
   enable-k8s-terminating-endpoint: "true"
+  annotate-k8s-node: "true"
+  remove-cilium-node-taints: "true"
+  set-cilium-is-up-condition: "true"
+  unmanaged-pod-watcher-interval: "15"
 ---
 # Source: cilium/templates/cilium-agent/clusterrole.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -194,34 +199,19 @@ rules:
   resources:
   - namespaces
   - services
-  - nodes
-  - endpoints
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - nodes
   - pods
+  - endpoints
+  - nodes
   verbs:
   - get
   - list
   - watch
-  - update
 - apiGroups:
   - ""
   resources:
-  - nodes
   - nodes/status
   verbs:
+  # To annotate the k8s node with Cilium's metadata
   - patch
 - apiGroups:
   - apiextensions.k8s.io
@@ -243,21 +233,15 @@ rules:
   resources:
   - ciliumnetworkpolicies
   - ciliumnetworkpolicies/status
-  - ciliumnetworkpolicies/finalizers
   - ciliumclusterwidenetworkpolicies
   - ciliumclusterwidenetworkpolicies/status
-  - ciliumclusterwidenetworkpolicies/finalizers
   - ciliumendpoints
   - ciliumendpoints/status
-  - ciliumendpoints/finalizers
   - ciliumnodes
   - ciliumnodes/status
-  - ciliumnodes/finalizers
   - ciliumidentities
-  - ciliumidentities/finalizers
   - ciliumlocalredirectpolicies
   - ciliumlocalredirectpolicies/status
-  - ciliumlocalredirectpolicies/finalizers
   - ciliumegressnatpolicies
   - ciliumendpointslices
   verbs:
@@ -272,14 +256,30 @@ rules:
 - apiGroups:
   - ""
   resources:
-  # to automatically delete [core|kube]dns pods so that are starting to being
-  # managed by Cilium
   - pods
   verbs:
   - get
   - list
   - watch
+  # to automatically delete [core|kube]dns pods so that are starting to being
+  # managed by Cilium
   - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  # To remove node taints
+  - nodes
+  # To set NetworkUnavailable false on startup
+  - nodes/status
+  verbs:
+  - patch
 - apiGroups:
   - discovery.k8s.io
   resources:
@@ -468,7 +468,7 @@ spec:
             topologyKey: kubernetes.io/hostname
       containers:
       - name: cilium-agent
-        image: "quay.io/cilium/cilium:v1.11.4@sha256:d9d4c7759175db31aa32eaa68274bb9355d468fbc87e23123c80052e3ed63116"
+        image: "quay.io/cilium/cilium:v1.11.5@sha256:79e66c3c2677e9ecc3fd5b2ed8e4ea7e49cf99ed6ee181f2ef43400c4db5eef0"
         imagePullPolicy: IfNotPresent
         command:
         - cilium-agent
@@ -598,7 +598,7 @@ spec:
       hostNetwork: true
       initContainers:
       - name: clean-cilium-state
-        image: "quay.io/cilium/cilium:v1.11.4@sha256:d9d4c7759175db31aa32eaa68274bb9355d468fbc87e23123c80052e3ed63116"
+        image: "quay.io/cilium/cilium:v1.11.5@sha256:79e66c3c2677e9ecc3fd5b2ed8e4ea7e49cf99ed6ee181f2ef43400c4db5eef0"
         imagePullPolicy: IfNotPresent
         command:
         - /init-container.sh
@@ -731,7 +731,7 @@ spec:
             topologyKey: kubernetes.io/hostname
       containers:
       - name: cilium-operator
-        image: quay.io/cilium/operator-generic:v1.11.4@sha256:bf75ad0dc47691a3a519b8ab148ed3a792ffa2f1e309e6efa955f30a40e95adc
+        image: quay.io/cilium/operator-generic:v1.11.5@sha256:8ace281328b27d4216218c604d720b9a63a8aec2bd1996057c79ab0168f9d6d8
         imagePullPolicy: IfNotPresent
         command:
         - cilium-operator-generic

openstack/deployments/openstack-cloud-controller-manager.yaml

@@ -107,9 +107,9 @@ roleRef:
   kind: ClusterRole
   name: system:openstack-cloud-controller-manager
 subjects:
-  - kind: ServiceAccount
-    name: openstack-cloud-controller-manager
-    namespace: kube-system
+- kind: ServiceAccount
+  name: openstack-cloud-controller-manager
+  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -187,7 +187,6 @@ spec:
             - --leader-elect-resource-name=cloud-controller-manager-openstack
             - --use-service-account-credentials
             - --bind-address=127.0.0.1
-            - --secure-port=10267
           env:
             - name: CLUSTER_NAME
               value: kubernetes

openstack/instances-controlplane.tf

@@ -44,5 +44,5 @@ module "controlplane" {
 
 locals {
   lbv4s    = compact([for c in module.controlplane : c.controlplane_lb])
-  endpoint = try(flatten([for c in module.controlplane : c.controlplane_endpoints])[0], "")
+  endpoint = [for ip in try(flatten([for c in module.controlplane : c.controlplane_endpoints]), []) : ip if length(split(".", ip)) > 1]
 }

openstack/modules/controlplane/main.tf

@@ -52,7 +52,7 @@ resource "openstack_compute_instance_v2" "controlplane" {
 }
 
 locals {
-  ipv4_local     = var.instance_count > 0 ? [for k in try(openstack_networking_port_v2.controlplane_public[0].all_fixed_ips, []) : k if length(regexall("[0-9]+.[0-9.]+", k)) > 0][0] : ""
+  ipv4_local     = var.instance_count > 0 ? [for ip in try(openstack_networking_port_v2.controlplane_public[0].all_fixed_ips, []) : ip if length(split(".", ip)) > 1][0] : ""
   ipv4_local_vip = var.instance_count > 0 ? cidrhost(var.network_internal.cidr, 5) : ""
 
   controlplane_labels = "project.io/cloudprovider-type=openstack,topology.kubernetes.io/region=${var.region},topology.kubernetes.io/zone=nova"

openstack/prepare/network-secgroup.tf

@@ -93,7 +93,7 @@ resource "openstack_networking_secgroup_rule_v2" "common_cilium_health_ipv6" {
   protocol          = "tcp"
   port_range_min    = 4240
   port_range_max    = 4240
-  remote_ip_prefix  = local.network_cidr_v6
+  remote_ip_prefix  = "::/0" # cilium uses sometimes public ipv6
 }
 
 resource "openstack_networking_secgroup_rule_v2" "common_cilium_vxvlan" {
@@ -138,6 +138,7 @@ resource "openstack_networking_secgroup_rule_v2" "controlplane_etcd_ipv4" {
   protocol          = "tcp"
   port_range_min    = 2379
   port_range_max    = 2380
+  remote_ip_prefix  = var.network_cidr
 }
 
 resource "openstack_networking_secgroup_rule_v2" "controlplane_kubernetes_ipv4" {

openstack/templates/controlplane.yaml.tpl

@@ -43,7 +43,7 @@ machine:
     net.core.netdev_max_backlog: 4096
 cluster:
   controlPlane:
-    endpoint: https://${ipv4_local_vip}:6443
+    endpoint: https://${apiDomain}:6443
   network:
     dnsDomain: ${domain}
     podSubnets: ${format("%#v",split(",",podSubnets))}

openstack/templates/worker.yaml.tpl

@@ -40,7 +40,7 @@ cluster:
   id: ${clusterID}
   secret: ${clusterSecret}
   controlPlane:
-    endpoint: https://${lbv4}:6443
+    endpoint: https://${apiDomain}:6443
   clusterName: ${clusterName}
   network:
     dnsDomain: ${domain}