Change flannel port from 4789 to 8472

* flannel and Cilium default to UDP 8472 for VXLAN traffic to
avoid conflicts with other VXLAN usage (e.g. Open vSwith)
* Aligning flannel and Cilium to use the same vxlan port makes
firewall rules or security policies simpler across clouds

Rel: https://github.com/poseidon/terraform-render-bootstrap/pull/403

CHANGES.md

@@ -9,6 +9,8 @@ Notable changes between versions.
 * Kubernetes [v1.32.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.32.md#v1320)
 * Change the default Pod CIDR from 10.2.0.0/16 to 10.20.0.0/14 ([#1555](https://github.com/poseidon/typhoon/pull/1555))
 * Configure Kubelets for parallel image pulls ([#1556](https://github.com/poseidon/typhoon/pull/1556))
+* Change flannel port from 4789 to 8472 to match Cilium ([#1561](https://github.com/poseidon/typhoon/pull/1561))
+  * Reverses a choice made in [#466](https://github.com/poseidon/typhoon/pull/466)
 * Remove support for Calico CNI (choose between `networking` cilium or flannel) ([#1558](https://github.com/poseidon/typhoon/pull/1558))
   * Remove Calico firewall rules or security group rules
   * Remove `network_mtu`, `network_encapsulation`, and `network_ip_autodetection_method` variables (Calico-specific)

addons/flannel/config.tf

@@ -35,7 +35,7 @@ resource "kubernetes_config_map" "config" {
         "Network": "${var.pod_cidr}",
         "Backend": {
           "Type": "vxlan",
-          "Port": 4789
+          "Port": 8472
         }
       }
     EOF

aws/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/fedora-coreos/kubernetes/security.tf

@@ -116,31 +116,6 @@ resource "aws_security_group_rule" "controller-cilium-metrics-self" {
   self      = true
 }
 
-# IANA VXLAN default
-resource "aws_security_group_rule" "controller-vxlan" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.controller.id
-
-  type                     = "ingress"
-  protocol                 = "udp"
-  from_port                = 4789
-  to_port                  = 4789
-  source_security_group_id = aws_security_group.worker.id
-}
-
-resource "aws_security_group_rule" "controller-vxlan-self" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.controller.id
-
-  type      = "ingress"
-  protocol  = "udp"
-  from_port = 4789
-  to_port   = 4789
-  self      = true
-}
-
 resource "aws_security_group_rule" "controller-apiserver" {
   security_group_id = aws_security_group.controller.id
 
@@ -152,9 +127,7 @@ resource "aws_security_group_rule" "controller-apiserver" {
 }
 
 # Linux VXLAN default
-resource "aws_security_group_rule" "controller-linux-vxlan" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "controller-vxlan" {
   security_group_id = aws_security_group.controller.id
 
   type                     = "ingress"
@@ -164,9 +137,7 @@ resource "aws_security_group_rule" "controller-linux-vxlan" {
   source_security_group_id = aws_security_group.worker.id
 }
 
-resource "aws_security_group_rule" "controller-linux-vxlan-self" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "controller-vxlan-self" {
   security_group_id = aws_security_group.controller.id
 
   type      = "ingress"
@@ -367,35 +338,8 @@ resource "aws_security_group_rule" "worker-cilium-metrics-self" {
   self      = true
 }
 
-# IANA VXLAN default
-resource "aws_security_group_rule" "worker-vxlan" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.worker.id
-
-  type                     = "ingress"
-  protocol                 = "udp"
-  from_port                = 4789
-  to_port                  = 4789
-  source_security_group_id = aws_security_group.controller.id
-}
-
-resource "aws_security_group_rule" "worker-vxlan-self" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.worker.id
-
-  type      = "ingress"
-  protocol  = "udp"
-  from_port = 4789
-  to_port   = 4789
-  self      = true
-}
-
 # Linux VXLAN default
-resource "aws_security_group_rule" "worker-linux-vxlan" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "worker-vxlan" {
   security_group_id = aws_security_group.worker.id
 
   type                     = "ingress"
@@ -405,9 +349,7 @@ resource "aws_security_group_rule" "worker-linux-vxlan" {
   source_security_group_id = aws_security_group.controller.id
 }
 
-resource "aws_security_group_rule" "worker-linux-vxlan-self" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "worker-vxlan-self" {
   security_group_id = aws_security_group.worker.id
 
   type      = "ingress"

aws/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/flatcar-linux/kubernetes/security.tf

@@ -116,31 +116,6 @@ resource "aws_security_group_rule" "controller-cilium-metrics-self" {
   self      = true
 }
 
-# IANA VXLAN default
-resource "aws_security_group_rule" "controller-vxlan" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.controller.id
-
-  type                     = "ingress"
-  protocol                 = "udp"
-  from_port                = 4789
-  to_port                  = 4789
-  source_security_group_id = aws_security_group.worker.id
-}
-
-resource "aws_security_group_rule" "controller-vxlan-self" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.controller.id
-
-  type      = "ingress"
-  protocol  = "udp"
-  from_port = 4789
-  to_port   = 4789
-  self      = true
-}
-
 resource "aws_security_group_rule" "controller-apiserver" {
   security_group_id = aws_security_group.controller.id
 
@@ -152,9 +127,7 @@ resource "aws_security_group_rule" "controller-apiserver" {
 }
 
 # Linux VXLAN default
-resource "aws_security_group_rule" "controller-linux-vxlan" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "controller-vxlan" {
   security_group_id = aws_security_group.controller.id
 
   type                     = "ingress"
@@ -164,9 +137,7 @@ resource "aws_security_group_rule" "controller-linux-vxlan" {
   source_security_group_id = aws_security_group.worker.id
 }
 
-resource "aws_security_group_rule" "controller-linux-vxlan-self" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "controller-vxlan-self" {
   security_group_id = aws_security_group.controller.id
 
   type      = "ingress"
@@ -367,35 +338,8 @@ resource "aws_security_group_rule" "worker-cilium-metrics-self" {
   self      = true
 }
 
-# IANA VXLAN default
-resource "aws_security_group_rule" "worker-vxlan" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.worker.id
-
-  type                     = "ingress"
-  protocol                 = "udp"
-  from_port                = 4789
-  to_port                  = 4789
-  source_security_group_id = aws_security_group.controller.id
-}
-
-resource "aws_security_group_rule" "worker-vxlan-self" {
-  count = var.networking == "flannel" ? 1 : 0
-
-  security_group_id = aws_security_group.worker.id
-
-  type      = "ingress"
-  protocol  = "udp"
-  from_port = 4789
-  to_port   = 4789
-  self      = true
-}
-
 # Linux VXLAN default
-resource "aws_security_group_rule" "worker-linux-vxlan" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "worker-vxlan" {
   security_group_id = aws_security_group.worker.id
 
   type                     = "ingress"
@@ -405,9 +349,7 @@ resource "aws_security_group_rule" "worker-linux-vxlan" {
   source_security_group_id = aws_security_group.controller.id
 }
 
-resource "aws_security_group_rule" "worker-linux-vxlan-self" {
-  count = var.networking == "cilium" ? 1 : 0
-
+resource "aws_security_group_rule" "worker-vxlan-self" {
   security_group_id = aws_security_group.worker.id
 
   type      = "ingress"

azure/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/fedora-coreos/kubernetes/security.tf

@@ -164,22 +164,6 @@ resource "azurerm_network_security_rule" "controller-vxlan" {
   direction                    = "Inbound"
   protocol                     = "Udp"
   source_port_range            = "*"
-  destination_port_range       = "4789"
-  source_address_prefixes      = local.cluster_subnets[each.key]
-  destination_address_prefixes = local.controller_subnets[each.key]
-}
-
-resource "azurerm_network_security_rule" "controller-linux-vxlan" {
-  for_each = local.controller_subnets
-
-  name                         = "allow-linux-vxlan-${each.key}"
-  resource_group_name          = azurerm_resource_group.cluster.name
-  network_security_group_name  = azurerm_network_security_group.controller.name
-  priority                     = 2022 + (each.key == "ipv4" ? 0 : 1)
-  access                       = "Allow"
-  direction                    = "Inbound"
-  protocol                     = "Udp"
-  source_port_range            = "*"
   destination_port_range       = "8472"
   source_address_prefixes      = local.cluster_subnets[each.key]
   destination_address_prefixes = local.controller_subnets[each.key]
@@ -368,22 +352,6 @@ resource "azurerm_network_security_rule" "worker-vxlan" {
   direction                    = "Inbound"
   protocol                     = "Udp"
   source_port_range            = "*"
-  destination_port_range       = "4789"
-  source_address_prefixes      = local.cluster_subnets[each.key]
-  destination_address_prefixes = local.worker_subnets[each.key]
-}
-
-resource "azurerm_network_security_rule" "worker-linux-vxlan" {
-  for_each = local.worker_subnets
-
-  name                         = "allow-linux-vxlan-${each.key}"
-  resource_group_name          = azurerm_resource_group.cluster.name
-  network_security_group_name  = azurerm_network_security_group.worker.name
-  priority                     = 2018 + (each.key == "ipv4" ? 0 : 1)
-  access                       = "Allow"
-  direction                    = "Inbound"
-  protocol                     = "Udp"
-  source_port_range            = "*"
   destination_port_range       = "8472"
   source_address_prefixes      = local.cluster_subnets[each.key]
   destination_address_prefixes = local.worker_subnets[each.key]

azure/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/flatcar-linux/kubernetes/security.tf

@@ -164,22 +164,6 @@ resource "azurerm_network_security_rule" "controller-vxlan" {
   direction                    = "Inbound"
   protocol                     = "Udp"
   source_port_range            = "*"
-  destination_port_range       = "4789"
-  source_address_prefixes      = local.cluster_subnets[each.key]
-  destination_address_prefixes = local.controller_subnets[each.key]
-}
-
-resource "azurerm_network_security_rule" "controller-linux-vxlan" {
-  for_each = local.controller_subnets
-
-  name                         = "allow-linux-vxlan-${each.key}"
-  resource_group_name          = azurerm_resource_group.cluster.name
-  network_security_group_name  = azurerm_network_security_group.controller.name
-  priority                     = 2022 + (each.key == "ipv4" ? 0 : 1)
-  access                       = "Allow"
-  direction                    = "Inbound"
-  protocol                     = "Udp"
-  source_port_range            = "*"
   destination_port_range       = "8472"
   source_address_prefixes      = local.cluster_subnets[each.key]
   destination_address_prefixes = local.controller_subnets[each.key]
@@ -368,22 +352,6 @@ resource "azurerm_network_security_rule" "worker-vxlan" {
   direction                    = "Inbound"
   protocol                     = "Udp"
   source_port_range            = "*"
-  destination_port_range       = "4789"
-  source_address_prefixes      = local.cluster_subnets[each.key]
-  destination_address_prefixes = local.worker_subnets[each.key]
-}
-
-resource "azurerm_network_security_rule" "worker-linux-vxlan" {
-  for_each = local.worker_subnets
-
-  name                         = "allow-linux-vxlan-${each.key}"
-  resource_group_name          = azurerm_resource_group.cluster.name
-  network_security_group_name  = azurerm_network_security_group.worker.name
-  priority                     = 2018 + (each.key == "ipv4" ? 0 : 1)
-  access                       = "Allow"
-  direction                    = "Inbound"
-  protocol                     = "Udp"
-  source_port_range            = "*"
   destination_port_range       = "8472"
   source_address_prefixes      = local.cluster_subnets[each.key]
   destination_address_prefixes = local.worker_subnets[each.key]

bare-metal/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name = var.cluster_name
   api_servers  = [var.k8s_domain_name]

bare-metal/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name = var.cluster_name
   api_servers  = [var.k8s_domain_name]

digital-ocean/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

digital-ocean/fedora-coreos/kubernetes/network.tf

@@ -39,14 +39,7 @@ resource "digitalocean_firewall" "rules" {
     source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
   }
 
-  # IANA vxlan (flannel)
-  inbound_rule {
-    protocol    = "udp"
-    port_range  = "4789"
-    source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
-  }
-
-  # Linux vxlan (Cilium)
+  # vxlan
   inbound_rule {
     protocol    = "udp"
     port_range  = "8472"

digital-ocean/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

digital-ocean/flatcar-linux/kubernetes/network.tf

@@ -39,14 +39,7 @@ resource "digitalocean_firewall" "rules" {
     source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
   }
 
-  # IANA vxlan (flannel)
-  inbound_rule {
-    protocol    = "udp"
-    port_range  = "4789"
-    source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
-  }
-
-  # Linux vxlan (Cilium)
+  # vxlan
   inbound_rule {
     protocol    = "udp"
     port_range  = "8472"

google-cloud/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

google-cloud/fedora-coreos/kubernetes/network.tf

@@ -75,16 +75,16 @@ resource "google_compute_firewall" "allow-apiserver" {
   target_tags   = ["${var.cluster_name}-controller"]
 }
 
-# flannel VXLAN
-resource "google_compute_firewall" "internal-vxlan" {
+# flannel
+resource "google_compute_firewall" "internal-flannel" {
   count = var.networking == "flannel" ? 1 : 0
 
-  name    = "${var.cluster_name}-internal-vxlan"
+  name    = "${var.cluster_name}-flannel"
   network = google_compute_network.network.name
 
   allow {
     protocol = "udp"
-    ports    = [4789]
+    ports    = [8472]
   }
 
   source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]

google-cloud/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3edb0ae646faaf79406e1bb5cc94038edab32f21"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

google-cloud/flatcar-linux/kubernetes/network.tf

@@ -75,16 +75,16 @@ resource "google_compute_firewall" "allow-apiserver" {
   target_tags   = ["${var.cluster_name}-controller"]
 }
 
-# flannel VXLAN
-resource "google_compute_firewall" "internal-vxlan" {
+# flannel
+resource "google_compute_firewall" "internal-flannel" {
   count = var.networking == "flannel" ? 1 : 0
 
-  name    = "${var.cluster_name}-internal-vxlan"
+  name    = "${var.cluster_name}-flannel"
   network = google_compute_network.network.name
 
   allow {
     protocol = "udp"
-    ports    = [4789]
+    ports    = [8472]
   }
 
   source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]