Add secgroups

openstack/Makefile

@@ -29,6 +29,7 @@ create-templates:
 
 create-kubeconfig:
 	talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.132.11 kubeconfig .
+	kubectl --kubeconfig=kubeconfig get pods -owide -A
 
 create-deployments:
 	helm template --namespace=kube-system   --version=1.11.4 -f deployments/cilium.yaml cilium \

openstack/instances-master.tf

@@ -15,6 +15,7 @@ module "controlplane" {
   instance_count       = lookup(try(var.controlplane[each.key], {}), "count", 0)
   instance_flavor      = lookup(try(var.controlplane[each.key], {}), "instance_type", "d2-2")
   instance_image       = data.openstack_images_image_v2.talos[each.key].id
+  instance_secgroups   = [local.network_secgroup[each.key].common.id, local.network_secgroup[each.key].controlplane.id]
   instance_params = merge(var.kubernetes, {
     lbv4   = local.lbv4
     routes = "\n${join("\n", formatlist("- network: %s", flatten([for zone in local.regions : local.network_subnets[zone] if zone != each.key])))}"
@@ -39,3 +40,7 @@ module "controlplane" {
   network_internal = local.network_public[each.key]
   network_external = local.network_external[each.key]
 }
+
+locals {
+  lbv4s = compact([for c in module.controlplane : c.controlplane_lb])
+}

openstack/instances-web.tf

@@ -1,16 +1,26 @@
 
+resource "openstack_compute_servergroup_v2" "web" {
+  for_each = { for idx, name in local.regions : name => idx }
+  region   = each.key
+  name     = "web"
+  policies = ["soft-anti-affinity"]
+}
+
 module "web" {
   source   = "./modules/worker"
   for_each = { for idx, name in local.regions : name => idx }
   region   = each.key
+
+  instance_servergroup = openstack_compute_servergroup_v2.web[each.key].id
   instance_count       = lookup(try(var.instances[each.key], {}), "web_count", 0)
   instance_name        = "web"
   instance_flavor      = lookup(try(var.instances[each.key], {}), "web_instance_type", 0)
   instance_image       = data.openstack_images_image_v2.talos[each.key].id
+  instance_secgroups   = [local.network_secgroup[each.key].common.id, local.network_secgroup[each.key].web.id]
   instance_params = merge(var.kubernetes, {
     ipv4_local_network = local.network[each.key].cidr
     ipv4_local_gw      = local.network_public[each.key].gateway
-    lbv4               = module.controlplane[each.key].controlplane_lb
+    lbv4               = module.controlplane[each.key].controlplane_lb != "" ? module.controlplane[each.key].controlplane_lb : one(local.lbv4s)
     routes             = "\n${join("\n", formatlist("- network: %s", flatten([for zone in local.regions : local.network_subnets[zone] if zone != each.key])))}"
   })
 

openstack/instances-workers.tf

@@ -3,14 +3,16 @@ module "worker" {
   source   = "./modules/worker"
   for_each = { for idx, name in local.regions : name => idx }
   region   = each.key
+
   instance_count     = lookup(try(var.instances[each.key], {}), "worker_count", 0)
   instance_name      = "worker"
   instance_flavor    = lookup(try(var.instances[each.key], {}), "worker_instance_type", 0)
   instance_image     = data.openstack_images_image_v2.talos[each.key].id
+  instance_secgroups = [local.network_secgroup[each.key].common.id]
   instance_params = merge(var.kubernetes, {
     ipv4_local_network = local.network[each.key].cidr
     ipv4_local_gw      = local.network_private[each.key].gateway
-    lbv4               = module.controlplane[each.key].controlplane_lb
+    lbv4               = module.controlplane[each.key].controlplane_lb != "" ? module.controlplane[each.key].controlplane_lb : one(local.lbv4s)
     routes             = "\n${join("\n", formatlist("- network: %s", flatten([for zone in local.regions : local.network_subnets[zone] if zone != each.key])))}"
   })
 

openstack/modules/controlplane/main.tf

@@ -5,7 +5,6 @@ resource "openstack_networking_port_v2" "controlplane" {
   name           = "controlplane-${lower(var.region)}-${count.index + 1}"
   network_id     = var.network_internal.network_id
   admin_state_up = true
-  # port_security_enabled = false ### FIXME
 
   fixed_ip {
     subnet_id  = var.network_internal.subnet_id
@@ -22,7 +21,8 @@ resource "openstack_networking_port_v2" "controlplane_public" {
   region             = var.region
   name               = "controlplane-${lower(var.region)}-${count.index + 1}"
   network_id         = var.network_external.id
-  admin_state_up = "true"
+  admin_state_up     = true
+  security_group_ids = var.instance_secgroups
 }
 
 resource "openstack_compute_instance_v2" "controlplane" {

openstack/modules/controlplane/outputs.tf

@@ -6,7 +6,7 @@ output "controlplane_lb" {
 
 output "controlplane_endpoints" {
   description = "Kubernetes controlplane endpoint"
-  value       = [for ip in try(openstack_networking_port_v2.controlplane_public[*].all_fixed_ips, []) : ip]
+  value       = flatten([for ip in try(openstack_networking_port_v2.controlplane_public[*].all_fixed_ips, []) : ip])
   depends_on  = [openstack_networking_port_v2.controlplane_public]
 }
 

openstack/modules/controlplane/variables.tf

@@ -33,6 +33,12 @@ variable "instance_image" {
   type        = string
 }
 
+variable "instance_secgroups" {
+  description = "Instance network security groups"
+  type        = list(string)
+  default     = []
+}
+
 variable "instance_params" {
   description = "Instance template parameters"
   type        = map(string)

openstack/modules/worker/main.tf

@@ -6,6 +6,9 @@ resource "openstack_networking_port_v2" "worker" {
   network_id     = var.network_internal.network_id
   admin_state_up = true
 
+  # port_security_enabled = len(var.instance_secgroups) > 0
+  # security_group_ids    = var.instance_secgroups
+
   fixed_ip {
     subnet_id  = var.network_internal.subnet_id
     ip_address = cidrhost(var.network_internal.cidr, var.instance_ip_start + count.index)
@@ -18,6 +21,7 @@ resource "openstack_networking_port_v2" "worker_public" {
   name               = "${var.instance_name}-${lower(var.region)}-${count.index + 1}"
   network_id         = var.network_external.id
   admin_state_up     = true
+  security_group_ids = var.instance_secgroups
 
   dynamic "fixed_ip" {
     for_each = try([var.network_external.subnet], [])
@@ -38,6 +42,12 @@ resource "openstack_compute_instance_v2" "worker" {
   flavor_name = var.instance_flavor
   image_id    = var.instance_image
 
+  scheduler_hints {
+    group = var.instance_servergroup
+  }
+
+  stop_before_destroy = true
+
   user_data = templatefile("${path.module}/../../templates/worker.yaml.tpl",
     merge(var.instance_params, {
       name        = "${var.instance_name}-${lower(var.region)}-${count.index + 1}"

openstack/modules/worker/outputs.tf

@@ -1,5 +1,5 @@
 
 output "worker_endpoints" {
   description = "Kubernetes worker endpoint"
-  value       = [for ip in try(openstack_networking_port_v2.worker[*].all_fixed_ips, []) : ip]
+  value       = flatten([for ip in try(openstack_networking_port_v2.worker_public[*].all_fixed_ips, []) : ip])
 }

openstack/modules/worker/variables.tf

@@ -13,6 +13,12 @@ variable "network_external" {
   default     = {}
 }
 
+variable "instance_servergroup" {
+  description = "Server Group"
+  type        = string
+  default     = ""
+}
+
 variable "instance_count" {
   description = "Instances in region"
   type        = number
@@ -34,6 +40,12 @@ variable "instance_image" {
   type        = string
 }
 
+variable "instance_secgroups" {
+  description = "Instance network security groups"
+  type        = list(string)
+  default     = []
+}
+
 variable "instance_params" {
   description = "Instance template parameters"
   type        = map(string)

openstack/outputs.tf

@@ -3,3 +3,8 @@ output "controlplane_endpoint" {
   description = "Kubernetes controlplane endpoint"
   value       = module.controlplane
 }
+
+output "web_endpoint" {
+  description = "Kubernetes controlplane endpoint"
+  value       = module.web
+}

openstack/prepare/network-secgroup.tf

@@ -1,102 +1,208 @@
 
-# resource "openstack_networking_secgroup_v2" "controlplane" {
+resource "openstack_networking_secgroup_v2" "common" {
-#   count       = length(var.regions)
+  for_each    = { for idx, name in var.regions : name => idx }
-#   region      = element(var.regions, count.index)
+  region      = each.key
-#   name        = "api"
+  name        = "common"
-#   description = "Security group for allowing controlplane access"
+  description = "Security group for all nodes"
-# }
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_icmp_access_ipv4" {
+resource "openstack_networking_secgroup_rule_v2" "common_icmp_ipv4" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv4"
+  direction         = "ingress"
-#   protocol          = "icmp"
+  ethertype         = "IPv4"
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  protocol          = "icmp"
-# }
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_icmp_access_ipv6" {
+resource "openstack_networking_secgroup_rule_v2" "common_icmp_ipv6" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv6"
+  direction         = "ingress"
-#   protocol          = "icmp"
+  ethertype         = "IPv6"
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  protocol          = "ipv6-icmp"
-# }
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_ssh_access_ipv4" {
+resource "openstack_networking_secgroup_rule_v2" "common_talos_ipv4" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv4"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv4"
-#   port_range_min    = 22
+  protocol          = "tcp"
-#   port_range_max    = 22
+  port_range_min    = 50000
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 50001
-# }
+  remote_ip_prefix  = var.network_cidr
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_talos_access_ipv4" {
+resource "openstack_networking_secgroup_rule_v2" "common_talos_ipv6" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv4"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv6"
-#   port_range_min    = 50000
+  protocol          = "tcp"
-#   port_range_max    = 50000
+  port_range_min    = 50000
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 50001
-# }
+  remote_ip_prefix  = local.network_cidr_v6
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_etcd_access_ipv4" {
+resource "openstack_networking_secgroup_rule_v2" "common_kubelet_ipv4" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv4"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv4"
-#   port_range_min    = 2379
+  protocol          = "tcp"
-#   port_range_max    = 2380
+  port_range_min    = 10250
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 10250
-# }
+  remote_ip_prefix  = var.network_cidr
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_kubernetes_access_ipv4" {
+resource "openstack_networking_secgroup_rule_v2" "common_kubelet_ipv6" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv4"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv6"
-#   port_range_min    = 6443
+  protocol          = "tcp"
-#   port_range_max    = 6443
+  port_range_min    = 10250
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 10250
-# }
+  remote_ip_prefix  = local.network_cidr_v6
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_kubernetes_access_ipv6" {
+resource "openstack_networking_secgroup_rule_v2" "common_cilium_health_ipv4" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv6"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv4"
-#   port_range_min    = 6443
+  protocol          = "tcp"
-#   port_range_max    = 6443
+  port_range_min    = 4240
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 4240
-# }
+  remote_ip_prefix  = var.network_cidr
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_cilium_health_access_ipv4" {
+resource "openstack_networking_secgroup_rule_v2" "common_cilium_health_ipv6" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv4"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv6"
-#   port_range_min    = 4240
+  protocol          = "tcp"
-#   port_range_max    = 4240
+  port_range_min    = 4240
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 4240
-# }
+  remote_ip_prefix  = local.network_cidr_v6
+}
 
-# resource "openstack_networking_secgroup_rule_v2" "controlplane_cilium_health_access_ipv6" {
+resource "openstack_networking_secgroup_rule_v2" "common_cilium_vxvlan" {
-#   count             = length(var.regions)
+  for_each          = { for idx, name in var.regions : name => idx }
-#   region            = element(var.regions, count.index)
+  region            = each.key
-#   direction         = "ingress"
+  security_group_id = openstack_networking_secgroup_v2.common[each.key].id
-#   ethertype         = "IPv6"
+  direction         = "ingress"
-#   protocol          = "tcp"
+  ethertype         = "IPv4"
-#   port_range_min    = 4240
+  protocol          = "udp"
-#   port_range_max    = 4240
+  port_range_min    = 8472
-#   security_group_id = openstack_networking_secgroup_v2.controlplane[count.index].id
+  port_range_max    = 8472
-# }
+  remote_ip_prefix  = var.network_cidr
+}
+
+### Controlplane
+
+resource "openstack_networking_secgroup_v2" "controlplane" {
+  for_each    = { for idx, name in var.regions : name => idx }
+  region      = each.key
+  name        = "controlplane"
+  description = "Security group for controlplane"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "controlplane_talos_admins" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.controlplane[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 50000
+  port_range_max    = 50000
+  remote_ip_prefix  = var.whitelist_admins[0]
+}
+
+resource "openstack_networking_secgroup_rule_v2" "controlplane_etcd_ipv4" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.controlplane[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 2379
+  port_range_max    = 2380
+}
+
+resource "openstack_networking_secgroup_rule_v2" "controlplane_kubernetes_ipv4" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.controlplane[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 6443
+  port_range_max    = 6443
+  remote_ip_prefix  = var.network_cidr
+}
+
+resource "openstack_networking_secgroup_rule_v2" "controlplane_kubernetes_ipv6" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.controlplane[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv6"
+  protocol          = "tcp"
+  port_range_min    = 6443
+  port_range_max    = 6443
+  remote_ip_prefix  = local.network_cidr_v6
+}
+
+resource "openstack_networking_secgroup_rule_v2" "controlplane_kubernetes_admins" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.controlplane[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 6443
+  port_range_max    = 6443
+  remote_ip_prefix  = var.whitelist_admins[0]
+}
+
+### Web
+
+resource "openstack_networking_secgroup_v2" "web" {
+  for_each    = { for idx, name in var.regions : name => idx }
+  region      = each.key
+  name        = "web"
+  description = "Security group for web"
+}
+
+resource "openstack_networking_secgroup_rule_v2" "web_http_v4" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.web[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 80
+  port_range_max    = 80
+}
+
+resource "openstack_networking_secgroup_rule_v2" "web_https_v4" {
+  for_each          = { for idx, name in var.regions : name => idx }
+  region            = each.key
+  security_group_id = openstack_networking_secgroup_v2.web[each.key].id
+  direction         = "ingress"
+  ethertype         = "IPv4"
+  protocol          = "tcp"
+  port_range_min    = 443
+  port_range_max    = 443
+}

openstack/prepare/network.tf

@@ -15,6 +15,7 @@ data "openstack_networking_network_v2" "main" {
 
 locals {
   network_id      = data.openstack_networking_network_v2.main
+  network_cidr_v6 = "fd60:${replace(cidrhost(var.network_cidr, 1), ".", ":")}::/56"
 }
 
 resource "openstack_networking_subnet_v2" "public" {
@@ -51,7 +52,7 @@ resource "openstack_networking_subnet_v2" "private_v6" {
   region            = each.key
   name              = "private-v6"
   network_id        = local.network_id[each.key].id
-  cidr              = cidrsubnet("fd60:${replace(cidrhost(var.network_cidr, 1), ".", ":")}::/56", 8, 4 * (var.network_shift + each.value))
+  cidr              = cidrsubnet(local.network_cidr_v6, 8, 1 + 4 * (var.network_shift + each.value))
   no_gateway        = true
   ip_version        = 6
   ipv6_address_mode = "slaac" # dhcpv6-stateless dhcpv6-stateful # slaac

openstack/prepare/outputs.tf

@@ -45,3 +45,12 @@ output "network_private" {
     mtu        = local.network_id[zone].mtu
   } }
 }
+
+output "network_secgroup" {
+  description = "The Network Security Groups"
+  value = { for idx, zone in var.regions : zone => {
+    common       = openstack_networking_secgroup_v2.common[zone]
+    controlplane = openstack_networking_secgroup_v2.controlplane[zone]
+    web          = openstack_networking_secgroup_v2.web[zone]
+  } }
+}

openstack/prepare/variables.tf

@@ -27,6 +27,11 @@ variable "network_cidr" {
   default     = "172.16.0.0/16"
 }
 
+variable "whitelist_admins" {
+  description = "Whitelist for administrators"
+  default     = ["0.0.0.0/0", "::/0"]
+}
+
 variable "network_shift" {
   description = "Network number shift"
   type        = number

openstack/variables.tf

@@ -14,6 +14,8 @@ locals {
   network_public  = data.terraform_remote_state.prepare.outputs.network_public
   network_private = data.terraform_remote_state.prepare.outputs.network_private
   network_subnets = { for zone in local.regions : zone => [local.network_public[zone].cidr, local.network_private[zone].cidr] }
+
+  network_secgroup = data.terraform_remote_state.prepare.outputs.network_secgroup
 }
 
 variable "ccm_username" {
@@ -61,7 +63,7 @@ variable "instances" {
   description = "Map of region's properties"
   type        = map(any)
   default = {
-    "GRA9" = {
+    "REGION" = {
       web_count            = 0,
       web_instance_type    = "d2-2",
       worker_count         = 0,
@@ -69,29 +71,3 @@ variable "instances" {
     },
   }
 }
-
-variable "whitelist_admins" {
-  description = "Whitelist for administrators"
-  default     = ["0.0.0.0/0", "::/0"]
-}
-
-variable "whitelist_web" {
-  description = "Whitelist for web (default Cloudflare network)"
-  default = [
-    "173.245.48.0/20",
-    "103.21.244.0/22",
-    "103.22.200.0/22",
-    "103.31.4.0/22",
-    "141.101.64.0/18",
-    "108.162.192.0/18",
-    "190.93.240.0/20",
-    "188.114.96.0/20",
-    "197.234.240.0/22",
-    "198.41.128.0/17",
-    "162.158.0.0/15",
-    "172.64.0.0/13",
-    "131.0.72.0/22",
-    "104.16.0.0/13",
-    "104.24.0.0/14",
-  ]
-}