diff --git a/openstack/deployments/openstack-cloud-controller-manager.yaml b/openstack/deployments/openstack-cloud-controller-manager.yaml index c3cba5e..a7ba1a3 100644 --- a/openstack/deployments/openstack-cloud-controller-manager.yaml +++ b/openstack/deployments/openstack-cloud-controller-manager.yaml @@ -160,7 +160,7 @@ spec: serviceAccountName: openstack-cloud-controller-manager containers: - name: cloud-controller-manager - image: ghcr.io/sergelogvinov/openstack-cloud-controller-manager-amd64:v1.17.0-465-gc47f994 + image: ghcr.io/sergelogvinov/openstack-cloud-controller-manager:v1.17.0-465-gc47f994 command: - /bin/openstack-cloud-controller-manager args: diff --git a/openstack/instances-controlplane.tf b/openstack/instances-controlplane.tf index 15bddd6..e8346af 100644 --- a/openstack/instances-controlplane.tf +++ b/openstack/instances-controlplane.tf @@ -16,7 +16,7 @@ module "controlplane" { instance_flavor = lookup(try(var.controlplane[each.key], {}), "instance_type", "d2-2") instance_image = data.openstack_images_image_v2.talos[each.key].id instance_tags = concat(var.tags, ["infra"]) - instance_secgroups = [local.network_secgroup[each.key].common.id, local.network_secgroup[each.key].controlplane.id] + instance_secgroups = [local.network_secgroup[each.key].common, local.network_secgroup[each.key].controlplane] instance_params = merge(var.kubernetes, { lbv4 = local.lbv4 routes = "\n${join("\n", formatlist("- network: %s", flatten([for zone in local.regions : local.network_subnets[zone] if zone != each.key])))}" diff --git a/openstack/instances-web.tf b/openstack/instances-web.tf index 83cc310..533851d 100644 --- a/openstack/instances-web.tf +++ b/openstack/instances-web.tf @@ -17,7 +17,7 @@ module "web" { instance_flavor = lookup(try(var.instances[each.key], {}), "web_instance_type", 0) instance_image = data.openstack_images_image_v2.talos[each.key].id instance_tags = concat(var.tags, ["web"]) - instance_secgroups = [local.network_secgroup[each.key].common.id, local.network_secgroup[each.key].web.id] + instance_secgroups = [local.network_secgroup[each.key].common, local.network_secgroup[each.key].web] instance_params = merge(var.kubernetes, { ipv4_local_network = local.network[each.key].cidr ipv4_local_gw = local.network_public[each.key].gateway diff --git a/openstack/instances-workers.tf b/openstack/instances-workers.tf index 6f3e1cb..4376291 100644 --- a/openstack/instances-workers.tf +++ b/openstack/instances-workers.tf @@ -9,7 +9,7 @@ module "worker" { instance_flavor = lookup(try(var.instances[each.key], {}), "worker_instance_type", 0) instance_image = data.openstack_images_image_v2.talos[each.key].id instance_tags = concat(var.tags, ["worker"]) - instance_secgroups = [local.network_secgroup[each.key].common.id] + instance_secgroups = [local.network_secgroup[each.key].common] instance_params = merge(var.kubernetes, { ipv4_local_network = local.network[each.key].cidr ipv4_local_gw = local.network_private[each.key].gateway diff --git a/openstack/prepare/network-gw.tf b/openstack/prepare/network-gw.tf index ea3863c..46a339c 100644 --- a/openstack/prepare/network-gw.tf +++ b/openstack/prepare/network-gw.tf @@ -13,76 +13,85 @@ data "openstack_networking_subnet_ids_v2" "external_v6" { ip_version = 6 } -resource "openstack_networking_router_v2" "gw" { +# resource "openstack_networking_port_v2" "nat" { +# for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) } +# region = each.key +# name = "nat-${lower(each.key)}-${openstack_networking_subnet_v2.private[each.key].name}" +# network_id = data.openstack_networking_network_v2.external[each.key].id +# admin_state_up = "true" +# } + +resource "openstack_networking_router_v2" "nat" { for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) } region = each.key - name = openstack_networking_subnet_v2.private[each.key].name + name = "nat-${openstack_networking_subnet_v2.private[each.key].name}" external_network_id = data.openstack_networking_network_v2.external[each.key].id admin_state_up = true # external_fixed_ip { # subnet_id = data.openstack_networking_network_v2.external[each.key].id - # ip_address = [for k in openstack_networking_port_v2.gw_external[each.key].all_fixed_ips : k if length(regexall("[0-9.]+", k)) > 0][0] + # ip_address = [for ip in openstack_networking_port_v2.nat[each.key].all_fixed_ips : ip if length(split(".", ip)) > 1][0] # } } -resource "openstack_networking_port_v2" "gw_external" { - for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) == false } - region = each.key - name = "gw-${lower(each.key)}-${openstack_networking_subnet_v2.private[each.key].name}" - network_id = data.openstack_networking_network_v2.external[each.key].id - admin_state_up = "true" -} - -resource "openstack_networking_port_v2" "gw_public" { - for_each = { for idx, name in var.regions : name => idx } - region = each.key - name = "gw-${lower(each.key)}-${openstack_networking_subnet_v2.public[each.key].name}" - network_id = local.network_id[each.key].id - admin_state_up = "true" - fixed_ip { - subnet_id = openstack_networking_subnet_v2.public[each.key].id - ip_address = cidrhost(openstack_networking_subnet_v2.public[each.key].cidr, 1) - } -} - -resource "openstack_networking_port_v2" "gw_private" { - for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) == false } - region = each.key - name = "gw-${lower(each.key)}-${openstack_networking_subnet_v2.private[each.key].name}" - network_id = local.network_id[each.key].id - admin_state_up = "true" - fixed_ip { - subnet_id = openstack_networking_subnet_v2.private[each.key].id - ip_address = cidrhost(openstack_networking_subnet_v2.private[each.key].cidr, 1) - } -} +# resource "openstack_networking_port_v2" "gw_external" { +# for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) == false } +# region = each.key +# name = "gw-${lower(each.key)}-${openstack_networking_subnet_v2.private[each.key].name}" +# network_id = data.openstack_networking_network_v2.external[each.key].id +# admin_state_up = "true" +# } resource "openstack_networking_router_interface_v2" "private" { for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) } region = each.key - router_id = openstack_networking_router_v2.gw[each.key].id + router_id = openstack_networking_router_v2.nat[each.key].id subnet_id = openstack_networking_subnet_v2.private[each.key].id # port_id = openstack_networking_port_v2.gw_private[each.key].id } -### Soft gateway, peering networks +### Soft router to peering networks -resource "openstack_compute_instance_v2" "gw" { +resource "openstack_networking_port_v2" "router_external" { + for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) } + region = each.key + name = "router-${lower(each.key)}-${openstack_networking_subnet_v2.private[each.key].name}" + network_id = data.openstack_networking_network_v2.external[each.key].id + security_group_ids = [openstack_networking_secgroup_v2.router[each.key].id] + admin_state_up = "true" +} + +resource "openstack_networking_port_v2" "router" { + for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) } + region = each.key + name = "router-${lower(each.key)}-${openstack_networking_subnet_v2.private[each.key].name}" + network_id = local.network_id[each.key].id + admin_state_up = "true" + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private[each.key].id + ip_address = cidrhost(openstack_networking_subnet_v2.private[each.key].cidr, try(var.capabilities[each.key].gateway, false) ? 2 : 1) + } + # fixed_ip { + # subnet_id = openstack_networking_subnet_v2.private_v6[each.key].id + # ip_address = cidrhost(openstack_networking_subnet_v2.private_v6[each.key].cidr, 1) + # } +} + +resource "openstack_compute_instance_v2" "router" { for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) } region = each.key - name = "gw-${lower(each.key)}" + name = "router-${lower(each.key)}" image_id = data.openstack_images_image_v2.debian[each.key].id flavor_name = "d2-2" key_pair = openstack_compute_keypair_v2.keypair[each.key].name network { - port = openstack_networking_port_v2.gw_external[each.key].id - uuid = data.openstack_networking_network_v2.external[each.key].id + port = openstack_networking_port_v2.router_external[each.key].id + uuid = openstack_networking_port_v2.router_external[each.key].network_id access_network = true } network { - port = openstack_networking_port_v2.gw_private[each.key].id + port = openstack_networking_port_v2.router[each.key].id } user_data = < 1][0]} - gateway ${cidrhost("${[for ip in openstack_networking_port_v2.gw_external[each.key].all_fixed_ips : ip if length(split(":", ip)) > 1][0]}/56", 1)} + address ${[for ip in openstack_networking_port_v2.router_external[each.key].all_fixed_ips : ip if length(split(":", ip)) > 1][0]} + gateway ${cidrhost("${[for ip in openstack_networking_port_v2.router_external[each.key].all_fixed_ips : ip if length(split(":", ip)) > 1][0]}/56", 1)} netmask 56 allow-hotplug ens4 iface ens4 inet static - address ${[for ip in openstack_networking_port_v2.gw_private[each.key].all_fixed_ips : ip if length(split(".", ip)) > 1][0]} + address ${[for ip in openstack_networking_port_v2.router[each.key].all_fixed_ips : ip if length(split(".", ip)) > 1][0]} netmask 24 mtu ${local.network_id[each.key].mtu} post-up ip ro add ${openstack_networking_subnet_v2.public[each.key].cidr} dev ens4 + iface ens4 inet6 static + address ${cidrhost(openstack_networking_subnet_v2.private_v6[each.key].cidr, 1)} + netmask 64 runcmd: - rm -f /etc/network/interfaces.d/50-cloud-init + - reboot EOF lifecycle { diff --git a/openstack/prepare/network-secgroup.tf b/openstack/prepare/network-secgroup.tf index ccc1561..b351ef1 100644 --- a/openstack/prepare/network-secgroup.tf +++ b/openstack/prepare/network-secgroup.tf @@ -206,3 +206,45 @@ resource "openstack_networking_secgroup_rule_v2" "web_https_v4" { port_range_min = 443 port_range_max = 443 } + +### + +resource "openstack_networking_secgroup_v2" "router" { + for_each = { for idx, name in var.regions : name => idx } + region = each.key + name = "router" + description = "Security group for router/peering node" +} + +resource "openstack_networking_secgroup_rule_v2" "router_ssh_v4" { + for_each = { for idx, name in var.regions : name => idx } + region = each.key + security_group_id = openstack_networking_secgroup_v2.router[each.key].id + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = 22 + port_range_max = 22 +} + +resource "openstack_networking_secgroup_rule_v2" "router_ssh_v6" { + for_each = { for idx, name in var.regions : name => idx } + region = each.key + security_group_id = openstack_networking_secgroup_v2.router[each.key].id + direction = "ingress" + ethertype = "IPv6" + protocol = "tcp" + port_range_min = 22 + port_range_max = 22 +} + +resource "openstack_networking_secgroup_rule_v2" "router_wireguard" { + for_each = { for idx, name in var.regions : name => idx } + region = each.key + security_group_id = openstack_networking_secgroup_v2.router[each.key].id + direction = "ingress" + ethertype = "IPv4" + protocol = "udp" + port_range_min = 443 + port_range_max = 443 +} diff --git a/openstack/prepare/network.tf b/openstack/prepare/network.tf index 8c926d4..865a221 100644 --- a/openstack/prepare/network.tf +++ b/openstack/prepare/network.tf @@ -15,7 +15,7 @@ data "openstack_networking_network_v2" "main" { locals { network_id = data.openstack_networking_network_v2.main - network_cidr_v6 = "fd60:${replace(cidrhost(var.network_cidr, 1), ".", ":")}::/56" + network_cidr_v6 = cidrsubnet("fd60:${replace(cidrhost(var.network_cidr, 0), ".", ":")}::/56", 0, 0) } resource "openstack_networking_subnet_v2" "public" { @@ -59,16 +59,23 @@ resource "openstack_networking_subnet_v2" "private_v6" { # ipv6_ra_mode = "slaac" # dhcpv6-stateless dhcpv6-stateful } -resource "openstack_networking_subnet_route_v2" "public" { - for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) } +resource "openstack_networking_subnet_route_v2" "public_v4" { + for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) } subnet_id = openstack_networking_subnet_v2.public[each.key].id destination_cidr = var.network_cidr - next_hop = cidrhost(openstack_networking_subnet_v2.public[each.key].cidr, 1) + next_hop = try(var.capabilities[each.key].gateway, false) ? cidrhost(openstack_networking_subnet_v2.private[each.key].cidr, 2) : cidrhost(openstack_networking_subnet_v2.private[each.key].cidr, 1) } -resource "openstack_networking_subnet_route_v2" "private" { - for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) } +resource "openstack_networking_subnet_route_v2" "private_v4" { + for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) } subnet_id = openstack_networking_subnet_v2.private[each.key].id destination_cidr = var.network_cidr - next_hop = openstack_networking_subnet_v2.private[each.key].gateway_ip + next_hop = try(var.capabilities[each.key].gateway, false) ? cidrhost(openstack_networking_subnet_v2.private[each.key].cidr, 2) : cidrhost(openstack_networking_subnet_v2.private[each.key].cidr, 1) +} + +resource "openstack_networking_subnet_route_v2" "private_v6" { + for_each = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) } + subnet_id = openstack_networking_subnet_v2.private_v6[each.key].id + destination_cidr = local.network_cidr_v6 + next_hop = cidrhost(openstack_networking_subnet_v2.private_v6[each.key].cidr, 1) } diff --git a/openstack/prepare/outputs.tf b/openstack/prepare/outputs.tf index 80c800a..85561a8 100644 --- a/openstack/prepare/outputs.tf +++ b/openstack/prepare/outputs.tf @@ -6,10 +6,11 @@ output "regions" { output "network" { value = { for zone, network in local.network_id : zone => { - name = var.network_name - id = network.id - cidr = var.network_cidr - mtu = network.mtu + name = var.network_name + id = network.id + cidr = var.network_cidr + cidr_v6 = local.network_cidr_v6 + mtu = network.mtu } } } @@ -30,6 +31,7 @@ output "network_public" { network_id = subnet.network_id subnet_id = subnet.id cidr = subnet.cidr + cidr_v6 = openstack_networking_subnet_v2.private_v6[zone].cidr gateway = subnet.gateway_ip != "" ? subnet.gateway_ip : cidrhost(subnet.cidr, 1) mtu = local.network_id[zone].mtu } } @@ -41,6 +43,7 @@ output "network_private" { network_id = subnet.network_id subnet_id = subnet.id cidr = subnet.cidr + cidr_v6 = openstack_networking_subnet_v2.private_v6[zone].cidr gateway = subnet.gateway_ip != "" ? subnet.gateway_ip : cidrhost(subnet.cidr, 1) mtu = local.network_id[zone].mtu } } @@ -49,8 +52,8 @@ output "network_private" { output "network_secgroup" { description = "The Network Security Groups" value = { for idx, zone in var.regions : zone => { - common = openstack_networking_secgroup_v2.common[zone] - controlplane = openstack_networking_secgroup_v2.controlplane[zone] - web = openstack_networking_secgroup_v2.web[zone] + common = openstack_networking_secgroup_v2.common[zone].id + controlplane = openstack_networking_secgroup_v2.controlplane[zone].id + web = openstack_networking_secgroup_v2.web[zone].id } } } diff --git a/openstack/prepare/variables.tf b/openstack/prepare/variables.tf index 0ee5749..d047ba4 100644 --- a/openstack/prepare/variables.tf +++ b/openstack/prepare/variables.tf @@ -47,7 +47,7 @@ variable "capabilities" { }, "GRA9" = { gateway = false - peering = true + peering = false }, } }