integration fixes

azure/prepare/network-gw.tf

@@ -50,7 +50,7 @@ resource "azurerm_network_interface" "router" {
 resource "azurerm_network_interface_security_group_association" "router" {
   for_each                  = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
   network_interface_id      = azurerm_network_interface.router[each.key].id
-  network_security_group_id = azurerm_network_security_group.gateway[each.key].id
+  network_security_group_id = azurerm_network_security_group.router[each.key].id
 }
 
 resource "azurerm_linux_virtual_machine" "router" {

azure/prepare/network.tf

@@ -76,15 +76,18 @@ resource "azurerm_route_table" "link" {
       name                   = "link-${each.key}-v${length(split(".", var.network_cidr[route.value])) > 1 ? "4" : "6"}"
       address_prefix         = var.network_cidr[route.value]
       next_hop_type          = "VirtualAppliance"
-      next_hop_in_ip_address = cidrhost(azurerm_subnet.public[each.key].address_prefixes[route.value], -2)
+      next_hop_in_ip_address = azurerm_network_interface.router[each.key].private_ip_addresses[route.value]
     }
   }
+  dynamic "route" {
+    for_each = [for ip in azurerm_network_interface.router[each.key].private_ip_addresses : ip if length(split(".", ip)) == 1]
 
-  route {
-    name                   = "link-${each.key}-default-v6"
-    address_prefix         = "::/0"
-    next_hop_type          = "VirtualAppliance"
-    next_hop_in_ip_address = cidrhost([for ip in azurerm_subnet.public[each.key].address_prefixes : ip if length(split(".", ip)) == 1][0], -2)
+    content {
+      name                   = "link-${each.key}-default-v6"
+      address_prefix         = "::/0"
+      next_hop_type          = "VirtualAppliance"
+      next_hop_in_ip_address = route.value
+    }
   }
 
   tags = merge(var.tags, { type = "infra" })

azure/prepare/outputs.tf

@@ -17,8 +17,9 @@ output "resource_group" {
 output "network" {
   description = "The network"
   value = { for zone, net in azurerm_virtual_network.main : zone => {
-    name = net.name
-    nat  = try(azurerm_public_ip.nat[zone].ip_address, "")
+    name    = net.name
+    nat     = try(azurerm_public_ip.nat[zone].ip_address, "")
+    peering = try(azurerm_linux_virtual_machine.router[zone].private_ip_addresses, [])
   } }
 }
 

azure/prepare/secgroup-gw.tf

@@ -1,8 +1,8 @@
 
-resource "azurerm_network_security_group" "gateway" {
+resource "azurerm_network_security_group" "router" {
   for_each            = { for idx, name in var.regions : name => idx }
   location            = each.key
-  name                = "gateway-${each.key}"
+  name                = "router-${each.key}"
   resource_group_name = var.resource_group
 
   dynamic "security_rule" {
@@ -47,5 +47,20 @@ resource "azurerm_network_security_group" "gateway" {
     destination_address_prefix = "*"
   }
 
+  dynamic "security_rule" {
+    for_each = var.network_cidr
+    content {
+      name                       = "Peering-${security_rule.key}"
+      priority                   = 1700 + security_rule.key
+      direction                  = "Inbound"
+      access                     = "Allow"
+      protocol                   = "*"
+      source_port_range          = "*"
+      source_address_prefix      = security_rule.value
+      destination_port_range     = "*"
+      destination_address_prefix = security_rule.value
+    }
+  }
+
   tags = merge(var.tags, { type = "infra" })
 }

openstack/deployments/coredns-local.yaml

@@ -18,7 +18,6 @@ data:
   hosts: |
     # static hosts
     169.254.2.53        dns.local
-    fd00::169:254:2:53  dns.local
 
   Corefile.local: |
     (empty) {
@@ -27,7 +26,7 @@ data:
 
     .:53 {
         errors
-        bind 169.254.2.53 fd00::169:254:2:53
+        bind 169.254.2.53
 
         health 127.0.0.1:8091 {
            lameduck 5s

openstack/modules/controlplane/main.tf

@@ -73,7 +73,7 @@ resource "local_file" "controlplane" {
       ipv4 = [for k in openstack_networking_port_v2.controlplane_public[count.index].all_fixed_ips : k if length(regexall("[0-9]+.[0-9.]+", k)) > 0][0]
       ipv6 = [for k in openstack_networking_port_v2.controlplane_public[count.index].all_fixed_ips : k if length(regexall("[0-9a-z]+:[0-9a-z:]+", k)) > 0][0]
 
-      nodeSubnets = var.network_internal.cidr
+      nodeSubnets = split(",", var.network_internal.cidr)
     })
   )
   filename        = "_cfgs/controlplane-${lower(var.region)}-${count.index + 1}.yaml"

openstack/prepare/network-gw.tf

@@ -66,58 +66,58 @@ resource "openstack_networking_router_interface_v2" "private" {
   # port_id = openstack_networking_port_v2.gw_private[each.key].id
 }
 
-### Soft gateway
+### Soft gateway, peering networks
 
-# resource "openstack_compute_instance_v2" "gw" {
-#   for_each    = { for idx, name in var.regions : name => idx if try(var.capabilities[name].gateway, false) == false }
-#   region      = each.key
-#   name        = "gw-${lower(each.key)}"
-#   image_id    = data.openstack_images_image_v2.debian[each.key].id
-#   flavor_name = "d2-2"
-#   key_pair    = openstack_compute_keypair_v2.keypair[each.key].name
+resource "openstack_compute_instance_v2" "gw" {
+  for_each    = { for idx, name in var.regions : name => idx if try(var.capabilities[name].peering, false) }
+  region      = each.key
+  name        = "gw-${lower(each.key)}"
+  image_id    = data.openstack_images_image_v2.debian[each.key].id
+  flavor_name = "d2-2"
+  key_pair    = openstack_compute_keypair_v2.keypair[each.key].name
 
-#   network {
-#     port           = openstack_networking_port_v2.gw_external[each.key].id
-#     uuid           = data.openstack_networking_network_v2.external[each.key].id
-#     access_network = true
-#   }
-#   network {
-#     port = openstack_networking_port_v2.gw[each.key].id
-#   }
+  network {
+    port           = openstack_networking_port_v2.gw_external[each.key].id
+    uuid           = data.openstack_networking_network_v2.external[each.key].id
+    access_network = true
+  }
+  network {
+    port = openstack_networking_port_v2.gw_private[each.key].id
+  }
 
-#   user_data = <<EOF
-# #cloud-config
-# apt_update: true
-# apt_upgrade: true
-# disable_root: false
-# write_files:
-#   - path: /etc/network/interfaces
-#     permissions: '0644'
-#     content: |
-#       auto lo
-#       iface lo inet loopback
-#         dns-nameservers 1.1.1.1 8.8.8.8
-#       iface lo inet6 loopback
+  user_data = <<EOF
+#cloud-config
+apt_update: true
+apt_upgrade: true
+disable_root: false
+write_files:
+  - path: /etc/network/interfaces
+    permissions: '0644'
+    content: |
+      auto lo
+      iface lo inet loopback
+      iface lo inet6 loopback
 
-#       allow-hotplug ens3
-#       iface ens3 inet dhcp
-#         mtu 1500
-#       iface ens3 inet6 static
-#         address ${[for ip in openstack_networking_port_v2.gw_external[each.key].all_fixed_ips : ip if length(regexall("[0-9a-z]+:[0-9a-z:]+", ip)) > 0][0]}
-#         gateway ${cidrhost("${[for ip in openstack_networking_port_v2.gw_external[each.key].all_fixed_ips : ip if length(regexall("[0-9a-z]+:[0-9a-z:]+", ip)) > 0][0]}/56", 1)}
-#         netmask 56
+      allow-hotplug ens3
+      iface ens3 inet dhcp
+        mtu 1500
+      iface ens3 inet6 static
+        address ${[for ip in openstack_networking_port_v2.gw_external[each.key].all_fixed_ips : ip if length(split(":", ip)) > 1][0]}
+        gateway ${cidrhost("${[for ip in openstack_networking_port_v2.gw_external[each.key].all_fixed_ips : ip if length(split(":", ip)) > 1][0]}/56", 1)}
+        netmask 56
 
-#       allow-hotplug ens4
-#       iface ens4 inet static
-#         address ${openstack_networking_port_v2.gw[each.key].all_fixed_ips[0]}
-#         netmask 24
-#         mtu ${local.network_id[each.key].mtu}
+      allow-hotplug ens4
+      iface ens4 inet static
+        address ${[for ip in openstack_networking_port_v2.gw_private[each.key].all_fixed_ips : ip if length(split(".", ip)) > 1][0]}
+        netmask 24
+        mtu ${local.network_id[each.key].mtu}
+        post-up ip ro add ${openstack_networking_subnet_v2.public[each.key].cidr} dev ens4
 
-# runcmd:
-#   - rm -f /etc/network/interfaces.d/50-cloud-init
-# EOF
+runcmd:
+  - rm -f /etc/network/interfaces.d/50-cloud-init
+EOF
 
-#   lifecycle {
-#     ignore_changes = [key_pair, user_data, image_id]
-#   }
-# }
+  lifecycle {
+    ignore_changes = [key_pair, user_data, image_id]
+  }
+}

openstack/prepare/variables.tf

@@ -43,9 +43,11 @@ variable "capabilities" {
   default = {
     "GRA7" = {
       gateway = false
+      peering = false
     },
     "GRA9" = {
-      gateway = true
+      gateway = false
+      peering = true
     },
   }
 }

openstack/templates/controlplane.yaml.tpl

@@ -16,10 +16,9 @@ machine:
       node-labels: "${labels}"
       rotate-server-certificates: true
     nodeIP:
-      validSubnets: ${format("%#v",split(",",nodeSubnets))}
+      validSubnets: ${format("%#v",nodeSubnets)}
     clusterDNS:
       - 169.254.2.53
-      - fd00::169:254:2:53
       - ${cidrhost(split(",",serviceSubnets)[0], 10)}
   network:
     hostname: "${name}"
@@ -33,7 +32,6 @@ machine:
       - interface: dummy0
         addresses:
           - 169.254.2.53/32
-          - fd00::169:254:2:53/128
     extraHostEntries:
       - ip: ${ipv4_local_vip}
         aliases:
@@ -69,7 +67,8 @@ cluster:
         node-cidr-mask-size-ipv4: 24
         node-cidr-mask-size-ipv6: 112
   scheduler: {}
-  etcd: {}
+  etcd:
+    subnet: ${nodeSubnets[0]}
   inlineManifests:
     - name: openstack-cloud-controller-config
       contents: |-