Split network, create rbac roles

azure/.gitignore

@@ -1,3 +1,3 @@
 _cfgs/
 *.yaml
-azure.json
+*.json

azure/README.md

@@ -0,0 +1,39 @@
+# Talos on Azure Cloud
+
+## Create IAM roles
+
+### Create roles
+
+Terraform will create the roles.
+```az ad sp``` creates the accounts and assign the roles.
+Do not forget to save account credits.
+
+```shell
+cd init
+terraform init
+terraform apply
+
+az ad sp create-for-rbac --name "kubernetes-ccm" --role kubernetes-ccm --scopes="/subscriptions/<subscription-id>" --output json
+az ad sp create-for-rbac --name "kubernetes-csi" --role kubernetes-csi --scopes="/subscriptions/<subscription-id>" --output json
+```
+
+## Local utilities
+
+* terraform
+* talosctl
+* kubectl
+* yq
+
+# Network diagram
+
+
+## Kubernetes addons
+
+* [Azure CCM](https://github.com/kubernetes-sigs/cloud-provider-azure)
+* [Azure CSI](https://github.com/kubernetes-sigs/azuredisk-csi-driver)
+* [Azure Node AutoScaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/azure/README.md)
+* [cilium](https://github.com/cilium/cilium) 1.11.5
+* [kubelet-serving-cert-approver](https://github.com/alex1989hu/kubelet-serving-cert-approver)
+* [metrics-server](https://github.com/kubernetes-sigs/metrics-server) 0.5.0
+* [rancher.io/local-path](https://github.com/rancher/local-path-provisioner) 0.0.19
+* [ingress-nginx](https://kubernetes.github.io/ingress-nginx/) 4.1.2

azure/common.tf

@@ -8,8 +8,17 @@
 data "azurerm_shared_image_version" "talos" {
   name                = "latest"
   image_name          = "talos"
-  gallery_name        = "293f5f4eea925204"
+  gallery_name        = var.gallery_name
   resource_group_name = local.resource_group
 }
 
 data "azurerm_client_config" "terraform" {}
+
+resource "azurerm_proximity_placement_group" "common" {
+  for_each            = { for idx, name in local.regions : name => idx }
+  location            = each.key
+  name                = "common-${lower(each.key)}"
+  resource_group_name = local.resource_group
+
+  tags = merge(var.tags)
+}

azure/init/account.tf

@@ -0,0 +1,23 @@
+
+resource "azurerm_resource_group" "kubernetes" {
+  location = var.regions[0]
+  name     = var.project
+
+  tags = var.tags
+}
+
+# resource "azurerm_user_assigned_identity" "ccm" {
+#   name                = "kubernetes-ccm"
+#   resource_group_name = azurerm_resource_group.kubernetes.name
+#   location            = azurerm_resource_group.kubernetes.location
+
+#   tags = var.tags
+# }
+
+# resource "azurerm_role_assignment" "ccm" {
+#   name               = "ea088185-27f1-4956-a58b-150d2ddd8eb3"
+#   description        = "kubernetes ccm"
+#   scope              = data.azurerm_subscription.current.id
+#   role_definition_id = azurerm_role_definition.ccm.role_definition_id
+#   principal_id       = azurerm_user_assigned_identity.ccm.principal_id
+# }

azure/init/auth.tf

@@ -0,0 +1,5 @@
+
+provider "azurerm" {
+  features {}
+  subscription_id = var.subscription_id
+}

azure/init/common.tf

@@ -0,0 +1,4 @@
+
+data "azurerm_subscription" "current" {}
+
+data "azurerm_client_config" "terraform" {}

azure/init/outputs.tf

@@ -0,0 +1,14 @@
+
+output "resource_group" {
+  description = "Azure resource group"
+  value       = azurerm_resource_group.kubernetes.name
+}
+
+output "role_definition" {
+  description = "Kubernetes role definition"
+  value = {
+    ccm        = azurerm_role_definition.ccm.id
+    csi        = azurerm_role_definition.csi.id
+    autoscaler = azurerm_role_definition.scaler.id
+  }
+}

azure/init/policy.tf

@@ -0,0 +1,92 @@
+
+# Source https://kubernetes-sigs.github.io/cloud-provider-azure/topics/azure-permissions/
+
+resource "azurerm_role_definition" "ccm" {
+  name              = "kubernetes-ccm"
+  description       = "This is a kubernetes role for CCM, created via Terraform"
+  scope             = data.azurerm_subscription.current.id
+  assignable_scopes = [data.azurerm_subscription.current.id]
+
+  permissions {
+    actions = [
+      # LoadBalancer
+      "Microsoft.Network/loadBalancers/read",
+      "Microsoft.Network/publicIPAddresses/read",
+      "Microsoft.Network/networkInterfaces/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/ipconfigurations/publicipaddresses/read",
+
+      "Microsoft.Network/networkSecurityGroups/read",
+      "Microsoft.Network/routeTables/read",
+      "Microsoft.Compute/virtualMachines/read",
+      "Microsoft.Compute/virtualMachineScaleSets/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read",
+    ]
+    not_actions = []
+  }
+}
+
+resource "azurerm_role_definition" "csi" {
+  name              = "kubernetes-csi"
+  description       = "This is a kubernetes role for CSI, created via Terraform"
+  scope             = data.azurerm_subscription.current.id
+  assignable_scopes = [data.azurerm_subscription.current.id]
+
+  permissions {
+    actions = [
+      "Microsoft.Storage/storageAccounts/delete",
+      "Microsoft.Storage/storageAccounts/listKeys/action",
+      "Microsoft.Storage/storageAccounts/read",
+      "Microsoft.Storage/storageAccounts/write",
+      "Microsoft.Storage/operations/read",
+
+      "Microsoft.Compute/disks/delete",
+      "Microsoft.Compute/disks/read",
+      "Microsoft.Compute/disks/write",
+
+      "Microsoft.Compute/snapshots/delete",
+      "Microsoft.Compute/snapshots/read",
+      "Microsoft.Compute/snapshots/write",
+
+      "Microsoft.Compute/locations/DiskOperations/read",
+      "Microsoft.Compute/locations/vmSizes/read",
+      "Microsoft.Compute/locations/operations/read",
+
+      "Microsoft.Compute/virtualMachines/read",
+      "Microsoft.Compute/virtualMachines/write",
+      "Microsoft.Compute/virtualMachineScaleSets/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/write",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read",
+    ]
+    not_actions = []
+  }
+}
+
+
+resource "azurerm_role_definition" "scaler" {
+  name              = "kubernetes-node-autoscaler"
+  description       = "This is a kubernetes role for node autoscaler system, created via Terraform"
+  scope             = data.azurerm_subscription.current.id
+  assignable_scopes = [data.azurerm_subscription.current.id]
+
+  permissions {
+    actions = [
+      "Microsoft.Compute/disks/read",
+
+      "Microsoft.Compute/locations/DiskOperations/read",
+      "Microsoft.Compute/locations/vmSizes/read",
+      "Microsoft.Compute/locations/operations/read",
+
+      "Microsoft.Compute/virtualMachines/read",
+      "Microsoft.Compute/virtualMachineScaleSets/read",
+      "Microsoft.Compute/virtualMachineScaleSets/skus/read",
+      "Microsoft.Compute/virtualMachineScaleSets/vmSizes/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/write",
+      "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read",
+    ]
+    not_actions = []
+  }
+}

azure/init/variables.tf

@@ -0,0 +1,24 @@
+
+variable "subscription_id" {
+  description = "The subscription id"
+  type        = string
+}
+
+variable "project" {
+  description = "The project name"
+  type        = string
+}
+
+variable "regions" {
+  description = "The region name list"
+  type        = list(string)
+  default     = ["uksouth", "ukwest"]
+}
+
+variable "tags" {
+  description = "Tags to set on resources"
+  type        = map(string)
+  default = {
+    environment = "Develop"
+  }
+}

azure/init/versions.tf

@@ -0,0 +1,9 @@
+
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "3.6.0"
+    }
+  }
+}

azure/instances-controlplane.tf

@@ -20,13 +20,13 @@ module "controlplane" {
   instance_availability_set = azurerm_availability_set.controlplane[each.key].id
   instance_count            = lookup(try(var.controlplane[each.key], {}), "count", 0)
   instance_resource_group   = local.resource_group
-  instance_type             = lookup(try(var.controlplane[each.key], {}), "instance_type", "Standard_B2s")
+  instance_type             = lookup(try(var.controlplane[each.key], {}), "instance_type", "Standard_B2ms")
   instance_image            = data.azurerm_shared_image_version.talos.id
   instance_tags             = merge(var.tags, { type = "infra" })
   instance_secgroup         = local.network_secgroup[each.key].controlplane
   instance_params = merge(var.kubernetes, {
-    lbv4   = local.network_public[each.key].controlplane_lb[0]
-    lbv6   = try(local.network_public[each.key].controlplane_lb[1], "")
+    lbv4   = local.network_controlplane[each.key].controlplane_lb[0]
+    lbv6   = try(local.network_controlplane[each.key].controlplane_lb[1], "")
     region = each.key
 
     ccm = templatefile("${path.module}/deployments/azure.json.tpl", {
@@ -40,10 +40,10 @@ module "controlplane" {
     })
   })
 
-  network_internal = local.network_public[each.key]
+  network_internal = local.network_controlplane[each.key]
 }
 
 locals {
-  lbv4s    = [for c in local.network_public : c.controlplane_lb]
+  lbv4s    = [for c in local.network_controlplane : c.controlplane_lb]
   endpoint = try(flatten([for c in module.controlplane : c.controlplane_endpoints])[0], "")
 }

azure/instances-web.tf

@@ -7,15 +7,21 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
   for_each = { for idx, name in local.regions : name => idx }
   location = each.key
 
-  instances            = lookup(try(var.instances[each.key], {}), "web_count", 0)
-  name                 = "web-${lower(each.key)}"
-  computer_name_prefix = "web-${lower(each.key)}-"
-  resource_group_name  = local.resource_group
-  sku                  = lookup(try(var.instances[each.key], {}), "web_instance_type", "Standard_B2s")
-  provision_vm_agent   = false
-  overprovision        = false
+  instances                    = lookup(try(var.instances[each.key], {}), "web_count", 0)
+  name                         = "web-${lower(each.key)}"
+  computer_name_prefix         = "web-${lower(each.key)}-"
+  resource_group_name          = local.resource_group
+  sku                          = lookup(try(var.instances[each.key], {}), "web_type", "Standard_B2s")
+  provision_vm_agent           = false
+  overprovision                = false
+  platform_fault_domain_count  = 2
+  proximity_placement_group_id = azurerm_proximity_placement_group.common[each.key].id
+  health_probe_id              = local.network_public[each.key].sku != "Basic" ? azurerm_lb_probe.web[each.key].id : null
 
-  # availability_set_id        = var.instance_availability_set
+  automatic_instance_repair {
+    enabled      = local.network_public[each.key].sku != "Basic"
+    grace_period = "PT60M"
+  }
 
   network_interface {
     name                      = "web-${lower(each.key)}"
@@ -37,7 +43,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
 
   custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
     merge(var.kubernetes, {
-      lbv4        = local.network_public[each.key].controlplane_lb[0]
+      lbv4        = local.network_controlplane[each.key].controlplane_lb[0]
       labels      = "topology.kubernetes.io/region=${each.key},${local.web_labels}"
       nodeSubnets = [local.network_public[each.key].cidr[0]]
     })

azure/instances-werker.tf

@@ -7,24 +7,21 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
   for_each = { for idx, name in local.regions : name => idx }
   location = each.key
 
-  instances            = lookup(try(var.instances[each.key], {}), "worker_count", 0)
-  name                 = "worker-${lower(each.key)}"
-  computer_name_prefix = "worker-${lower(each.key)}-"
-  resource_group_name  = local.resource_group
-  sku                  = lookup(try(var.instances[each.key], {}), "worker_instance_type", "Standard_B2s")
-  provision_vm_agent   = false
-  overprovision        = false
-
-  # availability_set_id        = var.instance_availability_set
-  # health_probe_id = ""
-  # automatic_instance_repair {
-  #   enabled      = true
-  #   grace_period = "PT30M"
-  # }
+  instances                    = lookup(try(var.instances[each.key], {}), "worker_count", 0)
+  name                         = "worker-${lower(each.key)}"
+  computer_name_prefix         = "worker-${lower(each.key)}-"
+  resource_group_name          = local.resource_group
+  sku                          = lookup(try(var.instances[each.key], {}), "worker_type", "Standard_B2s")
+  provision_vm_agent           = false
+  overprovision                = false
+  platform_fault_domain_count  = 2
+  proximity_placement_group_id = azurerm_proximity_placement_group.common[each.key].id
 
   network_interface {
     name    = "worker-${lower(each.key)}"
     primary = true
+
+    enable_accelerated_networking = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false)
     ip_configuration {
       name      = "worker-${lower(each.key)}-v4"
       primary   = true
@@ -40,7 +37,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
 
   custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
     merge(var.kubernetes, {
-      lbv4        = local.network_public[each.key].controlplane_lb[0]
+      lbv4        = local.network_controlplane[each.key].controlplane_lb[0]
       labels      = "topology.kubernetes.io/region=${each.key},${local.worker_labels}"
       nodeSubnets = [local.network_private[each.key].cidr[0]]
     })
@@ -55,15 +52,15 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
   os_disk {
     caching              = "ReadOnly"
     storage_account_type = "StandardSSD_LRS"
-    disk_size_gb         = 50
+    disk_size_gb         = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false) ? null : 50
 
-    # dynamic "diff_disk_settings" {
-    #   for_each = lookup(try(var.instances[each.key], {}), "worker_instance_type", "Standard_B2s") var.vm_os_ephemeral ? ["Local"] : []
-    #   content {
-    #     option    = diff_disk_settings.value
-    #     placement = "ResourceDisk"
-    #   }
-    # }
+    dynamic "diff_disk_settings" {
+      for_each = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false) ? ["Local"] : []
+      content {
+        option = diff_disk_settings.value
+        # placement = "ResourceDisk"
+      }
+    }
   }
 
   source_image_id = data.azurerm_shared_image_version.talos.id

azure/prepare/common.tf

@@ -1,16 +1,13 @@
 
-resource "azurerm_resource_group" "kubernetes" {
-  location = var.regions[0]
-  name     = var.project
-
-  tags = var.tags
-}
-
 resource "azurerm_ssh_public_key" "terraform" {
   name                = "Terraform"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
   location            = var.regions[0]
   public_key          = file("~/.ssh/terraform.pub")
 
   tags = var.tags
+
+  lifecycle {
+    ignore_changes = [public_key]
+  }
 }

azure/prepare/network-gw.tf

@@ -3,7 +3,7 @@ resource "azurerm_public_ip" "router_v4" {
   for_each            = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
   location            = each.key
   name                = "router-${lower(each.key)}-v4"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
   ip_version          = "IPv4"
   sku                 = azurerm_lb.controlplane[each.key].sku
   allocation_method   = azurerm_lb.controlplane[each.key].sku == "Standard" ? "Static" : "Dynamic"
@@ -15,7 +15,7 @@ resource "azurerm_public_ip" "router_v6" {
   for_each            = { for idx, name in var.regions : name => idx if azurerm_lb.controlplane[name].sku == "Standard" && try(var.capabilities[name].network_gw_enable, false) }
   location            = each.key
   name                = "router-${lower(each.key)}-v6"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
   ip_version          = "IPv6"
   sku                 = azurerm_lb.controlplane[each.key].sku
   allocation_method   = "Static"
@@ -27,16 +27,16 @@ resource "azurerm_network_interface" "router" {
   for_each             = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
   location             = each.key
   name                 = "router-${lower(each.key)}"
-  resource_group_name  = azurerm_resource_group.kubernetes.name
+  resource_group_name  = var.resource_group
   enable_ip_forwarding = true
 
   dynamic "ip_configuration" {
-    for_each = azurerm_subnet.public[each.key].address_prefixes
+    for_each = azurerm_subnet.controlplane[each.key].address_prefixes
 
     content {
       name                          = "router-${lower(each.key)}-v${length(split(".", ip_configuration.value)) > 1 ? "4" : "6"}"
       primary                       = length(split(".", ip_configuration.value)) > 1
-      subnet_id                     = azurerm_subnet.public[each.key].id
+      subnet_id                     = azurerm_subnet.controlplane[each.key].id
       private_ip_address            = cidrhost(ip_configuration.value, -2)
       private_ip_address_version    = length(split(".", ip_configuration.value)) > 1 ? "IPv4" : "IPv6"
       private_ip_address_allocation = "Static"
@@ -53,34 +53,12 @@ resource "azurerm_network_interface_security_group_association" "router" {
   network_security_group_id = azurerm_network_security_group.gateway[each.key].id
 }
 
-# resource "azurerm_network_interface" "router_2" {
-#   for_each            = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
-#   location            = each.key
-#   name                = "router-${lower(each.key)}-private"
-#   resource_group_name = azurerm_resource_group.kubernetes.name
-
-#   dynamic "ip_configuration" {
-#     for_each = azurerm_subnet.private[each.key].address_prefixes
-
-#     content {
-#       name                          = "router-${lower(each.key)}-v${length(split(".", ip_configuration.value)) > 1 ? "4" : "6"}"
-#       primary                       = length(split(".", ip_configuration.value)) > 1
-#       subnet_id                     = azurerm_subnet.private[each.key].id
-#       private_ip_address            = cidrhost(ip_configuration.value, -2)
-#       private_ip_address_version    = length(split(".", ip_configuration.value)) > 1 ? "IPv4" : "IPv6"
-#       private_ip_address_allocation = "Static"
-#     }
-#   }
-
-#   tags = merge(var.tags, { type = "infra" })
-# }
-
 resource "azurerm_linux_virtual_machine" "router" {
   for_each                   = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
   location                   = each.key
   name                       = "router-${lower(each.key)}"
   computer_name              = "router-${lower(each.key)}"
-  resource_group_name        = azurerm_resource_group.kubernetes.name
+  resource_group_name        = var.resource_group
   size                       = lookup(try(var.capabilities[each.key], {}), "network_gw_type", "Standard_B1s")
   allow_extension_operations = false
   provision_vm_agent         = false

azure/prepare/network-lb.tf

@@ -3,22 +3,22 @@ resource "azurerm_lb" "controlplane" {
   for_each            = { for idx, name in var.regions : name => idx }
   location            = each.key
   name                = "controlplane-${each.key}"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
   sku                 = try(var.capabilities[each.key].network_lb_type, "Basic")
 
   dynamic "frontend_ip_configuration" {
-    for_each = [for ip in azurerm_subnet.public[each.key].address_prefixes : ip if try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic" || length(split(".", ip)) > 1]
+    for_each = [for ip in azurerm_subnet.controlplane[each.key].address_prefixes : ip if try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic" || length(split(".", ip)) > 1]
 
     content {
       name                          = "controlplane-lb-v${length(split(".", frontend_ip_configuration.value)) > 1 ? "4" : "6"}"
-      subnet_id                     = azurerm_subnet.public[each.key].id
-      private_ip_address            = cidrhost(frontend_ip_configuration.value, 5)
+      subnet_id                     = azurerm_subnet.controlplane[each.key].id
+      private_ip_address            = cidrhost(frontend_ip_configuration.value, -6)
       private_ip_address_version    = length(split(".", frontend_ip_configuration.value)) > 1 ? "IPv4" : "IPv6"
       private_ip_address_allocation = "Static"
     }
   }
 
-  tags = var.tags
+  tags = merge(var.tags, { type = "infra" })
 }
 
 resource "azurerm_lb_probe" "controlplane" {
@@ -70,30 +70,30 @@ resource "azurerm_lb_rule" "kubernetes_v6" {
   enable_tcp_reset               = try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic"
 }
 
-resource "azurerm_lb_rule" "talos" {
-  for_each                       = { for idx, name in var.regions : name => idx }
-  name                           = "controlplane-talos-v4"
-  loadbalancer_id                = azurerm_lb.controlplane[each.key].id
-  frontend_ip_configuration_name = "controlplane-lb-v4"
-  probe_id                       = azurerm_lb_probe.controlplane[each.key].id
-  backend_address_pool_ids       = [azurerm_lb_backend_address_pool.controlplane_v4[each.key].id]
-  protocol                       = "Tcp"
-  frontend_port                  = 50000
-  backend_port                   = 50000
-  idle_timeout_in_minutes        = 30
-  enable_tcp_reset               = try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic"
-}
+# resource "azurerm_lb_rule" "talos" {
+#   for_each                       = { for idx, name in var.regions : name => idx }
+#   name                           = "controlplane-talos-v4"
+#   loadbalancer_id                = azurerm_lb.controlplane[each.key].id
+#   frontend_ip_configuration_name = "controlplane-lb-v4"
+#   probe_id                       = azurerm_lb_probe.controlplane[each.key].id
+#   backend_address_pool_ids       = [azurerm_lb_backend_address_pool.controlplane_v4[each.key].id]
+#   protocol                       = "Tcp"
+#   frontend_port                  = 50000
+#   backend_port                   = 50000
+#   idle_timeout_in_minutes        = 30
+#   enable_tcp_reset               = try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic"
+# }
 
-resource "azurerm_lb_rule" "talos_v6" {
-  for_each                       = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_lb_type, "Basic") != "Basic" }
-  name                           = "controlplane-talos-v6"
-  loadbalancer_id                = azurerm_lb.controlplane[each.key].id
-  frontend_ip_configuration_name = "controlplane-lb-v6"
-  probe_id                       = azurerm_lb_probe.controlplane[each.key].id
-  backend_address_pool_ids       = [azurerm_lb_backend_address_pool.controlplane_v6[each.key].id]
-  protocol                       = "Tcp"
-  frontend_port                  = 50000
-  backend_port                   = 50000
-  idle_timeout_in_minutes        = 30
-  enable_tcp_reset               = try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic"
-}
+# resource "azurerm_lb_rule" "talos_v6" {
+#   for_each                       = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_lb_type, "Basic") != "Basic" }
+#   name                           = "controlplane-talos-v6"
+#   loadbalancer_id                = azurerm_lb.controlplane[each.key].id
+#   frontend_ip_configuration_name = "controlplane-lb-v6"
+#   probe_id                       = azurerm_lb_probe.controlplane[each.key].id
+#   backend_address_pool_ids       = [azurerm_lb_backend_address_pool.controlplane_v6[each.key].id]
+#   protocol                       = "Tcp"
+#   frontend_port                  = 50000
+#   backend_port                   = 50000
+#   idle_timeout_in_minutes        = 30
+#   enable_tcp_reset               = try(var.capabilities[each.key].network_lb_type, "Basic") != "Basic"
+# }

azure/prepare/network-nat.tf

@@ -3,7 +3,7 @@ resource "azurerm_public_ip" "nat" {
   for_each                = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_nat_enable, false) }
   location                = each.key
   name                    = "nat-${each.key}"
-  resource_group_name     = azurerm_resource_group.kubernetes.name
+  resource_group_name     = var.resource_group
   sku                     = "Standard"
   allocation_method       = "Static"
   idle_timeout_in_minutes = 30
@@ -15,7 +15,7 @@ resource "azurerm_nat_gateway" "nat" {
   for_each                = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_nat_enable, false) }
   location                = each.key
   name                    = "nat-${each.key}"
-  resource_group_name     = azurerm_resource_group.kubernetes.name
+  resource_group_name     = var.resource_group
   sku_name                = "Standard"
   idle_timeout_in_minutes = 30
 

azure/prepare/network.tf

@@ -4,33 +4,58 @@ resource "azurerm_virtual_network" "main" {
   location            = each.key
   name                = "main-${each.key}"
   address_space       = [cidrsubnet(var.network_cidr[0], 6, var.network_shift + each.value * 4), cidrsubnet(var.network_cidr[1], 6, var.network_shift + each.value * 4)]
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
 
   tags = merge(var.tags, { type = "infra" })
 }
 
+resource "azurerm_subnet" "controlplane" {
+  for_each             = { for idx, name in var.regions : name => idx }
+  name                 = "controlplane"
+  resource_group_name  = var.resource_group
+  virtual_network_name = azurerm_virtual_network.main[each.key].name
+  address_prefixes = [
+    for cidr in azurerm_virtual_network.main[each.key].address_space : cidrsubnet(cidr, length(split(".", cidr)) > 1 ? 3 : 2, 0)
+  ]
+  service_endpoints = ["Microsoft.ContainerRegistry", "Microsoft.Storage"]
+}
+
+resource "azurerm_subnet" "services" {
+  for_each             = { for idx, name in var.regions : name => idx }
+  name                 = "services"
+  resource_group_name  = var.resource_group
+  virtual_network_name = azurerm_virtual_network.main[each.key].name
+  address_prefixes = [
+    for cidr in azurerm_virtual_network.main[each.key].address_space : cidrsubnet(cidr, 3, 1) if length(split(".", cidr)) > 1
+  ]
+}
+
 resource "azurerm_subnet" "public" {
   for_each             = { for idx, name in var.regions : name => idx }
   name                 = "public"
-  resource_group_name  = azurerm_resource_group.kubernetes.name
+  resource_group_name  = var.resource_group
   virtual_network_name = azurerm_virtual_network.main[each.key].name
-  address_prefixes     = [cidrsubnet(azurerm_virtual_network.main[each.key].address_space[0], 2, 0), cidrsubnet(azurerm_virtual_network.main[each.key].address_space[1], 2, 0)]
-  service_endpoints    = ["Microsoft.ContainerRegistry", "Microsoft.Storage"]
+  address_prefixes = [
+    for cidr in azurerm_virtual_network.main[each.key].address_space : cidrsubnet(cidr, 2, 1)
+  ]
+  service_endpoints = ["Microsoft.ContainerRegistry", "Microsoft.Storage"]
 }
 
 resource "azurerm_subnet" "private" {
   for_each             = { for idx, name in var.regions : name => idx }
   name                 = "private"
-  resource_group_name  = azurerm_resource_group.kubernetes.name
+  resource_group_name  = var.resource_group
   virtual_network_name = azurerm_virtual_network.main[each.key].name
-  address_prefixes     = [cidrsubnet(azurerm_virtual_network.main[each.key].address_space[0], 2, 1), cidrsubnet(azurerm_virtual_network.main[each.key].address_space[1], 2, 1)]
-  service_endpoints    = ["Microsoft.ContainerRegistry", "Microsoft.Storage"]
+  address_prefixes = [
+    for cidr in azurerm_virtual_network.main[each.key].address_space : cidrsubnet(cidr, 2, 2)
+  ]
+  service_endpoints = ["Microsoft.ContainerRegistry", "Microsoft.Storage"]
 }
 
 resource "azurerm_virtual_network_peering" "peering" {
   for_each                     = { for idx, name in var.regions : name => idx }
   name                         = "peering-from-${each.key}"
-  resource_group_name          = azurerm_resource_group.kubernetes.name
+  resource_group_name          = var.resource_group
   virtual_network_name         = azurerm_virtual_network.main[each.key].name
   remote_virtual_network_id    = element([for network in azurerm_virtual_network.main : network.id if network.location != each.key], 0)
   allow_virtual_network_access = true
@@ -42,7 +67,7 @@ resource "azurerm_route_table" "link" {
   for_each            = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
   location            = each.key
   name                = "link-${each.key}"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
 
   dynamic "route" {
     for_each = range(0, length(var.network_cidr))
@@ -65,6 +90,12 @@ resource "azurerm_route_table" "link" {
   tags = merge(var.tags, { type = "infra" })
 }
 
+resource "azurerm_subnet_route_table_association" "controlplane" {
+  for_each       = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
+  subnet_id      = azurerm_subnet.controlplane[each.key].id
+  route_table_id = azurerm_route_table.link[each.key].id
+}
+
 resource "azurerm_subnet_route_table_association" "public" {
   for_each       = { for idx, name in var.regions : name => idx if try(var.capabilities[name].network_gw_enable, false) }
   subnet_id      = azurerm_subnet.public[each.key].id

azure/prepare/outputs.tf

@@ -4,11 +4,6 @@ output "subscription" {
   value       = var.subscription_id
 }
 
-output "project" {
-  description = "Azure project name"
-  value       = var.project
-}
-
 output "regions" {
   description = "Azure regions"
   value       = var.regions
@@ -16,7 +11,7 @@ output "regions" {
 
 output "resource_group" {
   description = "Azure resource group"
-  value       = azurerm_resource_group.kubernetes.name
+  value       = var.resource_group
 }
 
 output "network" {
@@ -26,9 +21,9 @@ output "network" {
   } }
 }
 
-output "network_public" {
-  description = "The public network"
-  value = { for zone, subnet in azurerm_subnet.public : zone => {
+output "network_controlplane" {
+  description = "The controlplane network"
+  value = { for zone, subnet in azurerm_subnet.controlplane : zone => {
     network_id           = subnet.id
     cidr                 = subnet.address_prefixes
     sku                  = azurerm_lb.controlplane[zone].sku
@@ -38,6 +33,15 @@ output "network_public" {
   } }
 }
 
+output "network_public" {
+  description = "The public network"
+  value = { for zone, subnet in azurerm_subnet.public : zone => {
+    network_id = subnet.id
+    cidr       = subnet.address_prefixes
+    sku        = azurerm_lb.controlplane[zone].sku
+  } }
+}
+
 output "network_private" {
   description = "The private network"
   value = { for zone, subnet in azurerm_subnet.private : zone => {

azure/prepare/secgroup-common.tf

@@ -9,7 +9,7 @@ resource "azurerm_network_security_group" "common" {
   for_each            = { for idx, name in var.regions : name => idx }
   location            = each.key
   name                = "common-${each.key}"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
 
   tags = merge(var.tags, { type = "infra" })
 }

azure/prepare/secgroup-controlplane.tf

@@ -3,7 +3,7 @@ resource "azurerm_network_security_group" "controlplane" {
   for_each            = { for idx, name in var.regions : name => idx }
   location            = each.key
   name                = "controlplane-${each.key}"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
 
   dynamic "security_rule" {
     for_each = var.whitelist_admin

azure/prepare/secgroup-gw.tf

@@ -3,7 +3,7 @@ resource "azurerm_network_security_group" "gateway" {
   for_each            = { for idx, name in var.regions : name => idx }
   location            = each.key
   name                = "gateway-${each.key}"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
 
   dynamic "security_rule" {
     for_each = var.whitelist_admin

azure/prepare/secgroup-web.tf

@@ -3,7 +3,7 @@ resource "azurerm_network_security_group" "web" {
   for_each            = { for idx, name in var.regions : name => idx }
   location            = each.key
   name                = "web-${each.key}"
-  resource_group_name = azurerm_resource_group.kubernetes.name
+  resource_group_name = var.resource_group
 
   dynamic "security_rule" {
     for_each = var.whitelist_admin

azure/prepare/variables.tf

@@ -4,8 +4,8 @@ variable "subscription_id" {
   type        = string
 }
 
-variable "project" {
-  description = "The project name"
+variable "resource_group" {
+  description = "The resource group name"
   type        = string
 }
 

azure/variables.tf

@@ -7,6 +7,10 @@ variable "ccm_password" {
   default = ""
 }
 
+variable "gallery_name" {
+  default = ""
+}
+
 data "terraform_remote_state" "prepare" {
   backend = "local"
   config = {
@@ -16,14 +20,14 @@ data "terraform_remote_state" "prepare" {
 
 locals {
   subscription_id = data.terraform_remote_state.prepare.outputs.subscription
-  project         = data.terraform_remote_state.prepare.outputs.project
   regions         = data.terraform_remote_state.prepare.outputs.regions
   resource_group  = data.terraform_remote_state.prepare.outputs.resource_group
 
-  network          = data.terraform_remote_state.prepare.outputs.network
-  network_public   = data.terraform_remote_state.prepare.outputs.network_public
-  network_private  = data.terraform_remote_state.prepare.outputs.network_private
-  network_secgroup = data.terraform_remote_state.prepare.outputs.secgroups
+  network              = data.terraform_remote_state.prepare.outputs.network
+  network_controlplane = data.terraform_remote_state.prepare.outputs.network_controlplane
+  network_public       = data.terraform_remote_state.prepare.outputs.network_public
+  network_private      = data.terraform_remote_state.prepare.outputs.network_private
+  network_secgroup     = data.terraform_remote_state.prepare.outputs.secgroups
 }
 
 variable "tags" {
@@ -72,16 +76,16 @@ variable "instances" {
   type        = map(any)
   default = {
     "uksouth" = {
-      web_count            = 0,
-      web_instance_type    = "Standard_B2s",
-      worker_count         = 0,
-      worker_instance_type = "Standard_B4ms", # B4ms E2as_v4
+      web_count    = 0,
+      web_type     = "Standard_B2s",
+      worker_count = 0,
+      worker_type  = "Standard_B4ms", # B4ms E2as_v4
     },
     "ukwest" = {
-      web_count            = 0,
-      web_instance_type    = "Standard_B2s",
-      worker_count         = 0,
-      worker_instance_type = "Standard_B4ms", # B4ms E2as_v4
+      web_count    = 0,
+      web_type     = "Standard_B2s",
+      worker_count = 0,
+      worker_type  = "Standard_B4ms", # B4ms E2as_v4
     },
   }
 }