av-zones support

This commit is contained in:
Serge Logvinov
2023-06-28 19:14:33 +03:00
parent 840620fcf4
commit bfc7e141de
21 changed files with 245 additions and 68 deletions

View File

@@ -21,5 +21,5 @@ create-deployments: ## create templates
# helm template --namespace=local-path-storage -f vars/local-path-storage.yaml local-path-provisioner \
# ~/work/sergelogvinov/local-path-provisioner/deploy/chart/local-path-provisioner > vars/local-path-storage-result.yaml
helm template --namespace=ingress-nginx --version=4.6.0 -f vars/ingress.yaml ingress-nginx \
helm template --namespace=ingress-nginx --version=4.7.0 -f vars/ingress.yaml ingress-nginx \
ingress-nginx/ingress-nginx > vars/ingress-result.yaml

View File

@@ -4,10 +4,10 @@ apiVersion: v1
kind: ServiceAccount
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -20,10 +20,10 @@ apiVersion: v1
kind: ConfigMap
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -66,10 +66,10 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
name: ingress-nginx
@@ -150,10 +150,10 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
name: ingress-nginx
@@ -171,10 +171,10 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -264,10 +264,10 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -288,10 +288,10 @@ kind: Service
metadata:
annotations:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -325,10 +325,10 @@ apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -352,10 +352,10 @@ spec:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
@@ -363,7 +363,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: controller
image: "registry.k8s.io/ingress-nginx/controller:v1.7.0@sha256:7612338342a1e7b8090bef78f2a04fffcadd548ccaabe8a47bf7758ff549a5f7"
image: "registry.k8s.io/ingress-nginx/controller:v1.8.0@sha256:744ae2afd433a395eeb13dc03d3313facba92e96ad71d9feaafc85925493fee3"
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
@@ -376,6 +376,9 @@ spec:
- --controller-class=k8s.io/ingress-nginx
- --ingress-class=nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --enable-topology-aware-routing=true
- --post-shutdown-grace-period=30
- --report-node-internal-ip-address=true
securityContext:
capabilities:
drop:
@@ -455,10 +458,10 @@ apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
helm.sh/chart: ingress-nginx-4.6.0
helm.sh/chart: ingress-nginx-4.7.0
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: "1.7.0"
app.kubernetes.io/version: "1.8.0"
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller

View File

@@ -66,6 +66,11 @@ controller:
prometheus.io/scrape: "true"
prometheus.io/port: "10254"
extraArgs:
report-node-internal-ip-address: "true"
enable-topology-aware-routing: "true"
post-shutdown-grace-period: 30
extraEnvs:
- name: NODE_NAME
valueFrom:

View File

@@ -3,19 +3,3 @@ provider "azurerm" {
features {}
subscription_id = local.subscription_id
}
# data "azurerm_virtual_machine_size" "size" {
# name = "Standard_D2pls_v5"
# location = "westeurope"
# }
# resource "azurerm_linux_virtual_machine_scale_set" "worker" {
# source_image_reference {
# location = "westeurope"
# publisher = "Canonical"
# offer = "0001-com-ubuntu-server-jammy"
# sku = "22_04-lts-${data.azurerm_virtual_machine_size.size.architecture == "Arm64" ? "arm64" : "gen2"}"
# version = "latest"
# }
# }

View File

@@ -298,8 +298,10 @@ spec:
- --cloud-provider=azure
- --namespace=kube-system
- --node-group-auto-discovery=label:cluster-autoscaler-enabled=true,cluster-autoscaler-name=talos-uksouth
- --balance-similar-node-groups=true
- --cloud-config=/etc/azure/azure.json
- --logtostderr=true
- --node-delete-delay-after-taint=30s
- --regional=true
- --stderrthreshold=info
- --v=3

View File

@@ -15,6 +15,8 @@ autoDiscovery:
extraArgs:
cloud-config: /etc/azure/azure.json
regional: true
balance-similar-node-groups: true
node-delete-delay-after-taint: 30s
logtostderr: true
stderrthreshold: info
v: 3

View File

@@ -22,8 +22,8 @@ spec:
run: overprovisioning
spec:
nodeSelector:
node.cloudprovider.kubernetes.io/platform: azure
project.io/node-pool: web
# node.cloudprovider.kubernetes.io/platform: azure
project.io/node-pool: worker
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:

View File

@@ -6,7 +6,7 @@ resource "azurerm_availability_set" "controlplane" {
resource_group_name = local.resource_group
platform_update_domain_count = 1
platform_fault_domain_count = 3
platform_fault_domain_count = 2
tags = merge(var.tags, { type = "infra" })
}
@@ -22,7 +22,7 @@ locals {
region : region
availability_set : azurerm_availability_set.controlplane[region].id
image : data.azurerm_shared_image_version.talos[length(regexall("^Standard_[DE][\\d+]p", lookup(try(var.controlplane[region], {}), "db_type", ""))) > 0 ? "Arm64" : "x64"].id
image : data.azurerm_shared_image_version.talos[length(regexall("^Standard_[DE][\\d+]p", lookup(try(var.controlplane[region], {}), "type", ""))) > 0 ? "Arm64" : "x64"].id
type : lookup(try(var.controlplane[region], {}), "type", "Standard_B2ms")
ip : 11 + inx
@@ -117,7 +117,7 @@ resource "local_file" "controlplane" {
for_each = local.controlplanes
content = templatefile("${path.module}/templates/controlplane.yaml.tpl",
merge(var.kubernetes, {
merge(var.kubernetes, var.acr, {
name = each.value.name
labels = local.controlplane_labels
certSANs = flatten([
@@ -170,7 +170,7 @@ resource "azurerm_linux_virtual_machine" "controlplane" {
admin_username = "talos"
admin_ssh_key {
username = "talos"
public_key = file("~/.ssh/terraform.pub")
public_key = var.ssh_public_key
}
source_image_id = length(each.value.image) > 0 ? each.value.image : null

View File

@@ -15,10 +15,10 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" {
provision_vm_agent = false
overprovision = false
platform_fault_domain_count = 5
proximity_placement_group_id = azurerm_proximity_placement_group.common[each.key].id
proximity_placement_group_id = length(var.zones) == 1 ? azurerm_proximity_placement_group.common[each.key].id : null
# zone_balance = true
# zones = ["0", "1", "2"]
zone_balance = true
zones = var.zones
network_interface {
name = "db-${lower(each.key)}"
@@ -47,7 +47,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" {
}
custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
merge(var.kubernetes, {
merge(var.kubernetes, var.acr, {
lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "")
labels = local.db_labels
nodeSubnets = [local.network_public[each.key].cidr[0]]
@@ -57,7 +57,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" {
admin_username = "talos"
admin_ssh_key {
username = "talos"
public_key = file("~/.ssh/terraform.pub")
public_key = var.ssh_public_key
}
os_disk {

View File

@@ -15,10 +15,10 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
provision_vm_agent = false
overprovision = false
platform_fault_domain_count = 5
proximity_placement_group_id = azurerm_proximity_placement_group.common[each.key].id
proximity_placement_group_id = length(var.zones) == 1 ? azurerm_proximity_placement_group.common[each.key].id : null
# zone_balance = false
# zones = ["1"]
zone_balance = length(var.zones) > 0
zones = var.zones
# health_probe_id = local.network_public[each.key].sku != "Basic" ? azurerm_lb_probe.web[each.key].id : null
# automatic_instance_repair {
@@ -53,7 +53,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
}
custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
merge(var.kubernetes, {
merge(var.kubernetes, var.acr, {
lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "")
labels = local.web_labels
nodeSubnets = [local.network_public[each.key].cidr[0]]
@@ -63,7 +63,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
admin_username = "talos"
admin_ssh_key {
username = "talos"
public_key = file("~/.ssh/terraform.pub")
public_key = var.ssh_public_key
}
os_disk {

View File

@@ -15,10 +15,10 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
provision_vm_agent = false
overprovision = false
platform_fault_domain_count = 5
proximity_placement_group_id = azurerm_proximity_placement_group.common[each.key].id
proximity_placement_group_id = length(var.zones) == 1 ? azurerm_proximity_placement_group.common[each.key].id : null
# zone_balance = false
# zones = ["1"]
zone_balance = length(var.zones) > 0
zones = var.zones
# extension_operations_enabled = true
# extension {
@@ -65,7 +65,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
}
custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
merge(var.kubernetes, {
merge(var.kubernetes, var.acr, {
lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "")
labels = local.worker_labels
nodeSubnets = [local.network_private[each.key].cidr[0]]
@@ -75,13 +75,13 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
admin_username = "talos"
admin_ssh_key {
username = "talos"
public_key = file("~/.ssh/terraform.pub")
public_key = var.ssh_public_key
}
os_disk {
caching = "ReadOnly"
storage_account_type = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false) ? "Standard_LRS" : "StandardSSD_LRS"
disk_size_gb = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false) ? 32 : 50
disk_size_gb = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false) ? try(var.instances[each.key].worker_os_disk_size, 64) : 50
dynamic "diff_disk_settings" {
for_each = lookup(try(var.instances[each.key], {}), "worker_os_ephemeral", false) ? ["Local"] : []
@@ -115,3 +115,99 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
ignore_changes = [instances, admin_username, admin_ssh_key, source_image_id]
}
}
resource "azurerm_linux_virtual_machine_scale_set" "worker_as" {
for_each = { for idx, name in local.regions : name => idx }
location = each.key
instances = lookup(try(var.instances[each.key], {}), "worker_count", 0)
name = "worker-${lower(each.key)}-as"
computer_name_prefix = "worker-${lower(each.key)}-as-"
resource_group_name = local.resource_group
sku = lookup(try(var.instances[each.key], {}), "worker_type", "Standard_B2s")
provision_vm_agent = false
overprovision = false
platform_fault_domain_count = 1
proximity_placement_group_id = length(var.zones) == 1 ? azurerm_proximity_placement_group.common[each.key].id : null
zone_balance = length(var.zones) > 0
zones = var.zones
eviction_policy = "Delete"
priority = "Spot"
network_interface {
name = "worker-${lower(each.key)}-as"
primary = true
network_security_group_id = local.network_secgroup[each.key].common
enable_accelerated_networking = true
ip_configuration {
name = "worker-${lower(each.key)}-as-v4"
primary = true
version = "IPv4"
subnet_id = local.network_private[each.key].network_id
}
ip_configuration {
name = "worker-${lower(each.key)}-as-v6"
version = "IPv6"
subnet_id = local.network_private[each.key].network_id
dynamic "public_ip_address" {
for_each = local.network_private[each.key].sku == "Standard" ? ["IPv6"] : []
content {
name = "worker-${lower(each.key)}-as-v6"
version = public_ip_address.value
}
}
}
}
custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
merge(var.kubernetes, var.acr, {
lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "")
labels = local.worker_labels
nodeSubnets = [local.network_private[each.key].cidr[0]]
})
))
admin_username = "talos"
admin_ssh_key {
username = "talos"
public_key = var.ssh_public_key
}
os_disk {
caching = "ReadOnly"
storage_account_type = "Standard_LRS"
disk_size_gb = try(var.instances[each.key].worker_os_disk_size, 64)
diff_disk_settings {
option = "Local"
placement = "ResourceDisk"
}
}
source_image_id = data.azurerm_shared_image_version.talos[length(regexall("^Standard_[DE][\\d+]p", lookup(try(var.instances[each.key], {}), "worker_type", ""))) > 0 ? "Arm64" : "x64"].id
# source_image_reference {
# publisher = "talos"
# offer = "Talos"
# sku = "1.0-dev"
# version = "latest"
# }
tags = merge(var.tags, {
type = "worker",
"cluster-autoscaler-enabled" = "true",
"cluster-autoscaler-name" = "${local.resource_group}-${lower(each.key)}",
"min" = 0,
"max" = 3,
"k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "worker"
})
boot_diagnostics {}
lifecycle {
ignore_changes = [instances, admin_username, admin_ssh_key, source_image_id]
}
}

View File

@@ -6,8 +6,13 @@ resource "azurerm_public_ip" "web_v4" {
resource_group_name = local.resource_group
sku = local.network_public[each.key].sku
allocation_method = local.network_public[each.key].sku == "Standard" ? "Static" : "Dynamic"
zones = local.network_public[each.key].sku == "Standard" ? var.zones : []
tags = merge(var.tags, { type = "web" })
lifecycle {
ignore_changes = [zones]
}
}
resource "azurerm_lb" "web" {

View File

@@ -25,7 +25,7 @@ output "controlplane_endpoint_public" {
value = try(one([for ip in azurerm_public_ip.controlplane_v4 : ip.ip_address if ip.ip_address != ""]), "127.0.0.1")
}
# output "web_endpoint" {
# description = "Kubernetes controlplane endpoint"
# value = compact([for lb in azurerm_public_ip.web_v4 : lb.ip_address])
# }
output "web_endpoint" {
description = "Kubernetes controlplane endpoint"
value = compact([for lb in azurerm_public_ip.web_v4 : lb.ip_address])
}

View File

@@ -0,0 +1,9 @@
output "registry" {
value = "${azurerm_container_registry.registry.name}.azurecr.io"
}
output "registry_token" {
value = azurerm_container_registry_token_password.containerd.password1[0].value
sensitive = true
}

View File

@@ -0,0 +1,33 @@
resource "random_id" "registry" {
byte_length = 8
}
resource "azurerm_container_registry" "registry" {
name = "registry${random_id.registry.hex}"
resource_group_name = local.resource_group
location = local.regions[0]
sku = "Basic"
admin_enabled = false
tags = var.tags
}
data "azurerm_container_registry_scope_map" "pull" {
name = "_repositories_pull"
resource_group_name = local.resource_group
container_registry_name = azurerm_container_registry.registry.name
}
resource "azurerm_container_registry_token" "containerd" {
name = "containerd"
resource_group_name = local.resource_group
container_registry_name = azurerm_container_registry.registry.name
scope_map_id = data.azurerm_container_registry_scope_map.pull.id
}
resource "azurerm_container_registry_token_password" "containerd" {
container_registry_token_id = azurerm_container_registry_token.containerd.id
password1 {}
}

View File

@@ -3,8 +3,8 @@ terraform {
required_providers {
azurerm = {
source = "hashicorp/azurerm"
version = "~> 3.40.0"
version = "~> 3.62.1"
}
}
required_version = ">= 1.2"
required_version = ">= 1.5"
}

View File

@@ -57,6 +57,14 @@ machine:
- os:reader
allowedKubernetesNamespaces:
- kube-system
%{if acrRepo != "" }
registries:
config:
${acrRepo}:
auth:
username: ${acrUsername}
password: ${acrPassword}
%{endif}
cluster:
adminKubeconfig:
certLifetime: 8h0m0s
@@ -102,6 +110,7 @@ cluster:
manifests:
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-cloud-controller-manager.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-autoscaler-result.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azuredisk-csi-driver-result.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azuredisk-storage.yaml
- https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml

View File

@@ -59,6 +59,14 @@ machine:
keys:
- nodeID: {}
slot: 0
%{if acrRepo != "" }
registries:
config:
${acrRepo}:
auth:
username: ${acrUsername}
password: ${acrPassword}
%{endif}
cluster:
id: ${clusterID}
secret: ${clusterSecret}

View File

@@ -73,6 +73,27 @@ variable "kubernetes" {
sensitive = true
}
variable "acr" {
type = map(string)
default = {
acrRepo = ""
acrUsername = ""
acrPassword = ""
}
}
variable "zones" {
description = "The Azure zones"
type = list(string)
default = ["1", "3"]
}
variable "ssh_public_key" {
description = "The SSH-RSA public key, ssh-keygen -t rsa -b 2048 -f ~/.ssh/terraform -C 'terraform'"
type = string
default = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBx2qCSLlZ03TYqHm88pXZPyqZ3fvR1p2jWvsLt3uX+mBMr6B8S4vkX3oEBv43IEgi1bkIrdjJ50QvXNWS6fSOo6G0wZ0FHRCan3t4Kq2U+qoWkDsb5K0Kdgd9DZuaNM9412J2dWldYK7iD3hhQ3wh/E1gPlqrYb2AsPAarK+VA59n63QCDrpmGCW/Pki69e8Mt7HH/A1uw+4wvlrtaytrx6C3Y3/mQfBoas4XJliWHeTgEKeVdIzlOf9XrDnZ85pmvmQbFAtRtaRlfwCHMksVEwunYbg1RPrvQ8/YsSv6sFHwwvqjrJ7hdJcaa3afS3rUyAy7vkO0OXm4KdOEgE8X terraform"
}
variable "instances" {
description = "Map of region's properties"
type = map(any)

View File

@@ -39,7 +39,7 @@ spec:
priorityClassName: overprovisioning
containers:
- name: reserve-resources
image: registry.k8s.io/pause:3.6
image: registry.k8s.io/pause:3.9
resources:
requests:
cpu: "700m"

View File

@@ -39,7 +39,7 @@ spec:
priorityClassName: overprovisioning
containers:
- name: reserve-resources
image: registry.k8s.io/pause:3.6
image: registry.k8s.io/pause:3.9
resources:
requests:
cpu: "700m"