From 58417836e871b97cfc2d69466f796a61d35c581b Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Wed, 25 May 2022 19:46:59 +0300 Subject: [PATCH] Secrets --- azure/common.tf | 2 + .../deployments/azure-cluster-autoscaler.yaml | 41 ++++--------------- azure/deployments/azure-storage.yaml | 3 ++ azure/deployments/azure.json.tpl | 23 +++++++++++ azure/instances-controlplane.tf | 10 +++++ azure/prepare/outputs.tf | 7 ++++ azure/templates/controlplane.yaml.tpl | 11 +++++ azure/variables.tf | 9 ++++ 8 files changed, 74 insertions(+), 32 deletions(-) create mode 100644 azure/deployments/azure.json.tpl diff --git a/azure/common.tf b/azure/common.tf index 3724a04..fc55675 100644 --- a/azure/common.tf +++ b/azure/common.tf @@ -11,3 +11,5 @@ data "azurerm_shared_image_version" "talos" { gallery_name = "293f5f4eea925204" resource_group_name = local.resource_group } + +data "azurerm_client_config" "terraform" {} diff --git a/azure/deployments/azure-cluster-autoscaler.yaml b/azure/deployments/azure-cluster-autoscaler.yaml index 6b9bddf..31a466c 100644 --- a/azure/deployments/azure-cluster-autoscaler.yaml +++ b/azure/deployments/azure-cluster-autoscaler.yaml @@ -162,41 +162,18 @@ spec: - ./cluster-autoscaler - --v=3 - --logtostderr=true + - --cloud-config=/etc/azure/azure.json - --cloud-provider=azure # - --regional - --skip-nodes-with-local-storage=false - --ignore-daemonsets-utilization # - --nodes=0:3:web-uksouth - --node-group-auto-discovery=label:cluster-autoscaler-enabled=true,cluster-autoscaler-name=talos-uksouth - env: - - name: ARM_SUBSCRIPTION_ID - valueFrom: - secretKeyRef: - key: SubscriptionID - name: cluster-autoscaler-azure - - name: ARM_RESOURCE_GROUP - valueFrom: - secretKeyRef: - key: ResourceGroup - name: cluster-autoscaler-azure - - name: ARM_TENANT_ID - valueFrom: - secretKeyRef: - key: TenantID - name: cluster-autoscaler-azure - - name: ARM_CLIENT_ID - valueFrom: - secretKeyRef: - key: ClientID - name: cluster-autoscaler-azure - - name: ARM_CLIENT_SECRET - valueFrom: - secretKeyRef: - key: ClientSecret - name: cluster-autoscaler-azure - - name: ARM_VM_TYPE - valueFrom: - secretKeyRef: - key: VMType - name: cluster-autoscaler-azure - restartPolicy: Always + volumeMounts: + - name: cloud-config + mountPath: /etc/azure + readOnly: true + volumes: + - name: cloud-config + secret: + secretName: azure-cloud-controller-manager diff --git a/azure/deployments/azure-storage.yaml b/azure/deployments/azure-storage.yaml index e17c98d..7cd8a8a 100644 --- a/azure/deployments/azure-storage.yaml +++ b/azure/deployments/azure-storage.yaml @@ -13,6 +13,7 @@ parameters: zoned: "true" reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true # allowedTopologies: # - matchLabelExpressions: # - key: topology.disk.csi.azure.com/zone @@ -34,6 +35,7 @@ parameters: zoned: "true" reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true # allowedTopologies: # - matchLabelExpressions: # - key: topology.disk.csi.azure.com/zone @@ -55,6 +57,7 @@ parameters: zoned: "true" reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true # allowedTopologies: # - matchLabelExpressions: # - key: topology.disk.csi.azure.com/zone diff --git a/azure/deployments/azure.json.tpl b/azure/deployments/azure.json.tpl new file mode 100644 index 0000000..54e4bac --- /dev/null +++ b/azure/deployments/azure.json.tpl @@ -0,0 +1,23 @@ +{ + "cloud": "AzurePublicCloud", + "subscriptionId": "${subscriptionId}", + "tenantId": "${tenantId}", + "aadClientId": "${clientId}", + "aadClientSecret": "${clientSecret}", + "resourceGroup": "${resourceGroup}", + "location": "${region}", + "vmType": "vmss", + "vnetName": "${vnetName}", + "vnetResourceGroup": "${resourceGroup}", + "loadBalancerSku": "standard", + "cloudProviderBackoff": true, + "cloudProviderBackoffRetries": 6, + "cloudProviderBackoffExponent": 1.5, + "cloudProviderBackoffDuration": 5, + "cloudProviderBackoffJitter": 1, + "cloudProviderRatelimit": true, + "cloudProviderRateLimitQPS": 6, + "cloudProviderRateLimitBucket": 20, + "useManagedIdentityExtension": false, + "useInstanceMetadata": false +} \ No newline at end of file diff --git a/azure/instances-controlplane.tf b/azure/instances-controlplane.tf index 1aac098..b5a060d 100644 --- a/azure/instances-controlplane.tf +++ b/azure/instances-controlplane.tf @@ -28,6 +28,16 @@ module "controlplane" { lbv4 = local.network_public[each.key].controlplane_lb[0] lbv6 = try(local.network_public[each.key].controlplane_lb[1], "") region = each.key + + ccm = templatefile("${path.module}/deployments/azure.json.tpl", { + subscriptionId = local.subscription_id + tenantId = data.azurerm_client_config.terraform.tenant_id + clientId = var.ccm_username + clientSecret = var.ccm_password + region = each.key + resourceGroup = local.resource_group + vnetName = local.network[each.key].name + }) }) network_internal = local.network_public[each.key] diff --git a/azure/prepare/outputs.tf b/azure/prepare/outputs.tf index 9841e4c..1901601 100644 --- a/azure/prepare/outputs.tf +++ b/azure/prepare/outputs.tf @@ -19,6 +19,13 @@ output "resource_group" { value = azurerm_resource_group.kubernetes.name } +output "network" { + description = "The network" + value = { for zone, net in azurerm_virtual_network.main : zone => { + name = net.name + } } +} + output "network_public" { description = "The public network" value = { for zone, subnet in azurerm_subnet.public : zone => { diff --git a/azure/templates/controlplane.yaml.tpl b/azure/templates/controlplane.yaml.tpl index f4a1549..5a5218d 100644 --- a/azure/templates/controlplane.yaml.tpl +++ b/azure/templates/controlplane.yaml.tpl @@ -52,6 +52,17 @@ cluster: node-cidr-mask-size-ipv6: 112 scheduler: {} etcd: {} + inlineManifests: + - name: azure-cloud-controller-config + contents: |- + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: azure-cloud-controller-manager + namespace: kube-system + data: + cloud-config: ${base64encode(ccm)} externalCloudProvider: enabled: true manifests: diff --git a/azure/variables.tf b/azure/variables.tf index 0cc8ce3..cd978ad 100644 --- a/azure/variables.tf +++ b/azure/variables.tf @@ -1,4 +1,12 @@ +variable "ccm_username" { + default = "" +} + +variable "ccm_password" { + default = "" +} + data "terraform_remote_state" "prepare" { backend = "local" config = { @@ -12,6 +20,7 @@ locals { regions = data.terraform_remote_state.prepare.outputs.regions resource_group = data.terraform_remote_state.prepare.outputs.resource_group + network = data.terraform_remote_state.prepare.outputs.network network_public = data.terraform_remote_state.prepare.outputs.network_public network_private = data.terraform_remote_state.prepare.outputs.network_private network_secgroup = data.terraform_remote_state.prepare.outputs.secgroups