From 57e04ef7f6b731addf07f632c3feb0e84e973331 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Thu, 29 Jun 2023 12:43:44 +0300 Subject: [PATCH] use acr --- Makefile | 43 +++++++++++++++++++++++++++ azure/instances-controlplane.tf | 2 +- azure/instances-db.tf | 4 +-- azure/instances-web.tf | 4 +-- azure/instances-werker.tf | 6 ++-- azure/templates/controlplane.yaml.tpl | 14 +++++++++ azure/templates/worker.yaml.tpl | 15 ++++++++++ azure/variables.tf | 8 +++++ 8 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6d81168 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +# + +REGISTRY ?= ghcr.io/sergelogvinov +SYNCARGS ?= --multi-arch=all + +### + +KUBERNETES ?= 1.27.3 +PAUSE ?= 3.8 +ETCD ?= 3.5.9 +COREDNS ?= 1.10.1 +CILIUM ?= 1.12.7 +FLUENTBIT ?= 2.1.6 +NODEEXPORTER ?= 1.6.0 + +################################################################################ + +help: + @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +images-sync: + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://registry.k8s.io/kube-apiserver:v$(KUBERNETES) docker://$(REGISTRY)/kube-apiserver:v$(KUBERNETES) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://registry.k8s.io/kube-controller-manager:v$(KUBERNETES) docker://$(REGISTRY)/kube-controller-manager:v$(KUBERNETES) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://registry.k8s.io/kube-scheduler:v$(KUBERNETES) docker://$(REGISTRY)/kube-scheduler:v$(KUBERNETES) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://ghcr.io/siderolabs/kubelet:v$(KUBERNETES) docker://$(REGISTRY)/kubelet:v$(KUBERNETES) + + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://registry.k8s.io/pause:$(PAUSE) docker://$(REGISTRY)/pause:$(PAUSE) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://gcr.io/etcd-development/etcd:v$(ETCD) docker://$(REGISTRY)/etcd:v$(ETCD) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://coredns/coredns:$(COREDNS) docker://$(REGISTRY)/coredns:$(COREDNS) + + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://quay.io/cilium/cilium:v$(CILIUM) docker://$(REGISTRY)/cilium:v$(CILIUM) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://fluent/fluent-bit:$(FLUENTBIT) docker://$(REGISTRY)/fluent-bit:$(FLUENTBIT) + @skopeo copy $(SYNCARGS) --override-os=linux \ + docker://quay.io/prometheus/node-exporter:v$(NODEEXPORTER) docker://$(REGISTRY)/node-exporter:v$(NODEEXPORTER) diff --git a/azure/instances-controlplane.tf b/azure/instances-controlplane.tf index d7cf910..1f40011 100644 --- a/azure/instances-controlplane.tf +++ b/azure/instances-controlplane.tf @@ -117,7 +117,7 @@ resource "local_file" "controlplane" { for_each = local.controlplanes content = templatefile("${path.module}/templates/controlplane.yaml.tpl", - merge(var.kubernetes, var.acr, { + merge(var.kubernetes, var.acr, try(var.controlplane["all"], {}), { name = each.value.name labels = local.controlplane_labels certSANs = flatten([ diff --git a/azure/instances-db.tf b/azure/instances-db.tf index 01ededc..ee6c2d0 100644 --- a/azure/instances-db.tf +++ b/azure/instances-db.tf @@ -47,7 +47,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" { } custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, var.acr, { + merge(var.kubernetes, var.acr, try(var.instances["all"], {}), { lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "") labels = local.db_labels nodeSubnets = [local.network_public[each.key].cidr[0]] @@ -78,7 +78,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" { type = "db", "cluster-autoscaler-enabled" = "true", "cluster-autoscaler-name" = "${local.resource_group}-${lower(each.key)}", - "min" = 0, + "min" = lookup(try(var.instances[each.key], {}), "db_count", 0), "max" = 3, "k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "db" diff --git a/azure/instances-web.tf b/azure/instances-web.tf index 11e7bae..abcb173 100644 --- a/azure/instances-web.tf +++ b/azure/instances-web.tf @@ -53,7 +53,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" { } custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, var.acr, { + merge(var.kubernetes, var.acr, try(var.instances["all"], {}), { lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "") labels = local.web_labels nodeSubnets = [local.network_public[each.key].cidr[0]] @@ -84,7 +84,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" { type = "web", "cluster-autoscaler-enabled" = "true", "cluster-autoscaler-name" = "${local.resource_group}-${lower(each.key)}", - "min" = 0, + "min" = lookup(try(var.instances[each.key], {}), "web_count", 0), "max" = 3, "k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "web" diff --git a/azure/instances-werker.tf b/azure/instances-werker.tf index 0eab849..d701f25 100644 --- a/azure/instances-werker.tf +++ b/azure/instances-werker.tf @@ -65,7 +65,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" { } custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, var.acr, { + merge(var.kubernetes, var.acr, try(var.instances["all"], {}), { lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "") labels = local.worker_labels nodeSubnets = [local.network_private[each.key].cidr[0]] @@ -104,7 +104,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" { type = "worker", "cluster-autoscaler-enabled" = "true", "cluster-autoscaler-name" = "${local.resource_group}-${lower(each.key)}", - "min" = 0, + "min" = lookup(try(var.instances[each.key], {}), "worker_count", 0), "max" = 3, "k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "worker" @@ -164,7 +164,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker_as" { } custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, var.acr, { + merge(var.kubernetes, var.acr, try(var.instances["all"], {}), { lbv4 = try(local.network_controlplane[each.key].controlplane_lb[0], "") labels = local.worker_labels nodeSubnets = [local.network_private[each.key].cidr[0]] diff --git a/azure/templates/controlplane.yaml.tpl b/azure/templates/controlplane.yaml.tpl index 47c9d72..2b6e28e 100644 --- a/azure/templates/controlplane.yaml.tpl +++ b/azure/templates/controlplane.yaml.tpl @@ -1,6 +1,16 @@ machine: certSANs: ${format("%#v",certSANs)} +%{if repository != "registry.k8s.io"} + files: + - content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "${ repository }/pause:3.8" + path: /etc/cri/conf.d/20-customization.part + op: create +%{endif} kubelet: + image: %{if repository == "registry.k8s.io"}ghcr.io/siderolabs%{else}${ repository }%{endif}/kubelet:${ version } extraArgs: node-labels: "${labels}" rotate-server-certificates: true @@ -81,11 +91,15 @@ cluster: proxy: disabled: true apiServer: + image: ${ repository }/kube-apiserver:${ version } certSANs: ${format("%#v",certSANs)} controllerManager: + image: ${ repository }/kube-controller-manager:${ version } extraArgs: node-cidr-mask-size-ipv4: 24 node-cidr-mask-size-ipv6: 112 + scheduler: + image: ${ repository }/kube-scheduler:${ version } etcd: advertisedSubnets: - ${nodeSubnets[0]} diff --git a/azure/templates/worker.yaml.tpl b/azure/templates/worker.yaml.tpl index a529d46..87fdcc0 100644 --- a/azure/templates/worker.yaml.tpl +++ b/azure/templates/worker.yaml.tpl @@ -6,7 +6,18 @@ machine: token: ${tokenMachine} ca: crt: ${caMachine} +%{if repository != "registry.k8s.io"} + files: + - content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "${ repository }/pause:3.8" + path: /etc/cri/conf.d/20-customization.part + op: create +%{endif} kubelet: + image: %{if repository == "registry.k8s.io"}ghcr.io/siderolabs%{else}${ repository }%{endif}/kubelet:${ version } + defaultRuntimeSeccompProfileEnabled: true extraArgs: cloud-provider: external rotate-server-certificates: true @@ -59,6 +70,10 @@ machine: keys: - nodeID: {} slot: 0 + features: + rbac: true + stableHostname: true + apidCheckExtKeyUsage: true %{if acrRepo != "" } registries: config: diff --git a/azure/variables.tf b/azure/variables.tf index 96404da..507e5ff 100644 --- a/azure/variables.tf +++ b/azure/variables.tf @@ -44,6 +44,10 @@ variable "controlplane" { description = "Controlplane config" type = map(any) default = { + "all" = { + repository = "registry.k8s.io" + version = "v1.27.3" + } "uksouth" = { count = 0, type = "Standard_B2ms", @@ -98,6 +102,10 @@ variable "instances" { description = "Map of region's properties" type = map(any) default = { + "all" = { + repository = "registry.k8s.io" + version = "v1.27.3" + } "uksouth" = { web_count = 0, web_type = "Standard_B2s",