use acr

Makefile

@@ -0,0 +1,43 @@
+#
+
+REGISTRY ?= ghcr.io/sergelogvinov
+SYNCARGS ?= --multi-arch=all
+
+###
+
+KUBERNETES ?= 1.27.3
+PAUSE ?= 3.8
+ETCD ?= 3.5.9
+COREDNS ?= 1.10.1
+CILIUM ?= 1.12.7
+FLUENTBIT ?= 2.1.6
+NODEEXPORTER ?= 1.6.0
+
+################################################################################
+
+help:
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+images-sync:
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://registry.k8s.io/kube-apiserver:v$(KUBERNETES) docker://$(REGISTRY)/kube-apiserver:v$(KUBERNETES)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://registry.k8s.io/kube-controller-manager:v$(KUBERNETES) docker://$(REGISTRY)/kube-controller-manager:v$(KUBERNETES)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://registry.k8s.io/kube-scheduler:v$(KUBERNETES) docker://$(REGISTRY)/kube-scheduler:v$(KUBERNETES)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://ghcr.io/siderolabs/kubelet:v$(KUBERNETES) docker://$(REGISTRY)/kubelet:v$(KUBERNETES)
+
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://registry.k8s.io/pause:$(PAUSE) docker://$(REGISTRY)/pause:$(PAUSE)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://gcr.io/etcd-development/etcd:v$(ETCD) docker://$(REGISTRY)/etcd:v$(ETCD)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://coredns/coredns:$(COREDNS) docker://$(REGISTRY)/coredns:$(COREDNS)
+
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://quay.io/cilium/cilium:v$(CILIUM) docker://$(REGISTRY)/cilium:v$(CILIUM)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://fluent/fluent-bit:$(FLUENTBIT) docker://$(REGISTRY)/fluent-bit:$(FLUENTBIT)
+	@skopeo copy $(SYNCARGS) --override-os=linux \
+		docker://quay.io/prometheus/node-exporter:v$(NODEEXPORTER) docker://$(REGISTRY)/node-exporter:v$(NODEEXPORTER)

azure/instances-controlplane.tf

@@ -117,7 +117,7 @@ resource "local_file" "controlplane" {
   for_each = local.controlplanes
 
   content = templatefile("${path.module}/templates/controlplane.yaml.tpl",
-    merge(var.kubernetes, var.acr, {
+    merge(var.kubernetes, var.acr, try(var.controlplane["all"], {}), {
       name   = each.value.name
       labels = local.controlplane_labels
       certSANs = flatten([

azure/instances-db.tf

@@ -47,7 +47,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" {
   }
 
   custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
-    merge(var.kubernetes, var.acr, {
+    merge(var.kubernetes, var.acr, try(var.instances["all"], {}), {
       lbv4        = try(local.network_controlplane[each.key].controlplane_lb[0], "")
       labels      = local.db_labels
       nodeSubnets = [local.network_public[each.key].cidr[0]]
@@ -78,7 +78,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "db" {
     type                         = "db",
     "cluster-autoscaler-enabled" = "true",
     "cluster-autoscaler-name"    = "${local.resource_group}-${lower(each.key)}",
-    "min"                        = 0,
+    "min"                        = lookup(try(var.instances[each.key], {}), "db_count", 0),
     "max"                        = 3,
 
     "k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "db"

azure/instances-web.tf

@@ -53,7 +53,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
   }
 
   custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
-    merge(var.kubernetes, var.acr, {
+    merge(var.kubernetes, var.acr, try(var.instances["all"], {}), {
       lbv4        = try(local.network_controlplane[each.key].controlplane_lb[0], "")
       labels      = local.web_labels
       nodeSubnets = [local.network_public[each.key].cidr[0]]
@@ -84,7 +84,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "web" {
     type                         = "web",
     "cluster-autoscaler-enabled" = "true",
     "cluster-autoscaler-name"    = "${local.resource_group}-${lower(each.key)}",
-    "min"                        = 0,
+    "min"                        = lookup(try(var.instances[each.key], {}), "web_count", 0),
     "max"                        = 3,
 
     "k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "web"

azure/instances-werker.tf

@@ -65,7 +65,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
   }
 
   custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
-    merge(var.kubernetes, var.acr, {
+    merge(var.kubernetes, var.acr, try(var.instances["all"], {}), {
       lbv4        = try(local.network_controlplane[each.key].controlplane_lb[0], "")
       labels      = local.worker_labels
       nodeSubnets = [local.network_private[each.key].cidr[0]]
@@ -104,7 +104,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker" {
     type                         = "worker",
     "cluster-autoscaler-enabled" = "true",
     "cluster-autoscaler-name"    = "${local.resource_group}-${lower(each.key)}",
-    "min"                        = 0,
+    "min"                        = lookup(try(var.instances[each.key], {}), "worker_count", 0),
     "max"                        = 3,
 
     "k8s.io_cluster-autoscaler_node-template_label_project.io_node-pool" = "worker"
@@ -164,7 +164,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "worker_as" {
   }
 
   custom_data = base64encode(templatefile("${path.module}/templates/worker.yaml.tpl",
-    merge(var.kubernetes, var.acr, {
+    merge(var.kubernetes, var.acr, try(var.instances["all"], {}), {
       lbv4        = try(local.network_controlplane[each.key].controlplane_lb[0], "")
       labels      = local.worker_labels
       nodeSubnets = [local.network_private[each.key].cidr[0]]

azure/templates/controlplane.yaml.tpl

@@ -1,6 +1,16 @@
 machine:
   certSANs: ${format("%#v",certSANs)}
+%{if repository != "registry.k8s.io"}
+  files:
+    - content: |
+        [plugins]
+          [plugins."io.containerd.grpc.v1.cri"]
+            sandbox_image = "${ repository }/pause:3.8"
+      path: /etc/cri/conf.d/20-customization.part
+      op: create
+%{endif}
   kubelet:
+    image: %{if repository == "registry.k8s.io"}ghcr.io/siderolabs%{else}${ repository }%{endif}/kubelet:${ version }
     extraArgs:
       node-labels: "${labels}"
       rotate-server-certificates: true
@@ -81,11 +91,15 @@ cluster:
   proxy:
     disabled: true
   apiServer:
+    image: ${ repository }/kube-apiserver:${ version }
     certSANs: ${format("%#v",certSANs)}
   controllerManager:
+    image: ${ repository }/kube-controller-manager:${ version }
     extraArgs:
         node-cidr-mask-size-ipv4: 24
         node-cidr-mask-size-ipv6: 112
+  scheduler:
+    image: ${ repository }/kube-scheduler:${ version }
   etcd:
     advertisedSubnets:
       - ${nodeSubnets[0]}

azure/templates/worker.yaml.tpl

@@ -6,7 +6,18 @@ machine:
   token: ${tokenMachine}
   ca:
     crt: ${caMachine}
+%{if repository != "registry.k8s.io"}
+  files:
+    - content: |
+        [plugins]
+          [plugins."io.containerd.grpc.v1.cri"]
+            sandbox_image = "${ repository }/pause:3.8"
+      path: /etc/cri/conf.d/20-customization.part
+      op: create
+%{endif}
   kubelet:
+    image: %{if repository == "registry.k8s.io"}ghcr.io/siderolabs%{else}${ repository }%{endif}/kubelet:${ version }
+    defaultRuntimeSeccompProfileEnabled: true
     extraArgs:
       cloud-provider: external
       rotate-server-certificates: true
@@ -59,6 +70,10 @@ machine:
       keys:
         - nodeID: {}
           slot: 0
+  features:
+    rbac: true
+    stableHostname: true
+    apidCheckExtKeyUsage: true
 %{if acrRepo != "" }
   registries:
     config:

azure/variables.tf

@@ -44,6 +44,10 @@ variable "controlplane" {
   description = "Controlplane config"
   type        = map(any)
   default = {
+    "all" = {
+      repository = "registry.k8s.io"
+      version    = "v1.27.3"
+    }
     "uksouth" = {
       count = 0,
       type  = "Standard_B2ms",
@@ -98,6 +102,10 @@ variable "instances" {
   description = "Map of region's properties"
   type        = map(any)
   default = {
+    "all" = {
+      repository = "registry.k8s.io"
+      version    = "v1.27.3"
+    }
     "uksouth" = {
       web_count    = 0,
       web_type     = "Standard_B2s",