From c7376ef3c9a8164574066e2935d5be9f5b820027 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Thu, 10 Jul 2025 15:47:49 +0300 Subject: [PATCH 01/12] [kubernetes] User-selectable cluster version This patch adds a new version field to the kubernetes chart, letting end-users specify the version of kubernetes they want to deploy. [kubernetes] Let users specify desired version of tenant k8s cluster. Signed-off-by: Timofei Larkin Signed-off-by: IvanHunters --- packages/apps/kubernetes/Chart.yaml | 4 ++-- packages/apps/kubernetes/Makefile | 1 + packages/apps/kubernetes/README.md | 1 + packages/apps/kubernetes/templates/_versions.tpl | 7 +++++++ packages/apps/kubernetes/templates/cluster.yaml | 4 ++-- packages/apps/kubernetes/values.schema.json | 13 +++++++++++++ packages/apps/kubernetes/values.yaml | 3 ++- packages/apps/versions_map | 16 +++++----------- 8 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 packages/apps/kubernetes/templates/_versions.tpl diff --git a/packages/apps/kubernetes/Chart.yaml b/packages/apps/kubernetes/Chart.yaml index c3259ba3..2c49af9b 100644 --- a/packages/apps/kubernetes/Chart.yaml +++ b/packages/apps/kubernetes/Chart.yaml @@ -16,10 +16,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.25.2 +version: 0.26.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 1.32.4 +appVersion: 1.32.6 diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 82ca4e8b..08d55d9e 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -6,6 +6,7 @@ include ../../../scripts/package.mk generate: readme-generator -v values.yaml -s values.schema.json -r README.md + yq -o json -i '.properties.version.enum = ["v1.28","v1.29","v1.30","v1.31","v1.32","v1.33"]' values.schema.json yq -o json -i '.properties.addons.properties.ingressNginx.properties.exposeMethod.enum = ["Proxied","LoadBalancer"]' values.schema.json yq -o json -i '.properties.controlPlane.properties.apiServer.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json yq -o json -i '.properties.controlPlane.properties.controllerManager.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 7a936b83..66f93c09 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -86,6 +86,7 @@ See the reference for components utilized in this service: | `host` | Hostname used to access the Kubernetes cluster externally. Defaults to `.` when empty. | `""` | | `controlPlane.replicas` | Number of replicas for Kubernetes control-plane components. | `2` | | `storageClass` | StorageClass used to store user data. | `replicated` | +| `version` | Kubernetes version given as vMAJOR.MINOR | `v1.32` | | `nodeGroups` | nodeGroups configuration | `{}` | ### Cluster Addons diff --git a/packages/apps/kubernetes/templates/_versions.tpl b/packages/apps/kubernetes/templates/_versions.tpl new file mode 100644 index 00000000..67ef3f8b --- /dev/null +++ b/packages/apps/kubernetes/templates/_versions.tpl @@ -0,0 +1,7 @@ +{{- define "kubernetes.versionMap" }} +{{- $versionMap := dict `v1.28` `v1.28.15` `v1.29` `v1.29.15` `v1.30` `v1.30.14` `v1.31` `v1.31.10` `v1.32` `v1.32.6` `v1.33` `v1.33.2` }} +{{- if not (hasKey $versionMap .) }} +{{- printf `Kubernetes version %s is not supported, allowed versions are %s` . (keys $versionMap) | fail }} +{{- end }} +{{- index $versionMap . }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 7f3144a1..890a985b 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -151,7 +151,7 @@ spec: labels: policy.cozystack.io/allow-to-etcd: "true" replicas: 2 - version: {{ $.Chart.AppVersion }} + version: {{ include "kubernetes.versionMap" $.Values.version }} --- apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor @@ -290,7 +290,7 @@ spec: kind: KubevirtMachineTemplate name: {{ $.Release.Name }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }} namespace: {{ $.Release.Namespace }} - version: v{{ $.Chart.AppVersion }} + version: {{ include "kubernetes.versionMap" $.Values.version }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 9792d03a..7473e00e 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -127,6 +127,19 @@ "description": "StorageClass used to store user data.", "default": "replicated" }, + "version": { + "type": "string", + "description": "Kubernetes version given as vMAJOR.MINOR", + "default": "v1.32", + "enum": [ + "v1.28", + "v1.29", + "v1.30", + "v1.31", + "v1.32", + "v1.33" + ] + }, "addons": { "type": "object", "properties": { diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 68e7c244..c3a846c6 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -3,10 +3,11 @@ ## @param host Hostname used to access the Kubernetes cluster externally. Defaults to `.` when empty. ## @param controlPlane.replicas Number of replicas for Kubernetes control-plane components. ## @param storageClass StorageClass used to store user data. +## @param version Kubernetes version given as vMAJOR.MINOR ## host: "" storageClass: replicated - +version: "v1.32" ## @param nodeGroups [object] nodeGroups configuration ## nodeGroups: diff --git a/packages/apps/versions_map b/packages/apps/versions_map index 368386e0..ac7b8afe 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -55,8 +55,7 @@ kafka 0.7.1 4369b031 kafka 0.8.0 HEAD kubernetes 0.24.0 62cb694d kubernetes 0.25.0 70f82667 -kubernetes 0.25.1 acd4663a -kubernetes 0.25.2 HEAD +kubernetes 0.26.0 HEAD mysql 0.1.0 263e47be mysql 0.2.0 c24a103f mysql 0.3.0 53f2365e @@ -103,9 +102,7 @@ postgres 0.12.0 6130f43d postgres 0.12.1 632224a3 postgres 0.14.0 62cb694d postgres 0.15.1 4369b031 -postgres 0.16.0 70f82667 -postgres 0.17.0 acd4663a -postgres 0.17.1 HEAD +postgres 0.16.0 HEAD rabbitmq 0.1.0 263e47be rabbitmq 0.2.0 53f2365e rabbitmq 0.3.0 6c5cf5bf @@ -139,8 +136,7 @@ tcp-balancer 0.4.1 62cb694d tcp-balancer 0.4.2 4369b031 tcp-balancer 0.5.0 HEAD tenant 1.10.0 4369b031 -tenant 1.11.0 70f82667 -tenant 1.11.1 HEAD +tenant 1.11.0 HEAD virtual-machine 0.1.4 f2015d65 virtual-machine 0.1.5 263e47be virtual-machine 0.2.0 c0685f43 @@ -158,8 +154,7 @@ virtual-machine 0.9.1 93bdf411 virtual-machine 0.10.0 6130f43d virtual-machine 0.10.2 632224a3 virtual-machine 0.11.0 4369b031 -virtual-machine 0.12.0 70f82667 -virtual-machine 0.12.1 HEAD +virtual-machine 0.12.0 HEAD vm-disk 0.1.0 d971f2ff vm-disk 0.1.1 6130f43d vm-disk 0.1.2 632224a3 @@ -176,8 +171,7 @@ vm-instance 0.6.0 721c12a7 vm-instance 0.7.0 6130f43d vm-instance 0.7.2 632224a3 vm-instance 0.8.0 4369b031 -vm-instance 0.9.0 70f82667 -vm-instance 0.10.0 HEAD +vm-instance 0.9.0 HEAD vpn 0.1.0 263e47be vpn 0.2.0 53f2365e vpn 0.3.0 6c5cf5bf From 259a2f5cabb6f550e9bee395d83c9d5f751cb96a Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 11 Jul 2025 22:44:29 +0300 Subject: [PATCH 02/12] [kubernetes] modify tests for user-selectable cluster version case Signed-off-by: IvanHunters --- hack/e2e-apps/kubernetes.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats index 5f3ea777..adaf3660 100644 --- a/hack/e2e-apps/kubernetes.bats +++ b/hack/e2e-apps/kubernetes.bats @@ -60,9 +60,10 @@ spec: roles: - ingress-nginx storageClass: replicated + version: "v1.30" EOF kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active - timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test; do sleep 1; done' + timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test | grep -q "v1.30"; do sleep 1; done' kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test --timeout=4m kubectl wait tcp -n tenant-test kubernetes-test --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test kubernetes-test-cluster-autoscaler kubernetes-test-kccm kubernetes-test-kcsi-controller From d3f1dca1ad08ae169f39b327318e06cfb903e18d Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 14 Jul 2025 20:01:41 +0300 Subject: [PATCH 03/12] generate kubeversions from versions.yaml Signed-off-by: IvanHunters --- packages/apps/kubernetes/Makefile | 2 +- packages/apps/kubernetes/files/versions.yaml | 6 ++++++ packages/apps/kubernetes/templates/_versions.tpl | 9 +++++---- packages/apps/kubernetes/templates/cluster.yaml | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 packages/apps/kubernetes/files/versions.yaml diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 08d55d9e..7d61cb37 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -6,7 +6,7 @@ include ../../../scripts/package.mk generate: readme-generator -v values.yaml -s values.schema.json -r README.md - yq -o json -i '.properties.version.enum = ["v1.28","v1.29","v1.30","v1.31","v1.32","v1.33"]' values.schema.json + yq -o=json -i '.properties.version.enum = (load("files/versions.yaml") | keys)' values.schema.json yq -o json -i '.properties.addons.properties.ingressNginx.properties.exposeMethod.enum = ["Proxied","LoadBalancer"]' values.schema.json yq -o json -i '.properties.controlPlane.properties.apiServer.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json yq -o json -i '.properties.controlPlane.properties.controllerManager.properties.resourcesPreset.enum = ["none","nano","micro","small","medium","large","xlarge","2xlarge"]' values.schema.json diff --git a/packages/apps/kubernetes/files/versions.yaml b/packages/apps/kubernetes/files/versions.yaml new file mode 100644 index 00000000..f9236751 --- /dev/null +++ b/packages/apps/kubernetes/files/versions.yaml @@ -0,0 +1,6 @@ +"v1.28": "v1.28.15" +"v1.29": "v1.29.15" +"v1.30": "v1.30.14" +"v1.31": "v1.31.10" +"v1.32": "v1.32.6" +"v1.33": "v1.33.0" diff --git a/packages/apps/kubernetes/templates/_versions.tpl b/packages/apps/kubernetes/templates/_versions.tpl index 67ef3f8b..8e66d2cd 100644 --- a/packages/apps/kubernetes/templates/_versions.tpl +++ b/packages/apps/kubernetes/templates/_versions.tpl @@ -1,7 +1,8 @@ {{- define "kubernetes.versionMap" }} -{{- $versionMap := dict `v1.28` `v1.28.15` `v1.29` `v1.29.15` `v1.30` `v1.30.14` `v1.31` `v1.31.10` `v1.32` `v1.32.6` `v1.33` `v1.33.2` }} -{{- if not (hasKey $versionMap .) }} -{{- printf `Kubernetes version %s is not supported, allowed versions are %s` . (keys $versionMap) | fail }} +{{- $ := . }} +{{- $versionMap := $.Files.Get "files/versions.yaml" | fromYaml }} +{{- if not (hasKey $versionMap $.Values.version) }} +{{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }} {{- end }} -{{- index $versionMap . }} +{{- index $versionMap $.Values.version }} {{- end }} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 890a985b..7a87660e 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -151,7 +151,7 @@ spec: labels: policy.cozystack.io/allow-to-etcd: "true" replicas: 2 - version: {{ include "kubernetes.versionMap" $.Values.version }} + version: {{ include "kubernetes.versionMap" $ }} --- apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor @@ -290,7 +290,7 @@ spec: kind: KubevirtMachineTemplate name: {{ $.Release.Name }}-{{ $groupName }}-{{ $kubevirtmachinetemplateHash }} namespace: {{ $.Release.Namespace }} - version: {{ include "kubernetes.versionMap" $.Values.version }} + version: {{ include "kubernetes.versionMap" $}} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck From 7c918125e523d4f8d86fee67c00e0bb2c72e9ea0 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 14 Jul 2025 22:29:25 +0300 Subject: [PATCH 04/12] [kubernetes] add check for deployed Kubernetes server version using kubectl Signed-off-by: IvanHunters --- hack/e2e-apps/kubernetes-latest-version.bats | 103 +++++++++++++++++ .../e2e-apps/kubernetes-previous-version.bats | 104 ++++++++++++++++++ hack/e2e-apps/kubernetes.bats | 73 ------------ 3 files changed, 207 insertions(+), 73 deletions(-) create mode 100644 hack/e2e-apps/kubernetes-latest-version.bats create mode 100644 hack/e2e-apps/kubernetes-previous-version.bats delete mode 100644 hack/e2e-apps/kubernetes.bats diff --git a/hack/e2e-apps/kubernetes-latest-version.bats b/hack/e2e-apps/kubernetes-latest-version.bats new file mode 100644 index 00000000..4514af16 --- /dev/null +++ b/hack/e2e-apps/kubernetes-latest-version.bats @@ -0,0 +1,103 @@ +#!/usr/bin/env bats + +@test "Create a tenant Kubernetes control plane" { + LATEST_K8S_VERSION=$(yq 'keys | .[-1]' ../packages/apps/kubernetes/files/versions.yaml) + TEMPORAL_TENANT_PORT=59991 + + kubectl apply -f - < tenantkubeconfig + + # Update the kubeconfig to use localhost for the API server + yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig + + # Set up port forwarding to the Kubernetes API server for a 40 second timeout + bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-latest -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &' + + # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $LATEST_K8S_VERSION"; do sleep 5; done' + + # Wait for all machine deployment replicas to be ready (timeout after 10 minutes) + kubectl wait machinedeployment kubernetes-test-latest-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2 + + # Clean up by deleting the Kubernetes resource + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-latest +} diff --git a/hack/e2e-apps/kubernetes-previous-version.bats b/hack/e2e-apps/kubernetes-previous-version.bats new file mode 100644 index 00000000..c5745d95 --- /dev/null +++ b/hack/e2e-apps/kubernetes-previous-version.bats @@ -0,0 +1,104 @@ +#!/usr/bin/env bats + +@test "Create a previous version tenant Kubernetes control plane" { + # Get latest version of k8s from gcr + PREVIOUS_K8S_VERSION=$(yq 'keys | .[-2]' ../packages/apps/kubernetes/files/versions.yaml) + TEMPORAL_TENANT_PORT=59992 + + kubectl apply -f - < tenantkubeconfig + + # Update the kubeconfig to use localhost for the API server + yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig + + # Set up port forwarding to the Kubernetes API server for a 40 second timeout + bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-previous -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &' + + # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $PREVIOUS_K8S_VERSION"; do sleep 5; done' + + # Wait for all machine deployment replicas to be ready (timeout after 10 minutes) + kubectl wait machinedeployment kubernetes-test-previous-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2 + + # Clean up by deleting the Kubernetes resource + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-previous +} diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats deleted file mode 100644 index adaf3660..00000000 --- a/hack/e2e-apps/kubernetes.bats +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bats - -@test "Create a tenant Kubernetes control plane" { - kubectl apply -f - < Date: Mon, 14 Jul 2025 22:40:51 +0300 Subject: [PATCH 05/12] [kubernetes] add caching for loading kubernetes versions file Signed-off-by: IvanHunters --- packages/apps/kubernetes/templates/_versions.tpl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/apps/kubernetes/templates/_versions.tpl b/packages/apps/kubernetes/templates/_versions.tpl index 8e66d2cd..ee796ddf 100644 --- a/packages/apps/kubernetes/templates/_versions.tpl +++ b/packages/apps/kubernetes/templates/_versions.tpl @@ -1,8 +1,16 @@ {{- define "kubernetes.versionMap" }} {{- $ := . }} -{{- $versionMap := $.Files.Get "files/versions.yaml" | fromYaml }} -{{- if not (hasKey $versionMap $.Values.version) }} -{{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }} + +{{- if not (hasKey $ "cachedVersionMap") }} + {{- $versionMap := $.Files.Get "files/versions.yaml" | fromYaml }} + {{- $_ := set $ "cachedVersionMap" $versionMap }} {{- end }} + +{{- $versionMap := $.cachedVersionMap }} + +{{- if not (hasKey $versionMap $.Values.version) }} + {{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }} +{{- end }} + {{- index $versionMap $.Values.version }} {{- end }} From 7b9a19c94b9a9bbd4af2e529bda7bd7685c950d7 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 14 Jul 2025 23:35:01 +0300 Subject: [PATCH 06/12] [kubernetes] refactoring doubles for difference k8s versions Signed-off-by: IvanHunters --- hack/e2e-apps/kubernetes-latest-version.bats | 103 ------------------ ...-previous-version.bats => kubernetes.bats} | 49 ++++++--- 2 files changed, 31 insertions(+), 121 deletions(-) delete mode 100644 hack/e2e-apps/kubernetes-latest-version.bats rename hack/e2e-apps/{kubernetes-previous-version.bats => kubernetes.bats} (54%) diff --git a/hack/e2e-apps/kubernetes-latest-version.bats b/hack/e2e-apps/kubernetes-latest-version.bats deleted file mode 100644 index 4514af16..00000000 --- a/hack/e2e-apps/kubernetes-latest-version.bats +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bats - -@test "Create a tenant Kubernetes control plane" { - LATEST_K8S_VERSION=$(yq 'keys | .[-1]' ../packages/apps/kubernetes/files/versions.yaml) - TEMPORAL_TENANT_PORT=59991 - - kubectl apply -f - < tenantkubeconfig - - # Update the kubeconfig to use localhost for the API server - yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig - - # Set up port forwarding to the Kubernetes API server for a 40 second timeout - bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-latest -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &' - - # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) - timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $LATEST_K8S_VERSION"; do sleep 5; done' - - # Wait for all machine deployment replicas to be ready (timeout after 10 minutes) - kubectl wait machinedeployment kubernetes-test-latest-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2 - - # Clean up by deleting the Kubernetes resource - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-latest -} diff --git a/hack/e2e-apps/kubernetes-previous-version.bats b/hack/e2e-apps/kubernetes.bats similarity index 54% rename from hack/e2e-apps/kubernetes-previous-version.bats rename to hack/e2e-apps/kubernetes.bats index c5745d95..22f7c125 100644 --- a/hack/e2e-apps/kubernetes-previous-version.bats +++ b/hack/e2e-apps/kubernetes.bats @@ -1,15 +1,16 @@ #!/usr/bin/env bats -@test "Create a previous version tenant Kubernetes control plane" { - # Get latest version of k8s from gcr - PREVIOUS_K8S_VERSION=$(yq 'keys | .[-2]' ../packages/apps/kubernetes/files/versions.yaml) - TEMPORAL_TENANT_PORT=59992 +run_kubernetes_test() { + local version_expr="$1" + local test_name="$2" + local port="$3" + local k8s_version=$(yq "$version_expr" ../packages/apps/kubernetes/files/versions.yaml) kubectl apply -f - < tenantkubeconfig + kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig # Update the kubeconfig to use localhost for the API server - yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig + yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig # Set up port forwarding to the Kubernetes API server for a 40 second timeout - bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-previous -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &' + bash -c 'timeout 40s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &' # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) - timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $PREVIOUS_K8S_VERSION"; do sleep 5; done' + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -q "Server Version: ${k8s_version}"; do sleep 5; done' # Wait for all machine deployment replicas to be ready (timeout after 10 minutes) - kubectl wait machinedeployment kubernetes-test-previous-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2 + kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2 # Clean up by deleting the Kubernetes resource - kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-previous + kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name + +} + +@test "Create a tenant Kubernetes control plane with latest version" { + echo "#################_TESTING_OF_LATEST_VERSION_###################" + run_kubernetes_test 'keys | sort_by(.) | .[-1]' 'test-latest-version' '59991' + echo "#############################_END_#############################" +} +@test "Create a tenant Kubernetes control plane with previous version" { + echo "#################_TESTING_OF_PREVIOUS_VERSION_#################" + run_kubernetes_test 'keys | sort_by(.) | .[-2]' 'test-previous-version' '59992' + echo "#############################_END_#############################" } From d63aac727c2415c586d5e7f91ebb974512c463f5 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 14 Jul 2025 23:44:24 +0300 Subject: [PATCH 07/12] [kubernetes] refactoring checking k8s version for nitpick comment by rabbit Signed-off-by: IvanHunters --- hack/e2e-apps/kubernetes.bats | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats index 22f7c125..54e4ad10 100644 --- a/hack/e2e-apps/kubernetes.bats +++ b/hack/e2e-apps/kubernetes.bats @@ -95,7 +95,7 @@ EOF bash -c 'timeout 40s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &' # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) - timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -q "Server Version: ${k8s_version}"; do sleep 5; done' + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done' # Wait for all machine deployment replicas to be ready (timeout after 10 minutes) kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2 @@ -106,12 +106,8 @@ EOF } @test "Create a tenant Kubernetes control plane with latest version" { - echo "#################_TESTING_OF_LATEST_VERSION_###################" run_kubernetes_test 'keys | sort_by(.) | .[-1]' 'test-latest-version' '59991' - echo "#############################_END_#############################" } @test "Create a tenant Kubernetes control plane with previous version" { - echo "#################_TESTING_OF_PREVIOUS_VERSION_#################" run_kubernetes_test 'keys | sort_by(.) | .[-2]' 'test-previous-version' '59992' - echo "#############################_END_#############################" } From 6023dffd6de20914ab4da4f0f91652ddb046354d Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 15 Jul 2025 13:04:27 +0300 Subject: [PATCH 08/12] [tests] fix versions.yaml path Signed-off-by: IvanHunters --- hack/e2e-apps/kubernetes.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats index 54e4ad10..9d71d912 100644 --- a/hack/e2e-apps/kubernetes.bats +++ b/hack/e2e-apps/kubernetes.bats @@ -4,7 +4,7 @@ run_kubernetes_test() { local version_expr="$1" local test_name="$2" local port="$3" - local k8s_version=$(yq "$version_expr" ../packages/apps/kubernetes/files/versions.yaml) + local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) kubectl apply -f - < Date: Tue, 15 Jul 2025 13:04:27 +0300 Subject: [PATCH 09/12] [tests] fix versions.yaml path Signed-off-by: IvanHunters --- hack/e2e-apps/kubernetes.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats index 54e4ad10..9d71d912 100644 --- a/hack/e2e-apps/kubernetes.bats +++ b/hack/e2e-apps/kubernetes.bats @@ -4,7 +4,7 @@ run_kubernetes_test() { local version_expr="$1" local test_name="$2" local port="$3" - local k8s_version=$(yq "$version_expr" ../packages/apps/kubernetes/files/versions.yaml) + local k8s_version=$(yq "$version_expr" packages/apps/kubernetes/files/versions.yaml) kubectl apply -f - < Date: Tue, 15 Jul 2025 14:43:19 +0300 Subject: [PATCH 10/12] Fixing versions_map Signed-off-by: Timofei Larkin --- packages/apps/versions_map | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/apps/versions_map b/packages/apps/versions_map index ac7b8afe..e5c23e6e 100644 --- a/packages/apps/versions_map +++ b/packages/apps/versions_map @@ -102,7 +102,8 @@ postgres 0.12.0 6130f43d postgres 0.12.1 632224a3 postgres 0.14.0 62cb694d postgres 0.15.1 4369b031 -postgres 0.16.0 HEAD +postgres 0.16.0 70f82667 +postgres 0.17.1 HEAD rabbitmq 0.1.0 263e47be rabbitmq 0.2.0 53f2365e rabbitmq 0.3.0 6c5cf5bf @@ -136,7 +137,8 @@ tcp-balancer 0.4.1 62cb694d tcp-balancer 0.4.2 4369b031 tcp-balancer 0.5.0 HEAD tenant 1.10.0 4369b031 -tenant 1.11.0 HEAD +tenant 1.11.0 08cb7c0f +tenant 1.11.1 HEAD virtual-machine 0.1.4 f2015d65 virtual-machine 0.1.5 263e47be virtual-machine 0.2.0 c0685f43 @@ -154,7 +156,8 @@ virtual-machine 0.9.1 93bdf411 virtual-machine 0.10.0 6130f43d virtual-machine 0.10.2 632224a3 virtual-machine 0.11.0 4369b031 -virtual-machine 0.12.0 HEAD +virtual-machine 0.12.0 acd4663a +virtual-machine 0.12.1 HEAD vm-disk 0.1.0 d971f2ff vm-disk 0.1.1 6130f43d vm-disk 0.1.2 632224a3 @@ -171,7 +174,8 @@ vm-instance 0.6.0 721c12a7 vm-instance 0.7.0 6130f43d vm-instance 0.7.2 632224a3 vm-instance 0.8.0 4369b031 -vm-instance 0.9.0 HEAD +vm-instance 0.9.0 acd4663a +vm-instance 0.10.0 HEAD vpn 0.1.0 263e47be vpn 0.2.0 53f2365e vpn 0.3.0 6c5cf5bf From 7b87d555e415d0763b72e1d0c7f31baf37ee9efb Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 16 Jul 2025 17:36:42 +0300 Subject: [PATCH 11/12] [review] disable caching and remove reusing root context Signed-off-by: IvanHunters --- packages/apps/kubernetes/templates/_versions.tpl | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/apps/kubernetes/templates/_versions.tpl b/packages/apps/kubernetes/templates/_versions.tpl index ee796ddf..45233827 100644 --- a/packages/apps/kubernetes/templates/_versions.tpl +++ b/packages/apps/kubernetes/templates/_versions.tpl @@ -1,16 +1,9 @@ {{- define "kubernetes.versionMap" }} -{{- $ := . }} -{{- if not (hasKey $ "cachedVersionMap") }} - {{- $versionMap := $.Files.Get "files/versions.yaml" | fromYaml }} - {{- $_ := set $ "cachedVersionMap" $versionMap }} -{{- end }} - -{{- $versionMap := $.cachedVersionMap }} - -{{- if not (hasKey $versionMap $.Values.version) }} +{{- $versionMap := .Files.Get "files/versions.yaml" | fromYaml }} +{{- if not (hasKey $versionMap .Values.version) }} {{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }} {{- end }} -{{- index $versionMap $.Values.version }} +{{- index $versionMap .Values.version }} {{- end }} From 5b6ebbc7964122511e63244cd685ea3d4525c616 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 16 Jul 2025 17:41:15 +0300 Subject: [PATCH 12/12] [review] compact _versions.tpl Signed-off-by: IvanHunters --- packages/apps/kubernetes/templates/_versions.tpl | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/apps/kubernetes/templates/_versions.tpl b/packages/apps/kubernetes/templates/_versions.tpl index 45233827..fcd4cee1 100644 --- a/packages/apps/kubernetes/templates/_versions.tpl +++ b/packages/apps/kubernetes/templates/_versions.tpl @@ -1,9 +1,7 @@ {{- define "kubernetes.versionMap" }} - {{- $versionMap := .Files.Get "files/versions.yaml" | fromYaml }} {{- if not (hasKey $versionMap .Values.version) }} {{- printf `Kubernetes version %s is not supported, allowed versions are %s` $.Values.version (keys $versionMap) | fail }} {{- end }} - {{- index $versionMap .Values.version }} {{- end }}