From 7c918125e523d4f8d86fee67c00e0bb2c72e9ea0 Mon Sep 17 00:00:00 2001
From: IvanHunters <xorokhotnikov@gmail.com>
Date: Mon, 14 Jul 2025 22:29:25 +0300
Subject: [PATCH] [kubernetes] add check for deployed Kubernetes server version
 using kubectl

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
---
 hack/e2e-apps/kubernetes-latest-version.bats  | 103 +++++++++++++++++
 .../e2e-apps/kubernetes-previous-version.bats | 104 ++++++++++++++++++
 hack/e2e-apps/kubernetes.bats                 |  73 ------------
 3 files changed, 207 insertions(+), 73 deletions(-)
 create mode 100644 hack/e2e-apps/kubernetes-latest-version.bats
 create mode 100644 hack/e2e-apps/kubernetes-previous-version.bats
 delete mode 100644 hack/e2e-apps/kubernetes.bats

diff --git a/hack/e2e-apps/kubernetes-latest-version.bats b/hack/e2e-apps/kubernetes-latest-version.bats
new file mode 100644
index 00000000..4514af16
--- /dev/null
+++ b/hack/e2e-apps/kubernetes-latest-version.bats
@@ -0,0 +1,103 @@
+#!/usr/bin/env bats
+
+@test "Create a tenant Kubernetes control plane" {
+  LATEST_K8S_VERSION=$(yq 'keys | .[-1]' ../packages/apps/kubernetes/files/versions.yaml)
+  TEMPORAL_TENANT_PORT=59991
+
+  kubectl apply -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Kubernetes
+metadata:
+  name: test-latest
+  namespace: tenant-test
+spec:
+  addons:
+    certManager:
+      enabled: false
+      valuesOverride: {}
+    cilium:
+      valuesOverride: {}
+    fluxcd:
+      enabled: false
+      valuesOverride: {}
+    gatewayAPI:
+      enabled: false
+    gpuOperator:
+      enabled: false
+      valuesOverride: {}
+    ingressNginx:
+      enabled: true
+      hosts: []
+      valuesOverride: {}
+    monitoringAgents:
+      enabled: false
+      valuesOverride: {}
+    verticalPodAutoscaler:
+      valuesOverride: {}
+  controlPlane:
+    apiServer:
+      resources: {}
+      resourcesPreset: small
+    controllerManager:
+      resources: {}
+      resourcesPreset: micro
+    konnectivity:
+      server:
+        resources: {}
+        resourcesPreset: micro
+    replicas: 2
+    scheduler:
+      resources: {}
+      resourcesPreset: micro
+  host: ""
+  nodeGroups:
+    md0:
+      ephemeralStorage: 20Gi
+      gpus: []
+      instanceType: u1.medium
+      maxReplicas: 10
+      minReplicas: 0
+      resources:
+        cpu: ""
+        memory: ""
+      roles:
+      - ingress-nginx
+  storageClass: replicated
+  version: "$LATEST_K8S_VERSION"
+EOF
+  # Wait for the tenant-test namespace to be active
+  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+  
+  # Wait for the Kamaji control plane to be created (retry for up to 10 seconds)
+  timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test-latest; do sleep 1; done'
+
+  # Wait for the tenant control plane to be fully created (timeout after 4 minutes)
+  kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test-latest --timeout=4m
+  
+  # Wait for Kubernetes resources to be ready (timeout after 2 minutes)
+  kubectl wait tcp -n tenant-test kubernetes-test-latest --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
+  
+  # Wait for all required deployments to be available (timeout after 4 minutes)
+  kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test-latest kubernetes-test-latest-cluster-autoscaler kubernetes-test-latest-kccm kubernetes-test-latest-kcsi-controller
+  
+  # Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
+  kubectl wait machinedeployment kubernetes-test-latest-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
+
+  # Get the admin kubeconfig and save it to a file
+  kubectl get secret kubernetes-test-latest-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
+
+  # Update the kubeconfig to use localhost for the API server
+  yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig
+
+  # Set up port forwarding to the Kubernetes API server for a 40 second timeout
+  bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-latest -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &'
+
+  # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
+  timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $LATEST_K8S_VERSION"; do sleep 5; done'
+
+  # Wait for all machine deployment replicas to be ready (timeout after 10 minutes)
+  kubectl wait machinedeployment kubernetes-test-latest-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
+
+  # Clean up by deleting the Kubernetes resource
+  kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-latest
+}
diff --git a/hack/e2e-apps/kubernetes-previous-version.bats b/hack/e2e-apps/kubernetes-previous-version.bats
new file mode 100644
index 00000000..c5745d95
--- /dev/null
+++ b/hack/e2e-apps/kubernetes-previous-version.bats
@@ -0,0 +1,104 @@
+#!/usr/bin/env bats
+
+@test "Create a previous version tenant Kubernetes control plane" {
+  # Get latest version of k8s from gcr
+  PREVIOUS_K8S_VERSION=$(yq 'keys | .[-2]' ../packages/apps/kubernetes/files/versions.yaml)
+  TEMPORAL_TENANT_PORT=59992
+
+  kubectl apply -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Kubernetes
+metadata:
+  name: test-previous
+  namespace: tenant-test
+spec:
+  addons:
+    certManager:
+      enabled: false
+      valuesOverride: {}
+    cilium:
+      valuesOverride: {}
+    fluxcd:
+      enabled: false
+      valuesOverride: {}
+    gatewayAPI:
+      enabled: false
+    gpuOperator:
+      enabled: false
+      valuesOverride: {}
+    ingressNginx:
+      enabled: true
+      hosts: []
+      valuesOverride: {}
+    monitoringAgents:
+      enabled: false
+      valuesOverride: {}
+    verticalPodAutoscaler:
+      valuesOverride: {}
+  controlPlane:
+    apiServer:
+      resources: {}
+      resourcesPreset: small
+    controllerManager:
+      resources: {}
+      resourcesPreset: micro
+    konnectivity:
+      server:
+        resources: {}
+        resourcesPreset: micro
+    replicas: 2
+    scheduler:
+      resources: {}
+      resourcesPreset: micro
+  host: ""
+  nodeGroups:
+    md0:
+      ephemeralStorage: 20Gi
+      gpus: []
+      instanceType: u1.medium
+      maxReplicas: 10
+      minReplicas: 0
+      resources:
+        cpu: ""
+        memory: ""
+      roles:
+      - ingress-nginx
+  storageClass: replicated
+  version: "$PREVIOUS_K8S_VERSION"
+EOF
+  # Wait for the tenant-test namespace to be active
+  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+  
+  # Wait for the Kamaji control plane to be created (retry for up to 10 seconds)
+  timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test-previous; do sleep 1; done'
+  
+  # Wait for the tenant control plane to be fully created (timeout after 4 minutes)
+  kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test-previous --timeout=4m
+  
+  # Wait for Kubernetes resources to be ready (timeout after 2 minutes)
+  kubectl wait tcp -n tenant-test kubernetes-test-previous --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
+  
+  # Wait for all required deployments to be available (timeout after 4 minutes)
+  kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test-previous kubernetes-test-previous-cluster-autoscaler kubernetes-test-previous-kccm kubernetes-test-previous-kcsi-controller
+  
+  # Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
+  kubectl wait machinedeployment kubernetes-test-previous-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
+
+  # Get the admin kubeconfig and save it to a file
+  kubectl get secret kubernetes-test-previous-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
+
+  # Update the kubeconfig to use localhost for the API server
+  yq -i ".clusters[0].cluster.server = \"https://localhost:${TEMPORAL_TENANT_PORT}\"" tenantkubeconfig
+
+  # Set up port forwarding to the Kubernetes API server for a 40 second timeout
+  bash -c 'timeout 40s kubectl port-forward service/kubernetes-test-previous -n tenant-test '"${TEMPORAL_TENANT_PORT}"':6443 > /dev/null 2>&1 &'
+
+  # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
+  timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep "Server Version: $PREVIOUS_K8S_VERSION"; do sleep 5; done'
+
+  # Wait for all machine deployment replicas to be ready (timeout after 10 minutes)
+  kubectl wait machinedeployment kubernetes-test-previous-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
+
+  # Clean up by deleting the Kubernetes resource
+  kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test-previous
+}
diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats
deleted file mode 100644
index adaf3660..00000000
--- a/hack/e2e-apps/kubernetes.bats
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env bats
-
-@test "Create a tenant Kubernetes control plane" {
-  kubectl apply -f - <<EOF
-apiVersion: apps.cozystack.io/v1alpha1
-kind: Kubernetes
-metadata:
-  name: test
-  namespace: tenant-test
-spec:
-  addons:
-    certManager:
-      enabled: false
-      valuesOverride: {}
-    cilium:
-      valuesOverride: {}
-    fluxcd:
-      enabled: false
-      valuesOverride: {}
-    gatewayAPI:
-      enabled: false
-    gpuOperator:
-      enabled: false
-      valuesOverride: {}
-    ingressNginx:
-      enabled: true
-      hosts: []
-      valuesOverride: {}
-    monitoringAgents:
-      enabled: false
-      valuesOverride: {}
-    verticalPodAutoscaler:
-      valuesOverride: {}
-  controlPlane:
-    apiServer:
-      resources: {}
-      resourcesPreset: small
-    controllerManager:
-      resources: {}
-      resourcesPreset: micro
-    konnectivity:
-      server:
-        resources: {}
-        resourcesPreset: micro
-    replicas: 2
-    scheduler:
-      resources: {}
-      resourcesPreset: micro
-  host: ""
-  nodeGroups:
-    md0:
-      ephemeralStorage: 20Gi
-      gpus: []
-      instanceType: u1.medium
-      maxReplicas: 10
-      minReplicas: 0
-      resources:
-        cpu: ""
-        memory: ""
-      roles:
-      - ingress-nginx
-  storageClass: replicated
-  version: "v1.30"
-EOF
-  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
-  timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test | grep -q "v1.30"; do sleep 1; done'
-  kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test --timeout=4m
-  kubectl wait tcp -n tenant-test kubernetes-test --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
-  kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test kubernetes-test-cluster-autoscaler kubernetes-test-kccm kubernetes-test-kcsi-controller
-  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
-  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
-  kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test
-}