[kubernetes] Make worker version configurable (#1619)

## What this PR does

The kubelet version of tenant k8s clusters is baked into the worker VM
image. Previously, selecting any version of tenant k8s had an impact
only on the controlplane, the workers were fixed at v1.33. This patch
modifies the KubeadmConfigTemplate to attempt to download the
user-selected versions of kubelet and kubeadm and replace the baked-in
versions with those. If failing, the bootstrap continues with the
baked-in versions.

### Release note

```release-note
[kubernetes] Make kubelet versions on tenant k8s clusters' worker nodes
user-configurable.
```

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Automated Kubernetes component update during bootstrap with x86_64 and
ARM64 support; invoked as part of pre-bootstrap steps to ensure
kubelet/kubeadm versions.

* **Tests**
  * Per-test isolated kubeconfig filenames to avoid conflicts.
* Simplified, stricter per-node version validation and alignment of
readiness checks; increased machine deployment readiness timeout to 10
minutes.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

hack/e2e-apps/run-kubernetes.sh

@@ -80,58 +80,41 @@ EOF
   # Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
   kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
   # Get the admin kubeconfig and save it to a file
-  kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
+  kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig-${test_name}
 
   # Update the kubeconfig to use localhost for the API server
-  yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig
+  yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig-${test_name}
 
 
   # Set up port forwarding to the Kubernetes API server for a 200 second timeout
   bash -c 'timeout 300s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &'
   # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
-  timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
+  timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
 
   # Wait for the nodes to be ready (timeout after 2 minutes)
   timeout 3m bash -c '
-    until [ "$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do
+    until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do
       sleep 2
     done
   '
   # Verify the nodes are ready
-  kubectl --kubeconfig tenantkubeconfig wait node --all --timeout=2m --for=condition=Ready
-  kubectl --kubeconfig tenantkubeconfig get nodes -o wide
+  kubectl --kubeconfig tenantkubeconfig-${test_name} wait node --all --timeout=2m --for=condition=Ready
+  kubectl --kubeconfig tenantkubeconfig-${test_name} get nodes -o wide
 
   # Verify the kubelet version matches what we expect
-  versions=$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}')
+  versions=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" \
+    get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}')
+  
   node_ok=true
-
-  case "$k8s_version" in
-    v1.32*)
-      echo "⚠️  TODO: Temporary stub — allowing nodes with v1.33 while k8s_version is v1.32"
-      ;;
-  esac
-
+  
   for v in $versions; do
-    case "$k8s_version" in
-      v1.32|v1.32.*)
-        case "$v" in
-          v1.32 | v1.32.* | v1.32-* | v1.33 | v1.33.* | v1.33-*)
-            ;;
-          *)
-            node_ok=false
-            break
-            ;;
-        esac
+    case "$v" in
+      "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*)
+        # acceptable
         ;;
       *)
-        case "$v" in
-          "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*)
-            ;;
-          *)
-            node_ok=false
-            break
-            ;;
-        esac
+        node_ok=false
+        break
         ;;
     esac
   done

packages/apps/kubernetes/templates/cluster.yaml

@@ -182,6 +182,33 @@ metadata:
 spec:
   template:
     spec:
+      files:
+      - path: /usr/bin/update-k8s.sh
+        owner: root:root
+        permissions: "0755"
+        content: |
+          #!/usr/bin/env bash
+          set -euo pipefail
+
+          # Expected to be passed in via preKubeadmCommands
+          : "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}"
+
+          ARCH="$(uname -m)"
+          case "${ARCH}" in
+            x86_64) ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+          esac
+
+          # Use your internal mirror here for real-world use.
+          BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}"
+
+          echo "Installing kubelet and kubeadm  ${KUBELET_VERSION} for ${ARCH}..."
+          curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet
+          curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm
+          chmod 0755 /root/kubelet
+          chmod 0755 /root/kubeadm
+          if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi
+          if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi
       diskSetup:
         filesystems:
         - device: /dev/vdb
@@ -205,6 +232,7 @@ spec:
         {{- end }}
       {{- end }}
       preKubeadmCommands:
+      - KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true
       - sed -i 's|root:x:|root::|' /etc/passwd
       - systemctl stop containerd.service
       - mkdir -p /ephemeral/kubelet /ephemeral/containerd