From d65d293fbc41165497c3131b81ba4792f2ecba43 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Sat, 8 Nov 2025 13:02:42 +0300 Subject: [PATCH] [kubernetes] Make worker version configurable ## What this PR does The kubelet version of tenant k8s clusters is baked into the worker VM image. Previously, selecting any version of tenant k8s had an impact only on the controlplane, the workers were fixed at v1.33. This patch modifies the KubeadmConfigTemplate to attempt to download the user-selected versions of kubelet and kubeadm and replace the baked-in versions with those. If failing, the bootstrap continues with the baked-in versions. ### Release note ```release-note [kubernetes] Make kubelet versions on tenant k8s clusters' worker nodes user-configurable. ``` Signed-off-by: Timofei Larkin --- hack/e2e-apps/run-kubernetes.sh | 47 ++++++------------- .../apps/kubernetes/templates/cluster.yaml | 28 +++++++++++ 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 1446b093..f8b20b85 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -80,58 +80,41 @@ EOF # Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute) kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2 # Get the admin kubeconfig and save it to a file - kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig + kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig-${test_name} # Update the kubeconfig to use localhost for the API server - yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig + yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig-${test_name} # Set up port forwarding to the Kubernetes API server for a 200 second timeout bash -c 'timeout 300s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &' # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) - timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done' + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done' # Wait for the nodes to be ready (timeout after 2 minutes) timeout 3m bash -c ' - until [ "$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do + until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do sleep 2 done ' # Verify the nodes are ready - kubectl --kubeconfig tenantkubeconfig wait node --all --timeout=2m --for=condition=Ready - kubectl --kubeconfig tenantkubeconfig get nodes -o wide + kubectl --kubeconfig tenantkubeconfig-${test_name} wait node --all --timeout=2m --for=condition=Ready + kubectl --kubeconfig tenantkubeconfig-${test_name} get nodes -o wide # Verify the kubelet version matches what we expect - versions=$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}') + versions=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" \ + get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}') + node_ok=true - - case "$k8s_version" in - v1.32*) - echo "⚠️ TODO: Temporary stub — allowing nodes with v1.33 while k8s_version is v1.32" - ;; - esac - + for v in $versions; do - case "$k8s_version" in - v1.32|v1.32.*) - case "$v" in - v1.32 | v1.32.* | v1.32-* | v1.33 | v1.33.* | v1.33-*) - ;; - *) - node_ok=false - break - ;; - esac + case "$v" in + "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*) + # acceptable ;; *) - case "$v" in - "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*) - ;; - *) - node_ok=false - break - ;; - esac + node_ok=false + break ;; esac done diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 2343a90a..7edd07f5 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -182,6 +182,33 @@ metadata: spec: template: spec: + files: + - path: /usr/bin/update-k8s.sh + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + # Expected to be passed in via preKubeadmCommands + : "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}" + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + esac + + # Use your internal mirror here for real-world use. + BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}" + + echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..." + curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet + curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm + chmod 0755 /root/kubelet + chmod 0755 /root/kubeadm + if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi + if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi diskSetup: filesystems: - device: /dev/vdb @@ -205,6 +232,7 @@ spec: {{- end }} {{- end }} preKubeadmCommands: + - KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true - sed -i 's|root:x:|root::|' /etc/passwd - systemctl stop containerd.service - mkdir -p /ephemeral/kubelet /ephemeral/containerd