diff --git a/hack/e2e-apps/run-kubernetes.sh b/hack/e2e-apps/run-kubernetes.sh index 1446b093..f8b20b85 100644 --- a/hack/e2e-apps/run-kubernetes.sh +++ b/hack/e2e-apps/run-kubernetes.sh @@ -80,58 +80,41 @@ EOF # Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute) kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2 # Get the admin kubeconfig and save it to a file - kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig + kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig-${test_name} # Update the kubeconfig to use localhost for the API server - yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig + yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig-${test_name} # Set up port forwarding to the Kubernetes API server for a 200 second timeout bash -c 'timeout 300s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &' # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds) - timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done' + timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done' # Wait for the nodes to be ready (timeout after 2 minutes) timeout 3m bash -c ' - until [ "$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do + until [ "$(kubectl --kubeconfig tenantkubeconfig-'"${test_name}"' get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do sleep 2 done ' # Verify the nodes are ready - kubectl --kubeconfig tenantkubeconfig wait node --all --timeout=2m --for=condition=Ready - kubectl --kubeconfig tenantkubeconfig get nodes -o wide + kubectl --kubeconfig tenantkubeconfig-${test_name} wait node --all --timeout=2m --for=condition=Ready + kubectl --kubeconfig tenantkubeconfig-${test_name} get nodes -o wide # Verify the kubelet version matches what we expect - versions=$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}') + versions=$(kubectl --kubeconfig "tenantkubeconfig-${test_name}" \ + get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}') + node_ok=true - - case "$k8s_version" in - v1.32*) - echo "⚠️ TODO: Temporary stub — allowing nodes with v1.33 while k8s_version is v1.32" - ;; - esac - + for v in $versions; do - case "$k8s_version" in - v1.32|v1.32.*) - case "$v" in - v1.32 | v1.32.* | v1.32-* | v1.33 | v1.33.* | v1.33-*) - ;; - *) - node_ok=false - break - ;; - esac + case "$v" in + "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*) + # acceptable ;; *) - case "$v" in - "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*) - ;; - *) - node_ok=false - break - ;; - esac + node_ok=false + break ;; esac done diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 2343a90a..7edd07f5 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -182,6 +182,33 @@ metadata: spec: template: spec: + files: + - path: /usr/bin/update-k8s.sh + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + # Expected to be passed in via preKubeadmCommands + : "${KUBELET_VERSION:?KUBELET_VERSION must be set, e.g. v1.31.0}" + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) ARCH=amd64 ;; + aarch64) ARCH=arm64 ;; + esac + + # Use your internal mirror here for real-world use. + BASE_URL="https://dl.k8s.io/release/${KUBELET_VERSION}/bin/linux/${ARCH}" + + echo "Installing kubelet and kubeadm ${KUBELET_VERSION} for ${ARCH}..." + curl -fsSL "${BASE_URL}/kubelet" -o /root/kubelet + curl -fsSL "${BASE_URL}/kubeadm" -o /root/kubeadm + chmod 0755 /root/kubelet + chmod 0755 /root/kubeadm + if /root/kubelet --version ; then mv /root/kubelet /usr/bin/kubelet ; fi + if /root/kubeadm version ; then mv /root/kubeadm /usr/bin/kubeadm ; fi diskSetup: filesystems: - device: /dev/vdb @@ -205,6 +232,7 @@ spec: {{- end }} {{- end }} preKubeadmCommands: + - KUBELET_VERSION={{ include "kubernetes.versionMap" $}} /usr/bin/update-k8s.sh || true - sed -i 's|root:x:|root::|' /etc/passwd - systemctl stop containerd.service - mkdir -p /ephemeral/kubelet /ephemeral/containerd