feat/impruvement-kubernetes-tests (#1485)

<!-- Thank you for making a contribution! Here are some tips for you:
- Start the PR title with the [label] of Cozystack component:
- For system components: [platform], [system], [linstor], [cilium],
[kube-ovn], [dashboard], [cluster-api], etc.
- For managed apps: [apps], [tenant], [kubernetes], [postgres],
[virtual-machine] etc.
- For development and maintenance: [tests], [ci], [docs], [maintenance].
- If it's a work in progress, consider creating this PR as a draft.
- Don't hesistate to ask for opinion and review in the community chats,
even if it's still a draft.
- Add the label `backport` if it's a bugfix that needs to be backported
to a previous version.
-->

## What this PR does


### Release note

<!--  Write a release note:
- Explain what has changed internally and for users.
- Start with the same [label] as in the PR title
- Follow the guidelines at
https://github.com/kubernetes/community/blob/master/contributors/guide/release-notes.md.
-->

```release-note
Improved tests for verifying installed kubernetes client clusters
```

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* New Features
* Added node readiness checks (expected node count, detailed node
display) and kubelet version validation with compatibility handling.

* Improvements
* Increased API port-forward timeout and extended rollout/machine
deployment waits for more reliable rollouts.
  * Added per-component readiness waits for core cluster services.

* Chores
  * Bumped default Kubernetes version to v1.33.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

hack/e2e-apps/run-kubernetes.sh

@@ -79,22 +79,73 @@ EOF
   
   # Wait for the machine deployment to scale to 2 replicas (timeout after 1 minute)
   kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
-
   # Get the admin kubeconfig and save it to a file
   kubectl get secret kubernetes-${test_name}-admin-kubeconfig -ojsonpath='{.data.super-admin\.conf}' -n tenant-test | base64 -d > tenantkubeconfig
 
   # Update the kubeconfig to use localhost for the API server
   yq -i ".clusters[0].cluster.server = \"https://localhost:${port}\"" tenantkubeconfig
 
-  # Set up port forwarding to the Kubernetes API server for a 40 second timeout
-  bash -c 'timeout 40s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &'
 
+  # Set up port forwarding to the Kubernetes API server for a 200 second timeout
+  bash -c 'timeout 200s kubectl port-forward service/kubernetes-'"${test_name}"' -n tenant-test '"${port}"':6443 > /dev/null 2>&1 &'
   # Verify the Kubernetes version matches what we expect (retry for up to 20 seconds)
   timeout 20 sh -ec 'until kubectl --kubeconfig tenantkubeconfig version 2>/dev/null | grep -Fq "Server Version: ${k8s_version}"; do sleep 5; done'
 
+  # Wait for the nodes to be ready (timeout after 2 minutes)
+  timeout 2m bash -c '
+    until [ "$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath="{.items[*].metadata.name}" | wc -w)" -eq 2 ]; do
+      sleep 3
+    done
+  '
+  # Verify the nodes are ready
+  kubectl --kubeconfig tenantkubeconfig wait node --all --timeout=2m --for=condition=Ready
+  kubectl --kubeconfig tenantkubeconfig get nodes -o wide
+
+  # Verify the kubelet version matches what we expect
+  versions=$(kubectl --kubeconfig tenantkubeconfig get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}')
+  node_ok=true
+
+  if [[ "$k8s_version" == v1.32* ]]; then
+    echo "⚠️  TODO: Temporary stub — allowing nodes with v1.33 while k8s_version is v1.32"
+  fi
+
+  for v in $versions; do
+    case "$k8s_version" in
+      v1.32|v1.32.*)
+        case "$v" in
+          v1.32 | v1.32.* | v1.32-* | v1.33 | v1.33.* | v1.33-*)
+            ;;
+          *)
+            node_ok=false
+            break
+            ;;
+        esac
+        ;;
+      *)
+        case "$v" in
+          "${k8s_version}" | "${k8s_version}".* | "${k8s_version}"-*)
+            ;;
+          *)
+            node_ok=false
+            break
+            ;;
+        esac
+        ;;
+    esac
+  done
+
+  if ! $node_ok; then
+    echo "Kubelet versions did not match expected ${k8s_version}" >&2
+    exit 1
+  fi
+
   # Wait for all machine deployment replicas to be ready (timeout after 10 minutes)
   kubectl wait machinedeployment kubernetes-${test_name}-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
 
+  for component in cilium coredns csi ingress-nginx vsnap-crd; do
+      kubectl wait hr kubernetes-${test_name}-${component} -n tenant-test --timeout=1m --for=condition=ready
+    done
+
   # Clean up by deleting the Kubernetes resource
   kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $test_name
 

packages/apps/kubernetes/Makefile

@@ -1,4 +1,4 @@
-KUBERNETES_VERSION = v1.32
+KUBERNETES_VERSION = v1.33
 KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml)
 
 include ../../../scripts/common-envs.mk