From c769226e79dc1316b9f3db0242a3a4e1a0760f62 Mon Sep 17 00:00:00 2001 From: Mario Valderrama <15158349+avorima@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:20:21 +0200 Subject: [PATCH] test: add scale to zero e2e test (#776) * test: add scale to zero e2e test Signed-off-by: Mario Valderrama * fix: retry create token command * fix: use correct assertion --------- Signed-off-by: Mario Valderrama --- e2e/tcp_scale_test.go | 68 +++++++++++++++++++++++++++++++++ e2e/utils_test.go | 16 ++++++-- e2e/worker_kubeadm_join_test.go | 28 ++++++-------- 3 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 e2e/tcp_scale_test.go diff --git a/e2e/tcp_scale_test.go b/e2e/tcp_scale_test.go new file mode 100644 index 0000000..ea1260c --- /dev/null +++ b/e2e/tcp_scale_test.go @@ -0,0 +1,68 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + pointer "k8s.io/utils/ptr" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Scale a TenantControlPlane resource", func() { + // Fill TenantControlPlane object + tcp := &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-clusterip-scale", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: pointer.To(int32(1)), + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "ClusterIP", + }, + }, + NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{ + Address: "172.18.0.2", + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.23.6", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + AdmissionControllers: kamajiv1alpha1.AdmissionControllers{ + "LimitRanger", + "ResourceQuota", + }, + }, + Addons: kamajiv1alpha1.AddonsSpec{}, + }, + } + + // Create a TenantControlPlane resource into the cluster + JustBeforeEach(func() { + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) + + // Delete the TenantControlPlane resource after test is finished + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + }) + + // Scale TenantControlPlane resource and check the status + It("Should scale correctly", func() { + ScaleTenantControlPlane(tcp, 0) + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionSleeping) + ScaleTenantControlPlane(tcp, 1) + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) +}) diff --git a/e2e/utils_test.go b/e2e/utils_test.go index baddfc6..b29b340 100644 --- a/e2e/utils_test.go +++ b/e2e/utils_test.go @@ -121,11 +121,9 @@ func PrintKamajiLogs() { } func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1alpha1.KubernetesVersionStatus) { + GinkgoHelper() Eventually(func() kamajiv1alpha1.KubernetesVersionStatus { - err := k8sClient.Get(context.Background(), types.NamespacedName{ - Name: tcp.GetName(), - Namespace: tcp.GetNamespace(), - }, tcp) + err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(tcp), tcp) if err != nil { return "" } @@ -139,6 +137,7 @@ func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1al } func AllPodsLabelMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, label string, value string) { + GinkgoHelper() Eventually(func() bool { tcpPods := &corev1.PodList{} err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{ @@ -158,6 +157,7 @@ func AllPodsLabelMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, label strin } func AllPodsAnnotationMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, annotation string, value string) { + GinkgoHelper() Eventually(func() bool { tcpPods := &corev1.PodList{} err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{ @@ -177,6 +177,7 @@ func AllPodsAnnotationMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, annota } func PodsServiceAccountMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, sa *corev1.ServiceAccount) { + GinkgoHelper() saName := sa.GetName() Eventually(func() bool { tcpPods := &corev1.PodList{} @@ -195,3 +196,10 @@ func PodsServiceAccountMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, sa *c return true }, 5*time.Minute, time.Second).Should(BeTrue()) } + +func ScaleTenantControlPlane(tcp *kamajiv1alpha1.TenantControlPlane, replicas int32) { + GinkgoHelper() + Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(tcp), tcp)).To(Succeed()) + tcp.Spec.ControlPlane.Deployment.Replicas = &replicas + Expect(k8sClient.Update(context.Background(), tcp)).To(Succeed()) +} diff --git a/e2e/worker_kubeadm_join_test.go b/e2e/worker_kubeadm_join_test.go index a99a660..f89b900 100644 --- a/e2e/worker_kubeadm_join_test.go +++ b/e2e/worker_kubeadm_join_test.go @@ -31,14 +31,14 @@ import ( var _ = Describe("starting a kind worker with kubeadm", func() { ctx := context.Background() - var tcp kamajiv1alpha1.TenantControlPlane + var tcp *kamajiv1alpha1.TenantControlPlane var workerContainer testcontainers.Container var kubeconfigFile *os.File JustBeforeEach(func() { - tcp = kamajiv1alpha1.TenantControlPlane{ + tcp = &kamajiv1alpha1.TenantControlPlane{ ObjectMeta: metav1.ObjectMeta{ Name: "worker-nodes-join", Namespace: "default", @@ -69,7 +69,7 @@ var _ = Describe("starting a kind worker with kubeadm", func() { Addons: kamajiv1alpha1.AddonsSpec{}, }, } - Expect(k8sClient.Create(ctx, &tcp)).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, tcp)).NotTo(HaveOccurred()) var err error @@ -99,24 +99,13 @@ var _ = Describe("starting a kind worker with kubeadm", func() { JustAfterEach(func() { Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred()) - Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed()) Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred()) }) It("should join the Tenant Control Plane cluster", func() { By("waiting for the Tenant Control Plane being ready", func() { - Eventually(func() kamajiv1alpha1.KubernetesVersionStatus { - err := k8sClient.Get(ctx, types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, &tcp) - if err != nil { - return "" - } - - if tcp.Status.Kubernetes.Version.Status == nil { - return "" - } - - return *tcp.Status.Kubernetes.Version.Status - }, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady)) + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) }) By("downloading Tenant Control Plane kubeconfig", func() { @@ -139,7 +128,12 @@ var _ = Describe("starting a kind worker with kubeadm", func() { clientset, err := kubernetes.NewForConfig(config) Expect(err).ToNot(HaveOccurred()) - Expect(cmd.RunCreateToken(joinCommandBuffer, clientset, "", &kubeadmv1beta4.InitConfiguration{}, true, "", kubeconfigFile.Name())).ToNot(HaveOccurred()) + // Soot controller might take a while to set up RBAC. + Eventually(func() error { + joinCommandBuffer.Reset() + + return cmd.RunCreateToken(joinCommandBuffer, clientset, "", &kubeadmv1beta4.InitConfiguration{}, true, "", kubeconfigFile.Name()) + }, 1*time.Minute, 1*time.Second).Should(Succeed()) }) By("executing the command in the worker node", func() {