test: add scale to zero e2e test (#776)

* test: add scale to zero e2e test

Signed-off-by: Mario Valderrama <mario.valderrama@ionos.com>

* fix: retry create token command

* fix: use correct assertion

---------

Signed-off-by: Mario Valderrama <mario.valderrama@ionos.com>

e2e/tcp_scale_test.go

@@ -0,0 +1,68 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Scale a TenantControlPlane resource", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-clusterip-scale",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: pointer.To(int32(1)),
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+
+	// Scale TenantControlPlane resource and check the status
+	It("Should scale correctly", func() {
+		ScaleTenantControlPlane(tcp, 0)
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionSleeping)
+		ScaleTenantControlPlane(tcp, 1)
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+})

e2e/utils_test.go

@@ -121,11 +121,9 @@ func PrintKamajiLogs() {
 }
 
 func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1alpha1.KubernetesVersionStatus) {
+	GinkgoHelper()
 	Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
-		err := k8sClient.Get(context.Background(), types.NamespacedName{
-			Name:      tcp.GetName(),
-			Namespace: tcp.GetNamespace(),
-		}, tcp)
+		err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(tcp), tcp)
 		if err != nil {
 			return ""
 		}
@@ -139,6 +137,7 @@ func StatusMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, status kamajiv1al
 }
 
 func AllPodsLabelMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, label string, value string) {
+	GinkgoHelper()
 	Eventually(func() bool {
 		tcpPods := &corev1.PodList{}
 		err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{
@@ -158,6 +157,7 @@ func AllPodsLabelMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, label strin
 }
 
 func AllPodsAnnotationMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, annotation string, value string) {
+	GinkgoHelper()
 	Eventually(func() bool {
 		tcpPods := &corev1.PodList{}
 		err := k8sClient.List(context.Background(), tcpPods, client.MatchingLabels{
@@ -177,6 +177,7 @@ func AllPodsAnnotationMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, annota
 }
 
 func PodsServiceAccountMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, sa *corev1.ServiceAccount) {
+	GinkgoHelper()
 	saName := sa.GetName()
 	Eventually(func() bool {
 		tcpPods := &corev1.PodList{}
@@ -195,3 +196,10 @@ func PodsServiceAccountMustEqualTo(tcp *kamajiv1alpha1.TenantControlPlane, sa *c
 		return true
 	}, 5*time.Minute, time.Second).Should(BeTrue())
 }
+
+func ScaleTenantControlPlane(tcp *kamajiv1alpha1.TenantControlPlane, replicas int32) {
+	GinkgoHelper()
+	Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(tcp), tcp)).To(Succeed())
+	tcp.Spec.ControlPlane.Deployment.Replicas = &replicas
+	Expect(k8sClient.Update(context.Background(), tcp)).To(Succeed())
+}

e2e/worker_kubeadm_join_test.go

@@ -31,14 +31,14 @@ import (
 var _ = Describe("starting a kind worker with kubeadm", func() {
 	ctx := context.Background()
 
-	var tcp kamajiv1alpha1.TenantControlPlane
+	var tcp *kamajiv1alpha1.TenantControlPlane
 
 	var workerContainer testcontainers.Container
 
 	var kubeconfigFile *os.File
 
 	JustBeforeEach(func() {
-		tcp = kamajiv1alpha1.TenantControlPlane{
+		tcp = &kamajiv1alpha1.TenantControlPlane{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "worker-nodes-join",
 				Namespace: "default",
@@ -69,7 +69,7 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 				Addons: kamajiv1alpha1.AddonsSpec{},
 			},
 		}
-		Expect(k8sClient.Create(ctx, &tcp)).NotTo(HaveOccurred())
+		Expect(k8sClient.Create(ctx, tcp)).NotTo(HaveOccurred())
 
 		var err error
 
@@ -99,24 +99,13 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 
 	JustAfterEach(func() {
 		Expect(workerContainer.Terminate(ctx)).ToNot(HaveOccurred())
-		Expect(k8sClient.Delete(ctx, &tcp)).Should(Succeed())
+		Expect(k8sClient.Delete(ctx, tcp)).Should(Succeed())
 		Expect(os.Remove(kubeconfigFile.Name())).ToNot(HaveOccurred())
 	})
 
 	It("should join the Tenant Control Plane cluster", func() {
 		By("waiting for the Tenant Control Plane being ready", func() {
-			Eventually(func() kamajiv1alpha1.KubernetesVersionStatus {
-				err := k8sClient.Get(ctx, types.NamespacedName{Name: tcp.GetName(), Namespace: tcp.GetNamespace()}, &tcp)
-				if err != nil {
-					return ""
-				}
-
-				if tcp.Status.Kubernetes.Version.Status == nil {
-					return ""
-				}
-
-				return *tcp.Status.Kubernetes.Version.Status
-			}, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady))
+			StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
 		})
 
 		By("downloading Tenant Control Plane kubeconfig", func() {
@@ -139,7 +128,12 @@ var _ = Describe("starting a kind worker with kubeadm", func() {
 			clientset, err := kubernetes.NewForConfig(config)
 			Expect(err).ToNot(HaveOccurred())
 
-			Expect(cmd.RunCreateToken(joinCommandBuffer, clientset, "", &kubeadmv1beta4.InitConfiguration{}, true, "", kubeconfigFile.Name())).ToNot(HaveOccurred())
+			// Soot controller might take a while to set up RBAC.
+			Eventually(func() error {
+				joinCommandBuffer.Reset()
+
+				return cmd.RunCreateToken(joinCommandBuffer, clientset, "", &kubeadmv1beta4.InitConfiguration{}, true, "", kubeconfigFile.Name())
+			}, 1*time.Minute, 1*time.Second).Should(Succeed())
 		})
 
 		By("executing the command in the worker node", func() {