From 74e7e5cdfb5c192c03a3a43ea564607014324aa1 Mon Sep 17 00:00:00 2001
From: Andrei Kvapil <kvapss@gmail.com>
Date: Sat, 24 May 2025 22:42:54 +0200
Subject: [PATCH] [tests] Add tests for tenant Kubernetes cluster

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
---
 hack/e2e-apps.bats                  | 94 +++++++++++++++++++++++++++++
 hack/{e2e.bats => e2e-cluster.bats} |  8 +--
 packages/core/testing/Makefile      | 16 ++++-
 3 files changed, 111 insertions(+), 7 deletions(-)
 create mode 100755 hack/e2e-apps.bats
 rename hack/{e2e.bats => e2e-cluster.bats} (98%)

diff --git a/hack/e2e-apps.bats b/hack/e2e-apps.bats
new file mode 100755
index 00000000..9610a64a
--- /dev/null
+++ b/hack/e2e-apps.bats
@@ -0,0 +1,94 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# Cozystack end‑to‑end provisioning test (Bats)
+# -----------------------------------------------------------------------------
+
+@test "Create tenant with isolated mode enabled" {
+  kubectl create -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Tenant
+metadata:
+  name: test
+  namespace: tenant-root
+spec:
+  etcd: false
+  host: ""
+  ingress: false
+  isolated: true
+  monitoring: false
+  resourceQuotas: {}
+  seaweedfs: false
+EOF
+  kubectl wait hr/tenant-test -n tenant-root --timeout=1m --for=condition=ready
+  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+}
+
+@test "Create a tenant Kubernetes control plane" {
+  kubectl create -f - <<EOF
+apiVersion: apps.cozystack.io/v1alpha1
+kind: Kubernetes
+metadata:
+  name: test
+  namespace: tenant-test
+spec:
+  addons:
+    certManager:
+      enabled: false
+      valuesOverride: {}
+    cilium:
+      valuesOverride: {}
+    fluxcd:
+      enabled: false
+      valuesOverride: {}
+    gatewayAPI:
+      enabled: false
+    gpuOperator:
+      enabled: false
+      valuesOverride: {}
+    ingressNginx:
+      enabled: true
+      hosts: []
+      valuesOverride: {}
+    monitoringAgents:
+      enabled: false
+      valuesOverride: {}
+    verticalPodAutoscaler:
+      valuesOverride: {}
+  controlPlane:
+    apiServer:
+      resources: {}
+      resourcesPreset: small
+    controllerManager:
+      resources: {}
+      resourcesPreset: micro
+    konnectivity:
+      server:
+        resources: {}
+        resourcesPreset: micro
+    replicas: 2
+    scheduler:
+      resources: {}
+      resourcesPreset: micro
+  host: ""
+  nodeGroups:
+    md0:
+      ephemeralStorage: 20Gi
+      gpus: []
+      instanceType: u1.medium
+      maxReplicas: 10
+      minReplicas: 0
+      resources:
+        cpu: ""
+        memory: ""
+      roles:
+      - ingress-nginx
+  storageClass: replicated
+EOF
+  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
+  timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test; do sleep 1; done'
+  kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test --timeout=4m
+  kubectl wait tcp -n tenant-test kubernetes-test --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
+  kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test kubernetes-test-cluster-autoscaler kubernetes-test-kccm kubernetes-test-kcsi-controller
+  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
+  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=5m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
+}
diff --git a/hack/e2e.bats b/hack/e2e-cluster.bats
similarity index 98%
rename from hack/e2e.bats
rename to hack/e2e-cluster.bats
index 3e782162..39e2aff4 100755
--- a/hack/e2e.bats
+++ b/hack/e2e-cluster.bats
@@ -3,9 +3,6 @@
 # Cozystack end‑to‑end provisioning test (Bats)
 # -----------------------------------------------------------------------------
 
-export TALOSCONFIG=$PWD/talosconfig
-export KUBECONFIG=$PWD/kubeconfig
-
 @test "Environment variable COZYSTACK_INSTALLER_YAML is defined" {
   if [ -z "${COZYSTACK_INSTALLER_YAML:-}" ]; then
     echo 'COZYSTACK_INSTALLER_YAML environment variable is not set!' >&2
@@ -376,7 +373,10 @@ EOF
 
   # Verify Grafana via ingress
   ingress_ip=$(kubectl get svc root-ingress-controller -n tenant-root -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-  curl -sS -k "https://${ingress_ip}" -H 'Host: grafana.example.org' | grep -q Found
+  if ! curl -sS -k "https://${ingress_ip}" -H 'Host: grafana.example.org' --max-time 30 | grep -q Found; then
+    echo "Failed to access Grafana via ingress at ${ingress_ip}" >&2
+    exit 1
+  fi
 }
 
 @test "Keycloak OIDC stack is healthy" {
diff --git a/packages/core/testing/Makefile b/packages/core/testing/Makefile
index 2b67da45..f3a97eee 100755
--- a/packages/core/testing/Makefile
+++ b/packages/core/testing/Makefile
@@ -30,8 +30,13 @@ image-e2e-sandbox:
 		yq -i '.e2e.image = strenv(IMAGE)' values.yaml
 	rm -f images/e2e-sandbox.json
 
-test: ## Run the end-to-end tests in existing sandbox.
-	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && export COZYSTACK_INSTALLER_YAML=$$(helm template -n cozy-system installer ./packages/core/installer) && hack/cozytest.sh hack/e2e.bats'
+test: test-cluster test-apps ## Run the end-to-end tests in existing sandbox
+
+test-cluster: ## Run the end-to-end for creating a cluster
+	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && export COZYSTACK_INSTALLER_YAML=$$(helm template -n cozy-system installer ./packages/core/installer) && hack/cozytest.sh hack/e2e-cluster.bats'
+
+test-apps: ## Run the end-to-end tests for apps
+	docker exec "${SANDBOX_NAME}" sh -c 'cd /workspace && hack/cozytest.sh hack/e2e-apps.bats'
 
 delete: ## Remove sandbox from existing Kubernetes cluster.
 	docker rm -f "${SANDBOX_NAME}" || true
@@ -40,5 +45,10 @@ exec:  ## Opens an interactive shell in the sandbox container.
 	docker exec -ti "${SANDBOX_NAME}" bash
 
 apply: delete
-	docker run -d --rm --name "${SANDBOX_NAME}" --privileged "$$(yq .e2e.image values.yaml)" sleep infinity
+	docker run \
+		-d --rm --name "${SANDBOX_NAME}" --privileged \
+		-e TALOSCONFIG=/workspace/talosconfig \
+		-e KUBECONFIG=/workspace/kubeconfig \
+		"$$(yq .e2e.image values.yaml)" \
+		sleep infinity
 	docker cp "${ROOT_DIR}" "${SANDBOX_NAME}":/workspace