[tests] Run tests w/ requests and limits

Signed-off-by: Ahmad Murzahmatov <gwynbleidd2106@yandex.com>

hack/e2e-apps/clickhouse.bats

@@ -2,6 +2,18 @@
 
 @test "Create DB ClickHouse" {
   name='test'
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    resources=$(cat <<EOF
+  resources:
+    resources:
+      cpu: 500m
+      memory: 768Mi
+EOF
+  )
+  else
+    resources='  resources: {}'
+  fi
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: ClickHouse
@@ -27,15 +39,13 @@ spec:
     s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
     s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
     resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
-  resources: {}
+  $resources
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr clickhouse-$name --timeout=20s --for=condition=ready
-  timeout 180 sh -ec "until kubectl -n tenant-test get svc chendpoint-clickhouse-$name -o jsonpath='{.spec.ports[*].port}' | grep -q '8123 9000'; do sleep 10; done"
-  kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-0 --timeout=120s --for=jsonpath='{.status.replicas}'=1
-  timeout 80 sh -ec "until kubectl -n tenant-test get endpoints chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
-  timeout 100 sh -ec "until kubectl -n tenant-test get svc chi-clickhouse-$name-clickhouse-0-0 -o jsonpath='{.spec.ports[*].port}' | grep -q '9000 8123 9009'; do sleep 10; done"
-  timeout 80 sh -ec "until kubectl -n tenant-test get sts chi-clickhouse-$name-clickhouse-0-1 ; do sleep 10; done"
-  kubectl -n tenant-test wait statefulset.apps/chi-clickhouse-$name-clickhouse-0-1 --timeout=140s --for=jsonpath='{.status.replicas}'=1
+  kubectl -n tenant-test wait --timeout=40s hr clickhouse-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s clickhouses $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=120s sts chi-clickhouse-$name-clickhouse-0-0 --for=jsonpath='{.status.replicas}'=1
+  timeout 210 sh -ec "until kubectl -n tenant-test wait svc chendpoint-clickhouse-$name --for=jsonpath='{.spec.ports[0].port}'=8123; do sleep 10; done"
+  kubectl -n tenant-test delete clickhouse.apps.cozystack.io $name
 }

hack/e2e-apps/kafka.bats

@@ -2,6 +2,18 @@
 
 @test "Create Kafka" {
   name='test'
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    resources=$(cat <<EOF
+resources:
+  resources:
+    cpu: 500m
+    memory: 768Mi
+EOF
+  )
+  else
+    resources='resources: {}'
+  fi
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Kafka
@@ -14,13 +26,13 @@ spec:
     size: 10Gi
     replicas: 2
     storageClass: ""
-    resources: {}
+    $resources
     resourcesPreset: "nano"
   zookeeper:
     size: 5Gi
     replicas: 2
     storageClass: ""
-    resources:
+    $resources
     resourcesPreset: "nano"
   topics:
     - name: testResults
@@ -38,14 +50,9 @@ spec:
       replicas: 2
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr kafka-$name --timeout=30s --for=condition=ready
-  kubectl wait kafkas -n tenant-test test --timeout=60s --for=condition=ready
-  timeout 60 sh -ec "until kubectl -n tenant-test get pvc data-kafka-$name-zookeeper-0; do sleep 10; done"
-  kubectl -n tenant-test wait pvc data-kafka-$name-zookeeper-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
-  timeout 40 sh -ec "until kubectl -n tenant-test get svc kafka-$name-zookeeper-client -o jsonpath='{.spec.ports[0].port}' | grep -q '2181'; do sleep 10; done"
-  timeout 40 sh -ec "until kubectl -n tenant-test get svc kafka-$name-zookeeper-nodes -o jsonpath='{.spec.ports[*].port}' | grep -q '2181 2888 3888'; do sleep 10; done"
-  timeout 80 sh -ec "until kubectl -n tenant-test get endpoints kafka-$name-zookeeper-nodes -o jsonpath='{.subsets[*].addresses[0].ip}' | grep -q '[0-9]'; do sleep 10; done"
+  kubectl -n tenant-test wait --timeout=30s hr kafka-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=1m kafkas $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=50s pvc data-kafka-$name-zookeeper-0 --for=jsonpath='{.status.phase}'=Bound
+  kubectl -n tenant-test wait --timeout=40s svc kafka-$name-zookeeper-client --for=jsonpath='{.spec.ports[0].port}'=2181
   kubectl -n tenant-test delete kafka.apps.cozystack.io $name
-  kubectl -n tenant-test delete pvc data-kafka-$name-zookeeper-0
-  kubectl -n tenant-test delete pvc data-kafka-$name-zookeeper-1
 }

hack/e2e-apps/kubernetes.bats

@@ -1,16 +1,17 @@
 #!/usr/bin/env bats
 
 @test "Create a tenant Kubernetes control plane" {
+  name='test'
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Kubernetes
 metadata:
-  name: test
+  name: $name
   namespace: tenant-test
 spec:
   addons:
     certManager:
-      enabled: false
+      enabled: true
       valuesOverride: {}
     cilium:
       valuesOverride: {}
@@ -24,10 +25,12 @@ spec:
       valuesOverride: {}
     ingressNginx:
       enabled: true
-      hosts: []
+      hosts:
+        - example.org
+      exposeMethod: Proxied
       valuesOverride: {}
     monitoringAgents:
-      enabled: false
+      enabled: true
       valuesOverride: {}
     verticalPodAutoscaler:
       valuesOverride: {}
@@ -61,12 +64,39 @@ spec:
       - ingress-nginx
   storageClass: replicated
 EOF
-  kubectl wait namespace tenant-test --timeout=20s --for=jsonpath='{.status.phase}'=Active
-  timeout 10 sh -ec 'until kubectl get kamajicontrolplane -n tenant-test kubernetes-test; do sleep 1; done'
-  kubectl wait --for=condition=TenantControlPlaneCreated kamajicontrolplane -n tenant-test kubernetes-test --timeout=4m
-  kubectl wait tcp -n tenant-test kubernetes-test --timeout=2m --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
-  kubectl wait deploy --timeout=4m --for=condition=available -n tenant-test kubernetes-test kubernetes-test-cluster-autoscaler kubernetes-test-kccm kubernetes-test-kcsi-controller
-  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=1m --for=jsonpath='{.status.replicas}'=2
-  kubectl wait machinedeployment kubernetes-test-md0 -n tenant-test --timeout=10m --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
-  kubectl -n tenant-test delete kuberneteses.apps.cozystack.io test
+  sleep 10
+  kubectl wait --timeout=20s namespace tenant-test --for=jsonpath='{.status.phase}'=Active
+  kubectl -n tenant-test wait --timeout=10s kamajicontrolplane kubernetes-$name --for=jsonpath='{.status.conditions[0].status}'=True
+  kubectl -n tenant-test wait --timeout=4m kamajicontrolplane kubernetes-$name --for=condition=TenantControlPlaneCreated
+  kubectl -n tenant-test wait --timeout=210s tcp kubernetes-$name --for=jsonpath='{.status.kubernetesResources.version.status}'=Ready
+  kubectl -n tenant-test wait --timeout=4m deploy kubernetes-$name kubernetes-$name-cluster-autoscaler kubernetes-$name-kccm kubernetes-$name-kcsi-controller --for=condition=available
+  kubectl -n tenant-test wait --timeout=1m machinedeployment kubernetes-$name-md0 --for=jsonpath='{.status.replicas}'=2
+  kubectl -n tenant-test wait --timeout=10m machinedeployment kubernetes-$name-md0 --for=jsonpath='{.status.v1beta2.readyReplicas}'=2
+  # ingress / load balancer
+  kubectl -n tenant-test wait --timeout=5m hr kubernetes-$name-monitoring-agents --for=condition=ready
+  kubectl -n tenant-test wait --timeout=5m hr kubernetes-$name-ingress-nginx --for=condition=ready
+  kubectl -n tenant-test get secret kubernetes-$name-admin-kubeconfig -o go-template='{{ printf "%s\n" (index .data "admin.conf" | base64decode) }}' > admin.conf
+  KUBECONFIG=admin.conf kubectl -n cozy-ingress-nginx wait --timeout=3m deploy ingress-nginx-defaultbackend --for=jsonpath='{.status.conditions[0].status}'=True
+  KUBECONFIG=admin.conf kubectl -n cozy-monitoring wait --timeout=3m deploy cozy-monitoring-agents-metrics-server --for=jsonpath='{.status.conditions[0].status}'=True
 }
+
+@test "Create a PVC in tenant Kubernetes" {
+  name='test'
+  KUBECONFIG=admin.conf kubectl apply -f - <<EOF
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-$name
+  namespace: cozy-monitoring
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+EOF
+  sleep 10
+  KUBECONFIG=admin.conf kubectl -n cozy-monitoring wait --timeout=20s pvc pvc-$name --for=jsonpath='{.status.phase}'=Bound
+  KUBECONFIG=admin.conf kubectl -n cozy-monitoring delete pvc pvc-$name
+  kubectl -n tenant-test delete kuberneteses.apps.cozystack.io $name
+}

hack/e2e-apps/mysql.bats

@@ -2,6 +2,18 @@
 
 @test "Create DB MySQL" {
   name='test'
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    resources=$(cat <<EOF
+  resources:
+    resources:
+      cpu: 3000m
+      memory: 3Gi
+EOF
+  )
+  else
+    resources='  resources: {}'
+  fi
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: MySQL
@@ -31,16 +43,15 @@ spec:
     s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
     s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
     resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
-  resources: {}
+  $resources
   resourcesPreset: "nano"
 EOF
-  sleep 5
-  kubectl -n tenant-test wait hr mysql-$name --timeout=30s --for=condition=ready
-  timeout 80 sh -ec "until kubectl -n tenant-test get svc mysql-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done"
-  timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mysql-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
-  kubectl -n tenant-test wait statefulset.apps/mysql-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2
-  timeout 80 sh -ec "until kubectl -n tenant-test get svc mysql-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done"
-  timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mysql-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
-  kubectl -n tenant-test wait deployment.apps/mysql-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1
+  sleep 10
+  kubectl -n tenant-test wait --timeout=30s hr mysql-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s mysqls $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=110s sts mysql-$name --for=jsonpath='{.status.replicas}'=2
+  sleep 60
+  kubectl -n tenant-test wait --timeout=60s deploy mysql-$name-metrics --for=jsonpath='{.status.replicas}'=1
+  kubectl -n tenant-test wait --timeout=100s svc mysql-$name --for=jsonpath='{.spec.ports[0].port}'=3306
   kubectl -n tenant-test delete mysqls.apps.cozystack.io $name
 }

hack/e2e-apps/postgres.bats

@@ -2,6 +2,18 @@
 
 @test "Create DB PostgreSQL" {
   name='test'
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    resources=$(cat <<EOF
+  resources:
+    resources:
+      cpu: 500m
+      memory: 768Mi
+EOF
+  )
+  else
+    resources='  resources: {}'
+  fi
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Postgres
@@ -36,19 +48,14 @@ spec:
     s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
     s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
     resticPassword: ChaXoveekoh6eigh4siesheeda2quai0
-  resources: {}
+  $resources
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr postgres-$name --timeout=100s --for=condition=ready
-  kubectl -n tenant-test wait job.batch postgres-$name-init-job --timeout=50s --for=condition=Complete
-  timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-r -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"
-  timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-ro -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"
-  timeout 40 sh -ec "until kubectl -n tenant-test get svc postgres-$name-rw -o jsonpath='{.spec.ports[0].port}' | grep -q '5432'; do sleep 10; done"
-  timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-r -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
-  # for some reason it takes longer for the read-only endpoint to be ready
-  #timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-ro -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
-  timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-rw -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done"
+  kubectl -n tenant-test wait --timeout=200s hr postgres-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s postgreses $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=50s job.batch postgres-$name-init-job --for=condition=Complete
+  kubectl -n tenant-test wait --timeout=40s svc postgres-$name-r --for=jsonpath='{.spec.ports[0].port}'=5432
   kubectl -n tenant-test delete postgreses.apps.cozystack.io $name
   kubectl -n tenant-test delete job.batch/postgres-$name-init-job
 }

hack/e2e-apps/redis.bats

@@ -2,6 +2,18 @@
 
 @test "Create Redis" {
   name='test'
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    resources=$(cat <<EOF
+resources:
+  resources:
+    cpu: 500m
+    memory: 768Mi
+EOF
+  )
+  else
+    resources='resources: {}'
+  fi
   kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Redis
@@ -14,13 +26,15 @@ spec:
   replicas: 2
   storageClass: ""
   authEnabled: true
-  resources: {}
+  $resources
   resourcesPreset: "nano"
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr redis-$name --timeout=20s --for=condition=ready
-  kubectl -n tenant-test wait pvc redisfailover-persistent-data-rfr-redis-$name-0 --timeout=50s --for=jsonpath='{.status.phase}'=Bound
-  kubectl -n tenant-test wait deploy rfs-redis-$name --timeout=90s --for=condition=available
-  kubectl -n tenant-test wait sts rfr-redis-$name --timeout=90s --for=jsonpath='{.status.replicas}'=2
+  kubectl -n tenant-test wait --timeout=20s hr redis-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s redis.apps.cozystack.io $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=50s pvc redisfailover-persistent-data-rfr-redis-$name-0 --for=jsonpath='{.status.phase}'=Bound
+  kubectl -n tenant-test wait --timeout=90s sts rfr-redis-$name --for=jsonpath='{.status.replicas}'=2
+  sleep 45
+  kubectl -n tenant-test wait --timeout=45s deploy rfs-redis-$name --for=condition=available
   kubectl -n tenant-test delete redis.apps.cozystack.io $name
 }

hack/e2e-apps/virtualmachine.bats

@@ -2,6 +2,14 @@
 
 @test "Create a Virtual Machine" {
   name='test'
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    cores="1000m"
+    memory="1Gi
+  else
+    cores="2000m"
+    memory="2Gi
+  fi
   kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VirtualMachine
@@ -9,6 +17,12 @@ metadata:
   name: $name
   namespace: tenant-test
 spec:
+  domain:
+    cpu:
+      cores: "$cores"
+  resources:
+    requests:
+      memory: "$memory"
   external: false
   externalMethod: PortList
   externalPorts:
@@ -20,9 +34,6 @@ spec:
     storage: 5Gi
     storageClass: replicated
   gpus: []
-  resources:
-    cpu: ""
-    memory: ""
   sshKeys:
   - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF
     test@test
@@ -37,11 +48,12 @@ spec:
           - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF test@test
   cloudInitSeed: ""
 EOF
-  sleep 5
-  kubectl -n tenant-test wait hr virtual-machine-$name --timeout=10s --for=condition=ready
-  kubectl -n tenant-test wait dv virtual-machine-$name --timeout=150s --for=condition=ready
-  kubectl -n tenant-test wait pvc virtual-machine-$name --timeout=100s --for=jsonpath='{.status.phase}'=Bound
-  kubectl -n tenant-test wait vm virtual-machine-$name --timeout=100s --for=condition=ready
-  timeout 120 sh -ec "until kubectl -n tenant-test get vmi virtual-machine-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 10; done"
+  sleep 10
+  kubectl -n tenant-test wait --timeout=10s hr virtual-machine-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s virtualmachines $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s pvc virtual-machine-$name --for=jsonpath='{.status.phase}'=Bound
+  kubectl -n tenant-test wait --timeout=150s dv virtual-machine-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=100s vm virtual-machine-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=150s vmi virtual-machine-$name --for=jsonpath='{status.phase}'=Running
   kubectl -n tenant-test delete virtualmachines.apps.cozystack.io $name 
 }

hack/e2e-apps/vminstance.bats

@@ -17,21 +17,37 @@ spec:
   storageClass: replicated
 EOF
   sleep 5
-  kubectl -n tenant-test wait hr vm-disk-$name --timeout=5s --for=condition=ready
-  kubectl -n tenant-test wait dv vm-disk-$name --timeout=150s --for=condition=ready
-  kubectl -n tenant-test wait pvc vm-disk-$name --timeout=100s --for=jsonpath='{.status.phase}'=Bound
+  kubectl -n tenant-test wait --timeout=5s hr vm-disk-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s vmdisks $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s pvc vm-disk-$name --for=jsonpath='{.status.phase}'=Bound
+  kubectl -n tenant-test wait --timeout=150s dv vm-disk-$name --for=condition=ready
 }
 
 @test "Create a VM Instance" {
   diskName='test'
   name='test'
-  kubectl apply -f - <<EOF
+  withResources='true'
+  if [ "$withResources" == 'true' ]; then
+    cores="1000m"
+    memory="1Gi
+  else
+    cores="2000m"
+    memory="2Gi
+  fi
+  kubectl -n tenant-test get vminstances.apps.cozystack.io $name || 
+  kubectl create -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMInstance
 metadata:
   name: $name
   namespace: tenant-test
 spec:
+  domain:
+    cpu:
+      cores: "$cores"
+  resources:
+    requests:
+      memory: "$memory"
   external: false
   externalMethod: PortList
   externalPorts:
@@ -42,9 +58,6 @@ spec:
   disks:
     - name: $diskName
   gpus: []
-  resources:
-    cpu: ""
-    memory: ""
   sshKeys:
   - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPht0dPk5qQ+54g1hSX7A6AUxXJW5T6n/3d7Ga2F8gTF
     test@test
@@ -60,9 +73,10 @@ spec:
   cloudInitSeed: ""
 EOF
   sleep 5
-  timeout 20 sh -ec "until kubectl -n tenant-test get vmi vm-instance-$name -o jsonpath='{.status.interfaces[0].ipAddress}' | grep -q '[0-9]'; do sleep 5; done"
-  kubectl -n tenant-test wait hr vm-instance-$name --timeout=5s --for=condition=ready
-  kubectl -n tenant-test wait vm vm-instance-$name --timeout=20s --for=condition=ready
+  kubectl -n tenant-test wait --timeout=5s hr vm-instance-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=130s vminstances $name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=20s vm vm-instance-$name --for=condition=ready
+  kubectl -n tenant-test wait --timeout=40s vmi vm-instance-$name --for=jsonpath='{status.phase}'=Running
   kubectl -n tenant-test delete vminstances.apps.cozystack.io $name 
   kubectl -n tenant-test delete vmdisks.apps.cozystack.io $diskName 
 }