From d097433266733f6e5d8d60c1e0f2030bb853f072 Mon Sep 17 00:00:00 2001
From: Andrei Kvapil <kvapss@gmail.com>
Date: Thu, 26 Jun 2025 13:33:14 +0200
Subject: [PATCH] [e2e] Add retries

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
---
 .github/workflows/pull-requests.yaml | 33 +++++++++++++++++++++++++---
 hack/e2e-apps/clickhouse.bats        |  3 +--
 hack/e2e-apps/kafka.bats             |  2 +-
 hack/e2e-apps/kubernetes.bats        |  3 +--
 hack/e2e-apps/mysql.bats             |  3 +--
 hack/e2e-apps/postgres.bats          |  3 +--
 hack/e2e-apps/redis.bats             |  2 +-
 hack/e2e-apps/virtualmachine.bats    |  3 +--
 hack/e2e-apps/vminstance.bats        |  6 ++---
 9 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml
index 989e9f49..0ee7e671 100644
--- a/.github/workflows/pull-requests.yaml
+++ b/.github/workflows/pull-requests.yaml
@@ -165,7 +165,16 @@ jobs:
       - name: Prepare environment
         run: |
           cd /tmp/$SANDBOX_NAME
-          make SANDBOX_NAME=$SANDBOX_NAME prepare-env
+          attempt=0
+          until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
+            attempt=$((attempt + 1))
+            if [ $attempt -ge 3 ]; then
+              echo "❌ Attempt $attempt failed, exiting..."
+              exit 1
+            fi
+            echo "❌ Attempt $attempt failed, retrying..."
+          done
+          echo "✅ The task completed successfully after $attempt attempts"
 
   install_cozystack:
     name: "Install Cozystack"
@@ -205,7 +214,16 @@ jobs:
       - name: Install Cozystack into sandbox
         run: |
           cd /tmp/$SANDBOX_NAME
-          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack
+          attempt=0
+          until make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; do
+            attempt=$((attempt + 1))
+            if [ $attempt -ge 3 ]; then
+              echo "❌ Attempt $attempt failed, exiting..."
+              exit 1
+            fi
+            echo "❌ Attempt $attempt failed, retrying..."
+          done
+          echo "✅ The task completed successfully after $attempt attempts."
 
   detect_test_matrix:
     name: "Detect e2e test matrix"
@@ -236,7 +254,16 @@ jobs:
       - name: E2E Apps
         run: |
           cd /tmp/$SANDBOX_NAME
-          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-${{ matrix.app }}
+          attempt=0
+          until make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-${{ matrix.app }}; do
+            attempt=$((attempt + 1))
+            if [ $attempt -ge 3 ]; then
+              echo "❌ Attempt $attempt failed, exiting..."
+              exit 1
+            fi
+            echo "❌ Attempt $attempt failed, retrying..."
+          done
+          echo "✅ The task completed successfully after $attempt attempts"
 
   cleanup:
     name: Tear down environment
diff --git a/hack/e2e-apps/clickhouse.bats b/hack/e2e-apps/clickhouse.bats
index b1d1f29c..95d86d02 100644
--- a/hack/e2e-apps/clickhouse.bats
+++ b/hack/e2e-apps/clickhouse.bats
@@ -2,8 +2,7 @@
 
 @test "Create DB ClickHouse" {
   name='test'
-  kubectl -n tenant-test get clickhouses.apps.cozystack.io $name || 
-  kubectl create -f- <<EOF
+  kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: ClickHouse
 metadata:
diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats
index 284a4aa3..ceaf2cde 100644
--- a/hack/e2e-apps/kafka.bats
+++ b/hack/e2e-apps/kafka.bats
@@ -2,7 +2,7 @@
 
 @test "Create Kafka" {
   name='test'
-  kubectl create -f- <<EOF
+  kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Kafka
 metadata:
diff --git a/hack/e2e-apps/kubernetes.bats b/hack/e2e-apps/kubernetes.bats
index ac313a14..5f3ea777 100644
--- a/hack/e2e-apps/kubernetes.bats
+++ b/hack/e2e-apps/kubernetes.bats
@@ -1,8 +1,7 @@
 #!/usr/bin/env bats
 
 @test "Create a tenant Kubernetes control plane" {
-  kubectl -n tenant-test get kuberneteses.apps.cozystack.io test || 
-  kubectl create -f - <<EOF
+  kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Kubernetes
 metadata:
diff --git a/hack/e2e-apps/mysql.bats b/hack/e2e-apps/mysql.bats
index 6d9fad01..031ce832 100644
--- a/hack/e2e-apps/mysql.bats
+++ b/hack/e2e-apps/mysql.bats
@@ -2,8 +2,7 @@
 
 @test "Create DB MySQL" {
   name='test'
-  kubectl -n tenant-test get mysqls.apps.cozystack.io $name || 
-  kubectl create -f- <<EOF
+  kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: MySQL
 metadata:
diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats
index e1cc5dea..024ce086 100644
--- a/hack/e2e-apps/postgres.bats
+++ b/hack/e2e-apps/postgres.bats
@@ -2,8 +2,7 @@
 
 @test "Create DB PostgreSQL" {
   name='test'
-  kubectl -n tenant-test get postgreses.apps.cozystack.io $name || 
-  kubectl create -f - <<EOF
+  kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Postgres
 metadata:
diff --git a/hack/e2e-apps/redis.bats b/hack/e2e-apps/redis.bats
index 612c78d1..050f407b 100644
--- a/hack/e2e-apps/redis.bats
+++ b/hack/e2e-apps/redis.bats
@@ -2,7 +2,7 @@
 
 @test "Create Redis" {
   name='test'
-  kubectl create -f- <<EOF
+  kubectl apply -f- <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: Redis
 metadata:
diff --git a/hack/e2e-apps/virtualmachine.bats b/hack/e2e-apps/virtualmachine.bats
index 3a6bedf3..a0bbe8f4 100644
--- a/hack/e2e-apps/virtualmachine.bats
+++ b/hack/e2e-apps/virtualmachine.bats
@@ -2,8 +2,7 @@
 
 @test "Create a Virtual Machine" {
   name='test'
-  kubectl -n tenant-test get virtualmachines.apps.cozystack.io $name || 
-  kubectl create -f - <<EOF
+  kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VirtualMachine
 metadata:
diff --git a/hack/e2e-apps/vminstance.bats b/hack/e2e-apps/vminstance.bats
index 488dc909..59fe6c22 100644
--- a/hack/e2e-apps/vminstance.bats
+++ b/hack/e2e-apps/vminstance.bats
@@ -2,8 +2,7 @@
 
 @test "Create a VM Disk" {
   name='test'
-  kubectl -n tenant-test get vmdisks.apps.cozystack.io $name || 
-  kubectl create -f - <<EOF
+  kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMDisk
 metadata:
@@ -26,8 +25,7 @@ EOF
 @test "Create a VM Instance" {
   diskName='test'
   name='test'
-  kubectl -n tenant-test get vminstances.apps.cozystack.io $name || 
-  kubectl create -f - <<EOF
+  kubectl apply -f - <<EOF
 apiVersion: apps.cozystack.io/v1alpha1
 kind: VMInstance
 metadata: