From 8c2548f8bebc8d8346a0826587fe651a4d19e84f Mon Sep 17 00:00:00 2001
From: Ryan Cragun <me@ryan.ec>
Date: Fri, 9 Aug 2024 13:43:11 -0600
Subject: [PATCH] VAULT-29739: Wait for cluster unseal before checking version
 history (#28022)

Sometimes the replication scenario will race with other steps and
attempt to check the `v1/sys/version-history` API before the cluster is
ready. Eventually when it gets retried some of the original nodes are
down so it will fail. This makes the verification happen later, only
after we've ensured the cluster is unsealed and have gotten leader and
cluster IP addresses. We also make dependent steps require the version
verification so that if it does fail for some reason it will retry
before doing the rest of the scenario.

Signed-off-by: Ryan Cragun <me@ryan.ec>
---
 enos/enos-scenario-replication.hcl | 106 ++++++++++++++---------------
 1 file changed, 50 insertions(+), 56 deletions(-)

diff --git a/enos/enos-scenario-replication.hcl b/enos/enos-scenario-replication.hcl
index d828f9ab9c..8311a86247 100644
--- a/enos/enos-scenario-replication.hcl
+++ b/enos/enos-scenario-replication.hcl
@@ -590,58 +590,6 @@ scenario "replication" {
     }
   }
 
-  step "verify_vault_version" {
-    description = global.description.verify_vault_version
-    module      = module.vault_verify_version
-    depends_on = [
-      step.create_primary_cluster,
-      step.wait_for_primary_cluster_leader,
-    ]
-
-    providers = {
-      enos = local.enos_provider[matrix.distro]
-    }
-
-    verifies = [
-      quality.vault_api_sys_version_history_keys,
-      quality.vault_api_sys_version_history_key_info,
-      quality.vault_version_build_date,
-      quality.vault_version_edition,
-      quality.vault_version_release,
-    ]
-
-    variables {
-      hosts                 = step.create_primary_cluster_targets.hosts
-      vault_addr            = step.create_primary_cluster.api_addr_localhost
-      vault_edition         = matrix.edition
-      vault_install_dir     = global.vault_install_dir[matrix.artifact_type]
-      vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
-      vault_revision        = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
-      vault_build_date      = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
-      vault_root_token      = step.create_primary_cluster.root_token
-    }
-  }
-
-  step "verify_ui" {
-    description = global.description.verify_ui
-    module      = module.vault_verify_ui
-    depends_on = [
-      step.create_primary_cluster,
-      step.wait_for_primary_cluster_leader,
-    ]
-
-    providers = {
-      enos = local.enos_provider[matrix.distro]
-    }
-
-    verifies = quality.vault_ui_assets
-
-    variables {
-      vault_addr = step.create_primary_cluster.api_addr_localhost
-      hosts      = step.create_primary_cluster_targets.hosts
-    }
-  }
-
   step "get_primary_cluster_ips" {
     description = global.description.get_vault_cluster_ip_addresses
     module      = module.vault_get_cluster_ips
@@ -690,12 +638,57 @@ scenario "replication" {
     }
   }
 
+  step "verify_vault_version" {
+    description = global.description.verify_vault_version
+    module      = module.vault_verify_version
+    depends_on  = [step.get_primary_cluster_ips]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = [
+      quality.vault_api_sys_version_history_keys,
+      quality.vault_api_sys_version_history_key_info,
+      quality.vault_version_build_date,
+      quality.vault_version_edition,
+      quality.vault_version_release,
+    ]
+
+    variables {
+      hosts                 = step.create_primary_cluster_targets.hosts
+      vault_addr            = step.create_primary_cluster.api_addr_localhost
+      vault_edition         = matrix.edition
+      vault_install_dir     = global.vault_install_dir[matrix.artifact_type]
+      vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
+      vault_revision        = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
+      vault_build_date      = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
+      vault_root_token      = step.create_primary_cluster.root_token
+    }
+  }
+
+  step "verify_ui" {
+    description = global.description.verify_ui
+    module      = module.vault_verify_ui
+    depends_on  = [step.get_primary_cluster_ips]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    verifies = quality.vault_ui_assets
+
+    variables {
+      vault_addr = step.create_primary_cluster.api_addr_localhost
+      hosts      = step.create_primary_cluster_targets.hosts
+    }
+  }
+
   step "write_test_data_on_primary" {
     description = global.description.verify_write_test_data
     module      = module.vault_verify_write_data
     depends_on  = [step.get_primary_cluster_ips]
 
-
     providers = {
       enos = local.enos_provider[matrix.distro]
     }
@@ -724,9 +717,10 @@ scenario "replication" {
     EOF
     module      = module.vault_setup_perf_primary
     depends_on = [
-      step.get_primary_cluster_ips,
-      step.get_secondary_cluster_ips,
-      step.write_test_data_on_primary
+      step.write_test_data_on_primary,
+      // Do base verification before continuing on to our performance replication verification.
+      step.verify_vault_version,
+      step.verify_ui,
     ]
 
     providers = {