[QT-695] Add config_mode variant to some scenarios (#26380)

Add `config_mode` variant to some scenarios so we can dynamically change how we primarily configure the Vault cluster, either by a configuration file or with environment variables. As part of this change we also: * Start consuming the Enos terraform provider from public Terraform registry. * Remove the old `seal_ha_beta` variant as it is no longer required. * Add a module that performs a `vault operator step-down` so that we can force leader elections in scenarios. * Wire up an operator step-down into some scenarios to test both the old and new multiseal code paths during leader elections. Signed-off-by: Ryan Cragun <me@ryan.ec>
2025-11-02 03:27:54 +00:00 · 2024-04-22 12:34:47 -06:00
parent 20863afca5
commit 27ab988205
71 changed files with 329 additions and 97 deletions
--- a/enos/enos-globals.hcl
+++ b/enos/enos-globals.hcl
@@ -14,6 +14,7 @@ globals {
    "ent.hsm"          = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
    "ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
  }
+  config_modes    = ["env", "file"]
  consul_versions = ["1.14.11", "1.15.7", "1.16.3", "1.17.0"]
  distros         = ["ubuntu", "rhel"]
  distro_version = {
--- a/enos/enos-modules.hcl
+++ b/enos/enos-modules.hcl
@@ -183,6 +183,12 @@ module "vault_setup_perf_secondary" {
  vault_install_dir = var.vault_install_dir
 }

+module "vault_step_down" {
+  source = "./modules/vault_step_down"
+
+  vault_install_dir = var.vault_install_dir
+}
+
 module "vault_test_ui" {
  source = "./modules/vault_test_ui"

@@ -203,7 +209,6 @@ module "vault_upgrade" {
  vault_instance_count = var.vault_instance_count
 }

-
 module "vault_verify_autopilot" {
  source = "./modules/vault_verify_autopilot"

--- a/enos/enos-scenario-agent.hcl
+++ b/enos/enos-scenario-agent.hcl
@@ -7,11 +7,11 @@ scenario "agent" {
    artifact_source = global.artifact_sources
    artifact_type   = global.artifact_types
    backend         = global.backends
+    config_mode     = global.config_modes
    consul_version  = global.consul_versions
    distro          = global.distros
    edition         = global.editions
    seal            = global.seals
-    seal_ha_beta    = ["true", "false"]

    # Our local builder always creates bundles
    exclude {
@@ -195,6 +195,7 @@ scenario "agent" {
      backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      consul_release = matrix.backend == "consul" ? {
        edition = var.backend_edition
@@ -207,7 +208,6 @@ scenario "agent" {
      manage_service       = local.manage_service
      packages             = concat(global.packages, global.distro_packages[matrix.distro])
      seal_attributes      = step.create_seal_key.attributes
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_type            = matrix.seal
      storage_backend      = matrix.backend
      target_hosts         = step.create_vault_cluster_targets.hosts
--- a/enos/enos-scenario-autopilot.hcl
+++ b/enos/enos-scenario-autopilot.hcl
@@ -6,11 +6,11 @@ scenario "autopilot" {
    arch            = global.archs
    artifact_source = global.artifact_sources
    artifact_type   = global.artifact_types
+    config_mode     = global.config_modes
    distro          = global.distros
    edition         = global.editions
    initial_version = global.upgrade_initial_versions
    seal            = global.seals
-    seal_ha_beta    = ["true", "false"]

    # Autopilot wasn't available before 1.11.x
    exclude {
@@ -157,6 +157,7 @@ scenario "autopilot" {

    variables {
      cluster_name         = step.create_vault_cluster_targets.cluster_name
+      config_mode          = matrix.config_mode
      enable_audit_devices = var.vault_enable_audit_devices
      install_dir          = local.vault_install_dir
      license              = matrix.edition != "ce" ? step.read_license.license : null
@@ -166,7 +167,6 @@ scenario "autopilot" {
        version = matrix.initial_version
      }
      seal_attributes = step.create_seal_key.attributes
-      seal_ha_beta    = matrix.seal_ha_beta
      seal_type       = matrix.seal
      storage_backend = "raft"
      storage_backend_addl_config = {
@@ -241,6 +241,7 @@ scenario "autopilot" {
      artifactory_release         = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
      enable_audit_devices        = var.vault_enable_audit_devices
      cluster_name                = step.create_vault_cluster_targets.cluster_name
+      config_mode                 = matrix.config_mode
      log_level                   = var.vault_log_level
      force_unseal                = matrix.seal == "shamir"
      initialize_cluster          = false
@@ -250,7 +251,6 @@ scenario "autopilot" {
      manage_service              = local.manage_service
      packages                    = concat(global.packages, global.distro_packages[matrix.distro])
      root_token                  = step.create_vault_cluster.root_token
-      seal_ha_beta                = matrix.seal_ha_beta
      seal_attributes             = step.create_seal_key.attributes
      seal_type                   = matrix.seal
      shamir_unseal_keys          = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
--- a/enos/enos-scenario-proxy.hcl
+++ b/enos/enos-scenario-proxy.hcl
@@ -7,11 +7,11 @@ scenario "proxy" {
    artifact_source = global.artifact_sources
    artifact_type   = global.artifact_types
    backend         = global.backends
+    config_mode     = global.config_modes
    consul_version  = global.consul_versions
    distro          = global.distros
    edition         = global.editions
    seal            = global.seals
-    seal_ha_beta    = ["true", "false"]

    # Our local builder always creates bundles
    exclude {
@@ -195,6 +195,7 @@ scenario "proxy" {
      backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      consul_release = matrix.backend == "consul" ? {
        edition = var.backend_edition
@@ -206,7 +207,6 @@ scenario "proxy" {
      local_artifact_path  = local.artifact_path
      manage_service       = local.manage_service
      packages             = concat(global.packages, global.distro_packages[matrix.distro])
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_attributes      = step.create_seal_key.attributes
      seal_type            = matrix.seal
      storage_backend      = matrix.backend
--- a/enos/enos-scenario-replication.hcl
+++ b/enos/enos-scenario-replication.hcl
@@ -9,12 +9,12 @@ scenario "replication" {
    arch              = global.archs
    artifact_source   = global.artifact_sources
    artifact_type     = global.artifact_types
+    config_mode       = global.config_modes
    consul_version    = global.consul_versions
    distro            = global.distros
    edition           = global.editions
    primary_backend   = global.backends
    primary_seal      = global.seals
-    seal_ha_beta      = ["true", "false"]
    secondary_backend = global.backends
    secondary_seal    = global.seals

@@ -280,6 +280,7 @@ scenario "replication" {
      artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
      backend_cluster_name    = step.create_primary_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      cluster_name            = step.create_primary_cluster_targets.cluster_name
      consul_release = matrix.primary_backend == "consul" ? {
@@ -293,7 +294,6 @@ scenario "replication" {
      manage_service       = local.manage_service
      packages             = concat(global.packages, global.distro_packages[matrix.distro])
      seal_attributes      = step.create_primary_seal_key.attributes
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_type            = matrix.primary_seal
      storage_backend      = matrix.primary_backend
      target_hosts         = step.create_primary_cluster_targets.hosts
@@ -338,6 +338,7 @@ scenario "replication" {
      artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
      backend_cluster_name    = step.create_secondary_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.secondary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      cluster_name            = step.create_secondary_cluster_targets.cluster_name
      consul_release = matrix.secondary_backend == "consul" ? {
@@ -351,7 +352,6 @@ scenario "replication" {
      manage_service       = local.manage_service
      packages             = concat(global.packages, global.distro_packages[matrix.distro])
      seal_attributes      = step.create_secondary_seal_key.attributes
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_type            = matrix.secondary_seal
      storage_backend      = matrix.secondary_backend
      target_hosts         = step.create_secondary_cluster_targets.hosts
@@ -632,6 +632,7 @@ scenario "replication" {
      backend_cluster_name    = step.create_primary_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
      cluster_name            = step.create_primary_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      consul_release = matrix.primary_backend == "consul" ? {
        edition = var.backend_edition
@@ -646,7 +647,6 @@ scenario "replication" {
      manage_service       = local.manage_service
      packages             = concat(global.packages, global.distro_packages[matrix.distro])
      root_token           = step.create_primary_cluster.root_token
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_attributes      = step.create_primary_seal_key.attributes
      seal_type            = matrix.primary_seal
      shamir_unseal_keys   = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null
--- a/enos/enos-scenario-seal-ha.hcl
+++ b/enos/enos-scenario-seal-ha.hcl
@@ -7,6 +7,7 @@ scenario "seal_ha" {
    artifact_source = global.artifact_sources
    artifact_type   = global.artifact_types
    backend         = global.backends
+    config_mode     = global.config_modes
    consul_version  = global.consul_versions
    distro          = global.distros
    edition         = global.editions
@@ -218,6 +219,7 @@ scenario "seal_ha" {
      backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      consul_release = matrix.backend == "consul" ? {
        edition = var.backend_edition
@@ -365,7 +367,7 @@ scenario "seal_ha" {
  }

  // Wait for our cluster to elect a leader
-  step "wait_for_new_leader" {
+  step "wait_for_leader_election" {
    module     = module.vault_wait_for_leader
    depends_on = [step.add_ha_seal_to_cluster]

@@ -381,6 +383,54 @@ scenario "seal_ha" {
    }
  }

+  step "get_leader_ip_for_step_down" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_leader_election]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Force a step down to trigger a new leader election
+  step "vault_leader_step_down" {
+    module     = module.vault_step_down
+    depends_on = [step.get_leader_ip_for_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      leader_host       = step.get_leader_ip_for_step_down.leader_host
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_new_leader" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.vault_leader_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
  step "get_updated_cluster_ips" {
    module     = module.vault_get_cluster_ips
    depends_on = [step.wait_for_new_leader]
--- a/enos/enos-scenario-smoke.hcl
+++ b/enos/enos-scenario-smoke.hcl
@@ -7,11 +7,11 @@ scenario "smoke" {
    artifact_source = global.artifact_sources
    artifact_type   = global.artifact_types
    backend         = global.backends
+    config_mode     = global.config_modes
    consul_version  = global.consul_versions
    distro          = global.distros
    edition         = global.editions
    seal            = global.seals
-    seal_ha_beta    = ["true", "false"]

    # Our local builder always creates bundles
    exclude {
@@ -195,6 +195,7 @@ scenario "smoke" {
      backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      consul_release = matrix.backend == "consul" ? {
        edition = var.backend_edition
@@ -206,7 +207,6 @@ scenario "smoke" {
      local_artifact_path  = local.artifact_path
      manage_service       = local.manage_service
      packages             = concat(global.packages, global.distro_packages[matrix.distro])
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_attributes      = step.create_seal_key.attributes
      seal_type            = matrix.seal
      storage_backend      = matrix.backend
@@ -215,7 +215,7 @@ scenario "smoke" {
  }

  // Wait for our cluster to elect a leader
-  step "wait_for_leader" {
+  step "wait_for_new_leader" {
    module     = module.vault_wait_for_leader
    depends_on = [step.create_vault_cluster]

@@ -231,6 +231,54 @@ scenario "smoke" {
    }
  }

+  step "get_leader_ip_for_step_down" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_new_leader]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Force a step down to trigger a new leader election
+  step "vault_leader_step_down" {
+    module     = module.vault_step_down
+    depends_on = [step.get_leader_ip_for_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      leader_host       = step.get_leader_ip_for_step_down.leader_host
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_leader" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.vault_leader_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
  step "get_vault_cluster_ips" {
    module     = module.vault_get_cluster_ips
    depends_on = [step.wait_for_leader]
--- a/enos/enos-scenario-ui.hcl
+++ b/enos/enos-scenario-ui.hcl
@@ -3,9 +3,8 @@

 scenario "ui" {
  matrix {
-    backend      = global.backends
-    edition      = ["ce", "ent"]
-    seal_ha_beta = ["true", "false"]
+    backend = global.backends
+    edition = ["ce", "ent"]
  }

  terraform_cli = terraform_cli.default
@@ -180,7 +179,6 @@ scenario "ui" {
      license              = matrix.edition != "ce" ? step.read_vault_license.license : null
      local_artifact_path  = local.bundle_path
      packages             = global.distro_packages["ubuntu"]
-      seal_ha_beta         = matrix.seal_ha_beta
      seal_name            = step.create_seal_key.resource_name
      seal_type            = local.seal
      storage_backend      = matrix.backend
--- a/enos/enos-scenario-upgrade.hcl
+++ b/enos/enos-scenario-upgrade.hcl
@@ -7,6 +7,7 @@ scenario "upgrade" {
    artifact_source = global.artifact_sources
    artifact_type   = global.artifact_types
    backend         = global.backends
+    config_mode     = global.config_modes
    consul_version  = global.consul_versions
    distro          = global.distros
    edition         = global.editions
@@ -18,7 +19,6 @@ scenario "upgrade" {
    // no longer supporting setting the license via the license API.
    initial_version = global.upgrade_initial_versions
    seal            = global.seals
-    seal_ha_beta    = ["true", "false"]

    # Our local builder always creates bundles
    exclude {
@@ -207,8 +207,9 @@ scenario "upgrade" {
    variables {
      backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
      backend_cluster_tag_key = global.backend_tag_key
-      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      config_mode             = matrix.config_mode
+      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
      consul_release = matrix.backend == "consul" ? {
        edition = var.backend_edition
        version = matrix.consul_version
@@ -221,7 +222,6 @@ scenario "upgrade" {
        edition = matrix.edition
        version = matrix.initial_version
      }
-      seal_ha_beta    = matrix.seal_ha_beta
      seal_attributes = step.create_seal_key.attributes
      seal_type       = matrix.seal
      storage_backend = matrix.backend
@@ -308,12 +308,58 @@ scenario "upgrade" {
    }
  }

+  step "get_leader_ip_for_step_down" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_leader_after_upgrade]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Force a step down to trigger a new leader election
+  step "vault_leader_step_down" {
+    module     = module.vault_step_down
+    depends_on = [step.get_leader_ip_for_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      leader_host       = step.get_leader_ip_for_step_down.leader_host
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_leader_after_stepdown" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.vault_leader_step_down]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
  step "get_updated_vault_cluster_ips" {
    module = module.vault_get_cluster_ips
    depends_on = [
-      step.create_vault_cluster,
-      step.upgrade_vault,
-      step.wait_for_leader_after_upgrade,
+      step.wait_for_leader_after_stepdown,
    ]

    providers = {
--- a/enos/enos-terraform.hcl
+++ b/enos/enos-terraform.hcl
@@ -11,7 +11,7 @@ terraform_cli "default" {
  /*
  provider_installation {
    dev_overrides = {
-      "app.terraform.io/hashicorp-qti/enos" = abspath("../../enos-provider/dist")
+      "registry.terraform.io/hashicorp-forge/enos" = abspath("../../enos-provider/dist")
    }
    direct {}
  }
@@ -27,7 +27,7 @@ terraform "default" {
    }

    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.0"
    }
  }
--- a/enos/k8s/enos-terraform-k8s.hcl
+++ b/enos/k8s/enos-terraform-k8s.hcl
@@ -6,7 +6,7 @@ terraform "k8s" {

  required_providers {
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
    }

    helm = {
--- a/enos/modules/backend_consul/main.tf
+++ b/enos/modules/backend_consul/main.tf
@@ -6,7 +6,7 @@ terraform {

  required_providers {
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.4"
    }
  }
--- a/enos/modules/backend_raft/main.tf
+++ b/enos/modules/backend_raft/main.tf
@@ -7,7 +7,7 @@ terraform {

  required_providers {
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.0"
    }
  }
--- a/enos/modules/build_local/main.tf
+++ b/enos/modules/build_local/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/generate_secondary_token/main.tf
+++ b/enos/modules/generate_secondary_token/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
    random = {
      source  = "hashicorp/random"
--- a/enos/modules/get_local_metadata/main.tf
+++ b/enos/modules/get_local_metadata/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/install_packages/main.tf
+++ b/enos/modules/install_packages/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/k8s_deploy_vault/main.tf
+++ b/enos/modules/k8s_deploy_vault/main.tf
@@ -6,7 +6,7 @@ terraform {

  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }

    helm = {
--- a/enos/modules/k8s_vault_verify_build_date/main.tf
+++ b/enos/modules/k8s_vault_verify_build_date/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/k8s_vault_verify_replication/main.tf
+++ b/enos/modules/k8s_vault_verify_replication/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/k8s_vault_verify_ui/main.tf
+++ b/enos/modules/k8s_vault_verify_ui/main.tf
@@ -6,7 +6,7 @@ terraform {
  required_providers {
    enos = {
      version = "> 0.4.0"
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/k8s_vault_verify_version/main.tf
+++ b/enos/modules/k8s_vault_verify_version/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/k8s_vault_verify_write_data/main.tf
+++ b/enos/modules/k8s_vault_verify_write_data/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/load_docker_image/main.tf
+++ b/enos/modules/load_docker_image/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/local_kind_cluster/main.tf
+++ b/enos/modules/local_kind_cluster/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
    random = {
      source  = "hashicorp/random"
--- a/enos/modules/seal_awskms/main.tf
+++ b/enos/modules/seal_awskms/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/seal_pkcs11/main.tf
+++ b/enos/modules/seal_pkcs11/main.tf
@@ -44,7 +44,7 @@ Now you should be able to configure Vault Enterprise seal stanza.
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/seal_shamir/main.tf
+++ b/enos/modules/seal_shamir/main.tf
@@ -7,7 +7,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/shutdown_multiple_nodes/main.tf
+++ b/enos/modules/shutdown_multiple_nodes/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/shutdown_node/main.tf
+++ b/enos/modules/shutdown_node/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/softhsm_create_vault_keys/main.tf
+++ b/enos/modules/softhsm_create_vault_keys/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/softhsm_distribute_vault_keys/main.tf
+++ b/enos/modules/softhsm_distribute_vault_keys/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.9"
    }
  }
--- a/enos/modules/softhsm_init/main.tf
+++ b/enos/modules/softhsm_init/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.9"
    }
  }
--- a/enos/modules/softhsm_install/main.tf
+++ b/enos/modules/softhsm_install/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/start_vault/main.tf
+++ b/enos/modules/start_vault/main.tf
@@ -6,18 +6,14 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
-      version = ">= 0.4.8"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
+      version = ">= 0.4.10"
    }
  }
 }

 locals {
  bin_path = "${var.install_dir}/vault"
-  environment = local.seal_secondary == null ? var.environment : merge(
-    var.environment,
-    { VAULT_ENABLE_SEAL_HA_BETA : tobool(var.seal_ha_beta) },
-  )
  // In order to get Terraform to plan we have to use collections with keys that are known at plan
  // time. Here we're creating locals that keep track of index values that point to our target hosts.
  followers = toset(slice(local.instances, 1, length(local.instances)))
@@ -160,7 +156,8 @@ resource "enos_vault_start" "leader" {

  bin_path    = local.bin_path
  config_dir  = var.config_dir
-  environment = local.environment
+  config_mode = var.config_mode
+  environment = var.environment
  config = {
    api_addr     = "http://${var.target_hosts[each.value].private_ip}:8200"
    cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
@@ -200,7 +197,8 @@ resource "enos_vault_start" "followers" {

  bin_path    = local.bin_path
  config_dir  = var.config_dir
-  environment = local.environment
+  config_mode = var.config_mode
+  environment = var.environment
  config = {
    api_addr     = "http://${var.target_hosts[each.value].private_ip}:8200"
    cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
--- a/enos/modules/start_vault/variables.tf
+++ b/enos/modules/start_vault/variables.tf
@@ -12,6 +12,16 @@ variable "config_dir" {
  default     = "/etc/vault.d"
 }

+variable "config_mode" {
+  description = "The method to use when configuring Vault. When set to 'env' we will configure Vault using VAULT_ style environment variables if possible. When 'file' we'll use the HCL configuration file for all configuration options."
+  default     = "file"
+
+  validation {
+    condition     = contains(["env", "file"], var.config_mode)
+    error_message = "The config_mode must be either 'env' or 'file'. No other configuration modes are supported."
+  }
+}
+
 variable "environment" {
  description = "Optional Vault configuration environment variables to set starting Vault"
  type        = map(string)
@@ -48,11 +58,6 @@ variable "manage_service" {
  default     = true
 }

-variable "seal_ha_beta" {
-  description = "Enable using Seal HA on clusters that meet minimum version requirements and are enterprise editions"
-  default     = true
-}
-
 variable "seal_alias" {
  type        = string
  description = "The primary seal alias name"
--- a/enos/modules/stop_vault/main.tf
+++ b/enos/modules/stop_vault/main.tf
@@ -6,7 +6,7 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.0"
    }
  }
--- a/enos/modules/target_ec2_fleet/main.tf
+++ b/enos/modules/target_ec2_fleet/main.tf
@@ -6,7 +6,7 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.3.24"
    }
  }
--- a/enos/modules/target_ec2_instances/main.tf
+++ b/enos/modules/target_ec2_instances/main.tf
@@ -6,7 +6,7 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.3.24"
    }
  }
--- a/enos/modules/target_ec2_shim/main.tf
+++ b/enos/modules/target_ec2_shim/main.tf
@@ -6,7 +6,7 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.3.24"
    }
  }
--- a/enos/modules/target_ec2_spot_fleet/main.tf
+++ b/enos/modules/target_ec2_spot_fleet/main.tf
@@ -6,7 +6,7 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.3.24"
    }
  }
--- a/enos/modules/vault_agent/main.tf
+++ b/enos/modules/vault_agent/main.tf
@@ -7,7 +7,7 @@ terraform {
      source = "hashicorp/aws"
    }
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_artifactory_artifact/main.tf
+++ b/enos/modules/vault_artifactory_artifact/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.2.3"
    }
  }
--- a/enos/modules/vault_cluster/main.tf
+++ b/enos/modules/vault_cluster/main.tf
@@ -6,7 +6,7 @@ terraform {
    # We need to specify the provider source in each module until we publish it
    # to the public registry
    enos = {
-      source  = "app.terraform.io/hashicorp-qti/enos"
+      source  = "registry.terraform.io/hashicorp-forge/enos"
      version = ">= 0.4.0"
    }
  }
@@ -127,13 +127,13 @@ module "start_vault" {

  cluster_name              = var.cluster_name
  config_dir                = var.config_dir
+  config_mode               = var.config_mode
  install_dir               = var.install_dir
  license                   = var.license
  log_level                 = var.log_level
  manage_service            = var.manage_service
  seal_attributes           = var.seal_attributes
  seal_attributes_secondary = var.seal_attributes_secondary
-  seal_ha_beta              = var.seal_ha_beta
  seal_type                 = var.seal_type
  seal_type_secondary       = var.seal_type_secondary
  service_username          = local.vault_service_user
--- a/enos/modules/vault_cluster/variables.tf
+++ b/enos/modules/vault_cluster/variables.tf
@@ -36,6 +36,16 @@ variable "config_dir" {
  default     = "/etc/vault.d"
 }

+variable "config_mode" {
+  description = "The method to use when configuring Vault. When set to 'env' we will configure Vault using VAULT_ style environment variables if possible. When 'file' we'll use the HCL configuration file for all configuration options."
+  default     = "file"
+
+  validation {
+    condition     = contains(["env", "file"], var.config_mode)
+    error_message = "The config_mode must be either 'env' or 'file'. No other configuration modes are supported."
+  }
+}
+
 variable "config_env_vars" {
  description = "Optional Vault configuration environment variables to set starting Vault"
  type        = map(string)
--- a/enos/modules/vault_get_cluster_ips/main.tf
+++ b/enos/modules/vault_get_cluster_ips/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_proxy/main.tf
+++ b/enos/modules/vault_proxy/main.tf
@@ -7,7 +7,7 @@ terraform {
      source = "hashicorp/aws"
    }
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_raft_remove_peer/main.tf
+++ b/enos/modules/vault_raft_remove_peer/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_setup_perf_primary/main.tf
+++ b/enos/modules/vault_setup_perf_primary/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_setup_perf_secondary/main.tf
+++ b/enos/modules/vault_setup_perf_secondary/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_step_down/main.tf
+++ b/enos/modules/vault_step_down/main.tf
@@ -0,0 +1,51 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "registry.terraform.io/hashicorp-forge/enos"
+    }
+  }
+}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the Vault binary will be installed"
+}
+
+variable "vault_addr" {
+  type        = string
+  description = "The vault cluster listen address"
+  default     = "http://localhost:8200"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "leader_host" {
+  type = object({
+    private_ip = string
+    public_ip  = string
+  })
+
+  description = "The vault cluster host that can be expected as a leader"
+}
+
+resource "enos_remote_exec" "vault_operator_step_down" {
+  environment = {
+    VAULT_TOKEN       = var.vault_root_token
+    VAULT_ADDR        = var.vault_addr
+    VAULT_INSTALL_DIR = var.vault_install_dir
+  }
+
+  scripts = [abspath("${path.module}/scripts/operator-step-down.sh")]
+
+  transport = {
+    ssh = {
+      host = var.leader_host.public_ip
+    }
+  }
+}
--- a/enos/modules/vault_step_down/scripts/operator-step-down.sh
+++ b/enos/modules/vault_step_down/scripts/operator-step-down.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+
+set -eou pipefail
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+
+eval "$binpath" operator step-down
--- a/enos/modules/vault_test_ui/main.tf
+++ b/enos/modules/vault_test_ui/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_unseal_nodes/main.tf
+++ b/enos/modules/vault_unseal_nodes/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_upgrade/main.tf
+++ b/enos/modules/vault_upgrade/main.tf
@@ -7,7 +7,7 @@ terraform {
      source = "hashicorp/aws"
    }
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_agent_output/main.tf
+++ b/enos/modules/vault_verify_agent_output/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_autopilot/main.tf
+++ b/enos/modules/vault_verify_autopilot/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_default_lcq/main.tf
+++ b/enos/modules/vault_verify_default_lcq/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_performance_replication/main.tf
+++ b/enos/modules/vault_verify_performance_replication/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_raft_auto_join_voter/main.tf
+++ b/enos/modules/vault_verify_raft_auto_join_voter/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_read_data/main.tf
+++ b/enos/modules/vault_verify_read_data/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_replication/main.tf
+++ b/enos/modules/vault_verify_replication/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_ui/main.tf
+++ b/enos/modules/vault_verify_ui/main.tf
@@ -5,7 +5,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_undo_logs/main.tf
+++ b/enos/modules/vault_verify_undo_logs/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_unsealed/main.tf
+++ b/enos/modules/vault_verify_unsealed/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_version/main.tf
+++ b/enos/modules/vault_verify_version/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_verify_write_data/main.tf
+++ b/enos/modules/vault_verify_write_data/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_wait_for_leader/main.tf
+++ b/enos/modules/vault_wait_for_leader/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/vault_wait_for_seal_rewrap/main.tf
+++ b/enos/modules/vault_wait_for_seal_rewrap/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }
--- a/enos/modules/verify_seal_type/main.tf
+++ b/enos/modules/verify_seal_type/main.tf
@@ -4,7 +4,7 @@
 terraform {
  required_providers {
    enos = {
-      source = "app.terraform.io/hashicorp-qti/enos"
+      source = "registry.terraform.io/hashicorp-forge/enos"
    }
  }
 }