backport of commit a46def288f (#23869)

Co-authored-by: Ryan Cragun <me@ryan.ec>

.github/workflows/enos-lint.yml

@@ -41,7 +41,6 @@ jobs:
       - uses: hashicorp/setup-terraform@v2
         with:
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - uses: hashicorp/action-setup-enos@v1
         with:
           github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}

.github/workflows/enos-run-k8s.yml

@@ -38,7 +38,6 @@ jobs:
           # the Terraform wrapper will break Terraform execution in Enos because
           # it changes the output to text when we expect it to be JSON.
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - name: Set up Enos
         uses: hashicorp/action-setup-enos@v1
         with:

.github/workflows/test-enos-scenario-ui.yml

@@ -85,7 +85,6 @@ jobs:
         with:
           cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - name: Prepare scenario dependencies
         run: |
           mkdir -p ./enos/support/terraform-plugin-cache

.github/workflows/test-run-enos-scenario-matrix.yml

@@ -102,7 +102,6 @@ jobs:
           # the Terraform wrapper will break Terraform execution in Enos because
           # it changes the output to text when we expect it to be JSON.
           terraform_wrapper: false
-          terraform_version: 1.5.7 # QT-623: pin to terraform 1.5.x until a tfjson bug is resolved
       - uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }}

enos/ci/service-user-iam/main.tf

@@ -216,6 +216,7 @@ data "aws_iam_policy_document" "enos_scenario" {
       "kms:ListKeys",
       "kms:ListResourceTags",
       "kms:ScheduleKeyDeletion",
+      "kms:TagResource",
       "servicequotas:ListServiceQuotas"
     ]
 

enos/enos-modules.hcl

@@ -57,6 +57,18 @@ module "replication_data" {
   source = "./modules/replication_data"
 }
 
+module "seal_key_awskms" {
+  source = "./modules/seal_key_awskms"
+
+  common_tags = var.tags
+}
+
+module "seal_key_shamir" {
+  source = "./modules/seal_key_shamir"
+
+  common_tags = var.tags
+}
+
 module "shutdown_node" {
   source = "./modules/shutdown_node"
 }
@@ -65,6 +77,17 @@ module "shutdown_multiple_nodes" {
   source = "./modules/shutdown_multiple_nodes"
 }
 
+module "start_vault" {
+  source = "./modules/start_vault"
+
+  install_dir = var.vault_install_dir
+  log_level   = var.vault_log_level
+}
+
+module "stop_vault" {
+  source = "./modules/stop_vault"
+}
+
 # create target instances using ec2:CreateFleet
 module "target_ec2_fleet" {
   source = "./modules/target_ec2_fleet"
@@ -249,3 +272,10 @@ module "vault_wait_for_leader" {
   vault_install_dir    = var.vault_install_dir
   vault_instance_count = var.vault_instance_count
 }
+
+module "vault_wait_for_seal_rewrap" {
+  source = "./modules/vault_wait_for_seal_rewrap"
+
+  vault_install_dir    = var.vault_install_dir
+  vault_instance_count = var.vault_instance_count
+}

enos/enos-scenario-agent.hcl

@@ -11,6 +11,7 @@ scenario "agent" {
     distro          = ["ubuntu", "rhel"]
     edition         = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -81,6 +82,15 @@ scenario "agent" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -111,10 +121,10 @@ scenario "agent" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -128,10 +138,10 @@ scenario "agent" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.backend_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -171,7 +181,6 @@ scenario "agent" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
@@ -186,9 +195,11 @@ scenario "agent" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_seal_key.resource_name
+      seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
-      unseal_method        = matrix.seal
     }
   }
 
@@ -389,11 +400,6 @@ scenario "agent" {
     value       = step.create_vault_cluster.audit_device_file_path
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
-  }
-
   output "cluster_name" {
     description = "The Vault cluster name"
     value       = step.create_vault_cluster.cluster_name
@@ -434,6 +440,11 @@ scenario "agent" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The name of the cluster seal key"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64

enos/enos-scenario-autopilot.hcl

@@ -12,6 +12,7 @@ scenario "autopilot" {
     // release branch's version.
     initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -77,6 +78,15 @@ scenario "autopilot" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   step "read_license" {
     module = module.read_license
 
@@ -95,10 +105,10 @@ scenario "autopilot" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -114,8 +124,8 @@ scenario "autopilot" {
     }
 
     variables {
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_name         = step.create_vault_cluster_targets.cluster_name
+      enable_audit_devices = var.vault_enable_audit_devices
       install_dir          = local.vault_install_dir
       license              = matrix.edition != "ce" ? step.read_license.license : null
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
@@ -123,13 +133,14 @@ scenario "autopilot" {
         edition = matrix.edition
         version = matrix.initial_version
       }
+      seal_ha_beta    = matrix.seal_ha_beta
+      seal_key_name   = step.create_seal_key.resource_name
+      seal_type       = matrix.seal
       storage_backend = "raft"
       storage_backend_addl_config = {
         autopilot_upgrade_version = matrix.initial_version
       }
       target_hosts = step.create_vault_cluster_targets.hosts
-      unseal_method        = matrix.seal
-      enable_audit_devices = var.vault_enable_audit_devices
     }
   }
 
@@ -191,10 +202,10 @@ scenario "autopilot" {
 
     variables {
       ami_id         = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       common_tags    = global.tags
       cluster_name   = step.create_vault_cluster_targets.cluster_name
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names = step.create_seal_key.resource_names
+      vpc_id         = step.create_vpc.id
     }
   }
 
@@ -213,7 +224,7 @@ scenario "autopilot" {
 
     variables {
       artifactory_release         = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn       = step.create_vpc.kms_key_arn
+      enable_audit_devices        = var.vault_enable_audit_devices
       cluster_name                = step.create_vault_cluster_targets.cluster_name
       log_level                   = var.vault_log_level
       force_unseal                = matrix.seal == "shamir"
@@ -224,13 +235,14 @@ scenario "autopilot" {
       manage_service              = local.manage_service
       packages                    = concat(global.packages, global.distro_packages[matrix.distro])
       root_token                  = step.create_vault_cluster.root_token
+      seal_ha_beta                = matrix.seal_ha_beta
+      seal_key_name               = step.create_seal_key.resource_name
+      seal_type                   = matrix.seal
       shamir_unseal_keys          = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
       storage_backend             = "raft"
       storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config
       storage_node_prefix         = "upgrade_node"
       target_hosts                = step.create_vault_cluster_upgrade_targets.hosts
-      unseal_method               = matrix.seal
-      enable_audit_devices        = var.vault_enable_audit_devices
     }
   }
 
@@ -498,9 +510,9 @@ scenario "autopilot" {
     }
   }
 
-  output "awskms_unseal_key_arn" {
+  output "audit_device_file_path" {
-    description = "The Vault cluster KMS key arn"
+    description = "The file path for the file audit device, if enabled"
-    value       = step.create_vpc.kms_key_arn
+    value       = step.create_vault_cluster.audit_device_file_path
   }
 
   output "cluster_name" {
@@ -543,6 +555,11 @@ scenario "autopilot" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The Vault cluster seal key name"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64
@@ -567,9 +584,4 @@ scenario "autopilot" {
     description = "The Vault cluster public IPs"
     value       = step.upgrade_vault_cluster_with_autopilot.public_ips
   }
-
-  output "vault_audit_device_file_path" {
-    description = "The file path for the file audit device, if enabled"
-    value       = step.create_vault_cluster.audit_device_file_path
-  }
 }

enos/enos-scenario-proxy.hcl

@@ -11,6 +11,7 @@ scenario "proxy" {
     distro          = ["ubuntu", "rhel"]
     edition         = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -81,6 +82,15 @@ scenario "proxy" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -111,10 +121,10 @@ scenario "proxy" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -128,10 +138,10 @@ scenario "proxy" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.backend_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -171,7 +181,6 @@ scenario "proxy" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
@@ -186,9 +195,11 @@ scenario "proxy" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_seal_key.resource_name
+      seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
-      unseal_method        = matrix.seal
     }
   }
 
@@ -361,11 +372,6 @@ scenario "proxy" {
     value       = step.create_vault_cluster.audit_device_file_path
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
-  }
-
   output "cluster_name" {
     description = "The Vault cluster name"
     value       = step.create_vault_cluster.cluster_name
@@ -406,6 +412,11 @@ scenario "proxy" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The Vault cluster seal key name"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64

enos/enos-scenario-replication.hcl

@@ -14,6 +14,7 @@ scenario "replication" {
     edition           = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     primary_backend   = ["raft", "consul"]
     primary_seal      = ["awskms", "shamir"]
+    seal_ha_beta      = ["true", "false"]
     secondary_backend = ["raft", "consul"]
     secondary_seal    = ["awskms", "shamir"]
 
@@ -86,6 +87,26 @@ scenario "replication" {
     }
   }
 
+  step "create_primary_seal_key" {
+    module = "seal_key_${matrix.primary_seal}"
+
+    variables {
+      cluster_id   = step.create_vpc.cluster_id
+      cluster_meta = "primary"
+      common_tags  = global.tags
+    }
+  }
+
+  step "create_secondary_seal_key" {
+    module = "seal_key_${matrix.secondary_seal}"
+
+    variables {
+      cluster_id   = step.create_vpc.cluster_id
+      cluster_meta = "secondary"
+      common_tags  = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -118,10 +139,10 @@ scenario "replication" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_primary_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -137,10 +158,10 @@ scenario "replication" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.backend_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_primary_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -157,11 +178,11 @@ scenario "replication" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_name    = step.create_primary_cluster_targets.cluster_name
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_primary_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -175,10 +196,10 @@ scenario "replication" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_secondary_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -192,10 +213,10 @@ scenario "replication" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.backend_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_secondary_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -235,7 +256,6 @@ scenario "replication" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_primary_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       consul_license          = (matrix.primary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
@@ -250,9 +270,11 @@ scenario "replication" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_primary_seal_key.resource_name
+      seal_type            = matrix.primary_seal
       storage_backend      = matrix.primary_backend
       target_hosts         = step.create_primary_cluster_targets.hosts
-      unseal_method        = matrix.primary_seal
     }
   }
 
@@ -292,7 +314,6 @@ scenario "replication" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_secondary_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       consul_license          = (matrix.secondary_backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
@@ -307,9 +328,11 @@ scenario "replication" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_secondary_seal_key.resource_name
+      seal_type            = matrix.secondary_seal
       storage_backend      = matrix.secondary_backend
       target_hosts         = step.create_secondary_cluster_targets.hosts
-      unseal_method        = matrix.secondary_seal
     }
   }
 
@@ -584,7 +607,6 @@ scenario "replication" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_primary_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_primary_cluster_targets.cluster_name
@@ -602,11 +624,13 @@ scenario "replication" {
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
       root_token           = step.create_primary_cluster.root_token
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_primary_seal_key.resource_name
+      seal_type            = matrix.primary_seal
       shamir_unseal_keys   = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null
       storage_backend      = matrix.primary_backend
       storage_node_prefix  = "newprimary_node"
       target_hosts         = step.create_primary_cluster_additional_targets.hosts
-      unseal_method        = matrix.primary_seal
     }
   }
 

enos/enos-scenario-seal-ha.hcl

@@ -0,0 +1,551 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+scenario "seal_ha" {
+  matrix {
+    arch            = ["amd64", "arm64"]
+    artifact_source = ["local", "crt", "artifactory"]
+    artifact_type   = ["bundle", "package"]
+    backend         = ["consul", "raft"]
+    consul_version  = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"]
+    distro          = ["ubuntu", "rhel"]
+    edition         = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
+    primary_seal    = ["awskms"]
+    secondary_seal  = ["awskms"]
+
+    # Our local builder always creates bundles
+    exclude {
+      artifact_source = ["local"]
+      artifact_type   = ["package"]
+    }
+
+    # HSM and FIPS 140-2 are only supported on amd64
+    exclude {
+      arch    = ["arm64"]
+      edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
+    }
+  }
+
+  terraform_cli = terraform_cli.default
+  terraform     = terraform.default
+  providers = [
+    provider.aws.default,
+    provider.enos.ubuntu,
+    provider.enos.rhel
+  ]
+
+  locals {
+    artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
+    enos_provider = {
+      rhel   = provider.enos.rhel
+      ubuntu = provider.enos.ubuntu
+    }
+    manage_service    = matrix.artifact_type == "bundle"
+    vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro]
+  }
+
+  step "get_local_metadata" {
+    skip_step = matrix.artifact_source != "local"
+    module    = module.get_local_metadata
+  }
+
+  step "build_vault" {
+    module = "build_${matrix.artifact_source}"
+
+    variables {
+      build_tags           = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition]
+      artifact_path        = local.artifact_path
+      goarch               = matrix.arch
+      goos                 = "linux"
+      artifactory_host     = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
+      artifactory_repo     = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
+      artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
+      artifactory_token    = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
+      arch                 = matrix.artifact_source == "artifactory" ? matrix.arch : null
+      product_version      = var.vault_product_version
+      artifact_type        = matrix.artifact_type
+      distro               = matrix.artifact_source == "artifactory" ? matrix.distro : null
+      edition              = matrix.artifact_source == "artifactory" ? matrix.edition : null
+      revision             = var.vault_revision
+    }
+  }
+
+  step "ec2_info" {
+    module = module.ec2_info
+  }
+
+  step "create_vpc" {
+    module = module.create_vpc
+
+    variables {
+      common_tags = global.tags
+    }
+  }
+
+  step "create_primary_seal_key" {
+    module = "seal_key_${matrix.primary_seal}"
+
+    variables {
+      cluster_id   = step.create_vpc.cluster_id
+      cluster_meta = "primary"
+      common_tags  = global.tags
+    }
+  }
+
+  step "create_secondary_seal_key" {
+    module = "seal_key_${matrix.secondary_seal}"
+
+    variables {
+      cluster_id      = step.create_vpc.cluster_id
+      cluster_meta    = "secondary"
+      common_tags     = global.tags
+      other_resources = step.create_primary_seal_key.resource_names
+    }
+  }
+
+  // This step reads the contents of the backend license if we're using a Consul backend and
+  // the edition is "ent".
+  step "read_backend_license" {
+    skip_step = matrix.backend == "raft" || var.backend_edition == "ce"
+    module    = module.read_license
+
+    variables {
+      file_name = global.backend_license_path
+    }
+  }
+
+  step "read_vault_license" {
+    skip_step = matrix.edition == "ce"
+    module    = module.read_license
+
+    variables {
+      file_name = global.vault_license_path
+    }
+  }
+
+  step "create_vault_cluster_targets" {
+    module     = module.target_ec2_instances
+    depends_on = [step.create_vpc]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
+      cluster_tag_key = global.vault_tag_key
+      common_tags     = global.tags
+      seal_key_names  = step.create_secondary_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
+    }
+  }
+
+  step "create_vault_cluster_backend_targets" {
+    module     = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim
+    depends_on = [step.create_vpc]
+
+    providers = {
+      enos = provider.enos.ubuntu
+    }
+
+    variables {
+      ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
+      seal_key_names  = step.create_secondary_seal_key.resource_names
+      cluster_tag_key = global.backend_tag_key
+      common_tags     = global.tags
+      vpc_id          = step.create_vpc.id
+    }
+  }
+
+  step "create_backend_cluster" {
+    module = "backend_${matrix.backend}"
+    depends_on = [
+      step.create_vault_cluster_backend_targets
+    ]
+
+    providers = {
+      enos = provider.enos.ubuntu
+    }
+
+    variables {
+      cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
+      cluster_tag_key = global.backend_tag_key
+      license         = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
+      release = {
+        edition = var.backend_edition
+        version = matrix.consul_version
+      }
+      target_hosts = step.create_vault_cluster_backend_targets.hosts
+    }
+  }
+
+  step "create_vault_cluster" {
+    module = module.vault_cluster
+    depends_on = [
+      step.create_backend_cluster,
+      step.build_vault,
+      step.create_vault_cluster_targets
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
+      backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
+      backend_cluster_tag_key = global.backend_tag_key
+      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
+      consul_release = matrix.backend == "consul" ? {
+        edition = var.backend_edition
+        version = matrix.consul_version
+      } : null
+      enable_audit_devices = var.vault_enable_audit_devices
+      install_dir          = local.vault_install_dir
+      license              = matrix.edition != "ce" ? step.read_vault_license.license : null
+      local_artifact_path  = local.artifact_path
+      manage_service       = local.manage_service
+      packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      // Only configure our primary seal during our initial cluster setup
+      seal_type       = matrix.primary_seal
+      seal_key_name   = step.create_primary_seal_key.resource_name
+      storage_backend = matrix.backend
+      target_hosts    = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_leader" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.create_vault_cluster]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "get_vault_cluster_ips" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_leader]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "verify_vault_unsealed" {
+    module     = module.vault_verify_unsealed
+    depends_on = [step.wait_for_leader]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      vault_instances   = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Write some test data before we create the new seal
+  step "verify_write_test_data" {
+    module = module.vault_verify_write_data
+    depends_on = [
+      step.create_vault_cluster,
+      step.get_vault_cluster_ips,
+      step.verify_vault_unsealed,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      leader_public_ip  = step.get_vault_cluster_ips.leader_public_ip
+      leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
+      vault_instances   = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Wait for the initial seal rewrap to complete before we add our HA seal.
+  step "wait_for_initial_seal_rewrap" {
+    module = module.vault_wait_for_seal_rewrap
+    depends_on = [
+      step.verify_write_test_data,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  // Stop the vault service on all nodes before we restart with new seal config
+  step "stop_vault" {
+    module = module.stop_vault
+    depends_on = [
+      step.create_vault_cluster,
+      step.verify_write_test_data,
+      step.wait_for_initial_seal_rewrap,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      target_hosts = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Add the secondary seal to the cluster
+  step "add_ha_seal_to_cluster" {
+    module     = module.start_vault
+    depends_on = [step.stop_vault]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      cluster_name            = step.create_vault_cluster_targets.cluster_name
+      install_dir             = local.vault_install_dir
+      license                 = matrix.edition != "ce" ? step.read_vault_license.license : null
+      manage_service          = local.manage_service
+      seal_type               = matrix.primary_seal
+      seal_key_name           = step.create_primary_seal_key.resource_name
+      seal_type_secondary     = matrix.secondary_seal
+      seal_key_name_secondary = step.create_secondary_seal_key.resource_name
+      storage_backend         = matrix.backend
+      target_hosts            = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Wait for our cluster to elect a leader
+  step "wait_for_new_leader" {
+    module     = module.vault_wait_for_leader
+    depends_on = [step.add_ha_seal_to_cluster]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      timeout           = 120 # seconds
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "get_updated_cluster_ips" {
+    module     = module.vault_get_cluster_ips
+    depends_on = [step.wait_for_new_leader]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "verify_vault_unsealed_with_new_seal" {
+    module     = module.vault_verify_unsealed
+    depends_on = [step.wait_for_new_leader]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      vault_instances   = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  // Wait for the seal rewrap to complete and verify that no entries failed
+  step "wait_for_seal_rewrap" {
+    module = module.vault_wait_for_seal_rewrap
+    depends_on = [
+      step.add_ha_seal_to_cluster,
+      step.verify_vault_unsealed_with_new_seal,
+    ]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_hosts       = step.create_vault_cluster_targets.hosts
+      vault_install_dir = local.vault_install_dir
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "verify_vault_version" {
+    module     = module.vault_verify_version
+    depends_on = [step.wait_for_seal_rewrap]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_instances       = step.create_vault_cluster_targets.hosts
+      vault_edition         = matrix.edition
+      vault_install_dir     = local.vault_install_dir
+      vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
+      vault_revision        = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
+      vault_build_date      = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
+      vault_root_token      = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "verify_raft_auto_join_voter" {
+    skip_step  = matrix.backend != "raft"
+    module     = module.vault_verify_raft_auto_join_voter
+    depends_on = [step.wait_for_seal_rewrap]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_install_dir = local.vault_install_dir
+      vault_instances   = step.create_vault_cluster_targets.hosts
+      vault_root_token  = step.create_vault_cluster.root_token
+    }
+  }
+
+  step "verify_replication" {
+    module     = module.vault_verify_replication
+    depends_on = [step.wait_for_seal_rewrap]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_edition     = matrix.edition
+      vault_install_dir = local.vault_install_dir
+      vault_instances   = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  step "verify_read_test_data" {
+    module     = module.vault_verify_read_data
+    depends_on = [step.wait_for_seal_rewrap]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      node_public_ips   = step.get_updated_cluster_ips.follower_public_ips
+      vault_install_dir = local.vault_install_dir
+    }
+  }
+
+  step "verify_ui" {
+    module     = module.vault_verify_ui
+    depends_on = [step.wait_for_seal_rewrap]
+
+    providers = {
+      enos = local.enos_provider[matrix.distro]
+    }
+
+    variables {
+      vault_instances = step.create_vault_cluster_targets.hosts
+    }
+  }
+
+  output "audit_device_file_path" {
+    description = "The file path for the file audit device, if enabled"
+    value       = step.create_vault_cluster.audit_device_file_path
+  }
+
+  output "cluster_name" {
+    description = "The Vault cluster name"
+    value       = step.create_vault_cluster.cluster_name
+  }
+
+  output "hosts" {
+    description = "The Vault cluster target hosts"
+    value       = step.create_vault_cluster.target_hosts
+  }
+
+  output "primary_seal_key_name" {
+    description = "The Vault cluster primary seal key name"
+    value       = step.create_primary_seal_key.resource_name
+  }
+
+  output "private_ips" {
+    description = "The Vault cluster private IPs"
+    value       = step.create_vault_cluster.private_ips
+  }
+
+  output "public_ips" {
+    description = "The Vault cluster public IPs"
+    value       = step.create_vault_cluster.public_ips
+  }
+
+  output "root_token" {
+    description = "The Vault cluster root token"
+    value       = step.create_vault_cluster.root_token
+  }
+
+  output "recovery_key_shares" {
+    description = "The Vault cluster recovery key shares"
+    value       = step.create_vault_cluster.recovery_key_shares
+  }
+
+  output "recovery_keys_b64" {
+    description = "The Vault cluster recovery keys b64"
+    value       = step.create_vault_cluster.recovery_keys_b64
+  }
+
+  output "recovery_keys_hex" {
+    description = "The Vault cluster recovery keys hex"
+    value       = step.create_vault_cluster.recovery_keys_hex
+  }
+
+  output "secondary_seal_key_name" {
+    description = "The Vault cluster secondary seal key name"
+    value       = step.create_secondary_seal_key.resource_name
+  }
+
+  output "unseal_keys_b64" {
+    description = "The Vault cluster unseal keys"
+    value       = step.create_vault_cluster.unseal_keys_b64
+  }
+
+  output "unseal_keys_hex" {
+    description = "The Vault cluster unseal keys hex"
+    value       = step.create_vault_cluster.unseal_keys_hex
+  }
+}

enos/enos-scenario-smoke.hcl

@@ -11,6 +11,7 @@ scenario "smoke" {
     distro          = ["ubuntu", "rhel"]
     edition         = ["ce", "ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -81,6 +82,15 @@ scenario "smoke" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -111,10 +121,10 @@ scenario "smoke" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -128,10 +138,10 @@ scenario "smoke" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.backend_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -171,7 +181,6 @@ scenario "smoke" {
 
     variables {
       artifactory_release     = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
@@ -186,9 +195,11 @@ scenario "smoke" {
       local_artifact_path  = local.artifact_path
       manage_service       = local.manage_service
       packages             = concat(global.packages, global.distro_packages[matrix.distro])
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_seal_key.resource_name
+      seal_type            = matrix.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
-      unseal_method        = matrix.seal
     }
   }
 
@@ -352,11 +363,6 @@ scenario "smoke" {
     value       = step.create_vault_cluster.audit_device_file_path
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
-  }
-
   output "cluster_name" {
     description = "The Vault cluster name"
     value       = step.create_vault_cluster.cluster_name
@@ -397,6 +403,11 @@ scenario "smoke" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The Vault cluster seal key name"
+    value       = step.create_seal_key.name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64

enos/enos-scenario-ui.hcl

@@ -5,6 +5,7 @@ scenario "ui" {
   matrix {
     edition      = ["ce", "ent"]
     backend      = ["consul", "raft"]
+    seal_ha_beta = ["true", "false"]
   }
 
   terraform_cli = terraform_cli.default
@@ -68,6 +69,15 @@ scenario "ui" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${local.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -98,10 +108,10 @@ scenario "ui" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[local.arch][local.distro][var.ubuntu_distro_version]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = local.vault_tag_key
       common_tags     = local.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -115,10 +125,10 @@ scenario "ui" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = local.backend_tag_key
       common_tags     = local.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -157,7 +167,6 @@ scenario "ui" {
     }
 
     variables {
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = local.backend_tag_key
       cluster_name            = step.create_vault_cluster_targets.cluster_name
@@ -171,9 +180,11 @@ scenario "ui" {
       license              = matrix.edition != "ce" ? step.read_vault_license.license : null
       local_artifact_path  = local.bundle_path
       packages             = global.distro_packages["ubuntu"]
+      seal_ha_beta         = matrix.seal_ha_beta
+      seal_key_name        = step.create_seal_key.resource_name
+      seal_type            = local.seal
       storage_backend      = matrix.backend
       target_hosts         = step.create_vault_cluster_targets.hosts
-      unseal_method        = local.seal
     }
   }
 
@@ -212,11 +223,6 @@ scenario "ui" {
     value       = step.create_vault_cluster.audit_device_file_path
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
-  }
-
   output "cluster_name" {
     description = "The Vault cluster name"
     value       = step.create_vault_cluster.cluster_name
@@ -257,6 +263,11 @@ scenario "ui" {
     value       = step.create_vault_cluster.root_token
   }
 
+  output "seal_key_name" {
+    description = "The Vault cluster seal key name"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "ui_test_environment" {
     value       = step.test_ui.ui_test_environment
     description = "The environment variables that are required in order to run the test:enos yarn target"

enos/enos-scenario-upgrade.hcl

@@ -16,6 +16,7 @@ scenario "upgrade" {
     // those earlier versions.
     initial_version = ["1.11.12", "1.12.11", "1.13.6", "1.14.2"]
     seal            = ["awskms", "shamir"]
+    seal_ha_beta    = ["true", "false"]
 
     # Our local builder always creates bundles
     exclude {
@@ -93,6 +94,15 @@ scenario "upgrade" {
     }
   }
 
+  step "create_seal_key" {
+    module = "seal_key_${matrix.seal}"
+
+    variables {
+      cluster_id  = step.create_vpc.cluster_id
+      common_tags = global.tags
+    }
+  }
+
   // This step reads the contents of the backend license if we're using a Consul backend and
   // the edition is "ent".
   step "read_backend_license" {
@@ -123,10 +133,10 @@ scenario "upgrade" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.vault_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -140,10 +150,10 @@ scenario "upgrade" {
 
     variables {
       ami_id          = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
-      awskms_unseal_key_arn = step.create_vpc.kms_key_arn
       cluster_tag_key = global.backend_tag_key
       common_tags     = global.tags
-      vpc_id                = step.create_vpc.vpc_id
+      seal_key_names  = step.create_seal_key.resource_names
+      vpc_id          = step.create_vpc.id
     }
   }
 
@@ -182,7 +192,6 @@ scenario "upgrade" {
     }
 
     variables {
-      awskms_unseal_key_arn   = step.create_vpc.kms_key_arn
       backend_cluster_name    = step.create_vault_cluster_backend_targets.cluster_name
       backend_cluster_tag_key = global.backend_tag_key
       consul_license          = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
@@ -199,9 +208,11 @@ scenario "upgrade" {
         edition = matrix.edition
         version = matrix.initial_version
       }
+      seal_ha_beta    = matrix.seal_ha_beta
+      seal_key_name   = step.create_seal_key.resource_name
+      seal_type       = matrix.seal
       storage_backend = matrix.backend
       target_hosts    = step.create_vault_cluster_targets.hosts
-      unseal_method   = matrix.seal
     }
   }
 
@@ -413,11 +424,6 @@ scenario "upgrade" {
     value       = step.create_vault_cluster.audit_device_file_path
   }
 
-  output "awskms_unseal_key_arn" {
-    description = "The Vault cluster KMS key arn"
-    value       = step.create_vpc.kms_key_arn
-  }
-
   output "cluster_name" {
     description = "The Vault cluster name"
     value       = step.create_vault_cluster.cluster_name
@@ -458,6 +464,11 @@ scenario "upgrade" {
     value       = step.create_vault_cluster.recovery_keys_hex
   }
 
+  output "seal_key_name" {
+    description = "The Vault cluster seal key name"
+    value       = step.create_seal_key.resource_name
+  }
+
   output "unseal_keys_b64" {
     description = "The Vault cluster unseal keys"
     value       = step.create_vault_cluster.unseal_keys_b64

enos/modules/create_vpc/main.tf

@@ -18,18 +18,6 @@ resource "random_string" "cluster_id" {
   special = false
 }
 
-resource "aws_kms_key" "key" {
-  count                   = var.create_kms_key ? 1 : 0
-  description             = "vault-ci-kms-key"
-  deletion_window_in_days = 7 // 7 is the shortest allowed window
-}
-
-resource "aws_kms_alias" "alias" {
-  count         = var.create_kms_key ? 1 : 0
-  name          = "alias/enos_key-${random_string.cluster_id.result}"
-  target_key_id = aws_kms_key.key[0].key_id
-}
-
 resource "aws_vpc" "vpc" {
   cidr_block           = var.cidr
   enable_dns_hostnames = true

enos/modules/create_vpc/outputs.tf

@@ -1,22 +1,17 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
-output "vpc_id" {
+output "id" {
   description = "Created VPC ID"
   value       = aws_vpc.vpc.id
 }
 
-output "vpc_cidr" {
+output "cidr" {
   description = "CIDR for whole VPC"
   value       = var.cidr
 }
 
-output "kms_key_arn" {
+output "cluster_id" {
-  description = "ARN of the generated KMS key"
+  description = "A unique string associated with the VPC"
-  value       = try(aws_kms_key.key[0].arn, null)
+  value       = random_string.cluster_id.result
-}
-
-output "kms_key_alias" {
-  description = "Alias of the generated KMS key"
-  value       = try(aws_kms_alias.alias[0].name, null)
 }

enos/modules/create_vpc/variables.tf

@@ -24,9 +24,3 @@ variable "common_tags" {
   type        = map(string)
   default     = { "Project" : "vault-ci" }
 }
-
-variable "create_kms_key" {
-  description = "Whether or not to create an key management service key"
-  type        = bool
-  default     = true
-}

enos/modules/seal_key_awskms/main.tf

@@ -0,0 +1,56 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "cluster_id" {
+  type = string
+}
+
+variable "cluster_meta" {
+  type    = string
+  default = null
+}
+
+variable "common_tags" {
+  type    = map(string)
+  default = null
+}
+
+variable "other_resources" {
+  type    = list(string)
+  default = []
+}
+
+locals {
+  cluster_name = var.cluster_meta == null ? var.cluster_id : "${var.cluster_id}-${var.cluster_meta}"
+}
+
+resource "aws_kms_key" "key" {
+  description             = "auto-unseal-key-${local.cluster_name}"
+  deletion_window_in_days = 7 // 7 is the shortest allowed window
+  tags                    = var.common_tags
+}
+
+resource "aws_kms_alias" "alias" {
+  name          = "alias/auto-unseal-key-${local.cluster_name}"
+  target_key_id = aws_kms_key.key.key_id
+}
+
+output "alias" {
+  description = "The key alias name"
+  value       = aws_kms_alias.alias.name
+}
+
+output "id" {
+  description = "The key ID"
+  value       = aws_kms_key.key.key_id
+}
+
+output "resource_name" {
+  description = "The ARN"
+  value       = aws_kms_key.key.arn
+}
+
+output "resource_names" {
+  description = "The list of names"
+  value       = compact(concat([aws_kms_key.key.arn], var.other_resources))
+}

enos/modules/seal_key_shamir/main.tf

@@ -0,0 +1,17 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# A shim unseal key module for shamir seal types
+
+variable "cluster_id" { default = null }
+variable "cluster_meta" { default = null }
+variable "common_tags" { default = null }
+variable "names" {
+  type    = list(string)
+  default = []
+}
+
+output "alias" { value = null }
+output "id" { value = null }
+output "resource_name" { value = null }
+output "resource_names" { value = var.names }

enos/modules/start_vault/main.tf

@@ -0,0 +1,167 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    # We need to specify the provider source in each module until we publish it
+    # to the public registry
+    enos = {
+      source  = "app.terraform.io/hashicorp-qti/enos"
+      version = ">= 0.4.7"
+    }
+  }
+}
+
+data "enos_environment" "localhost" {}
+
+locals {
+  bin_path = "${var.install_dir}/vault"
+  environment = local.seal_secondary == null ? var.environment : merge(
+    var.environment,
+    { VAULT_ENABLE_SEAL_HA_BETA : tobool(var.seal_ha_beta) },
+  )
+  // In order to get Terraform to plan we have to use collections with keys
+  // that are known at plan time. In order for our module to work our var.target_hosts
+  // must be a map with known keys at plan time. Here we're creating locals
+  // that keep track of index values that point to our target hosts.
+  followers = toset(slice(local.instances, 1, length(local.instances)))
+  instances = [for idx in range(length(var.target_hosts)) : tostring(idx)]
+  key_shares = {
+    "awskms" = null
+    "shamir" = 5
+  }
+  key_threshold = {
+    "awskms" = null
+    "shamir" = 3
+  }
+  leader = toset(slice(local.instances, 0, 1))
+  recovery_shares = {
+    "awskms" = 5
+    "shamir" = null
+  }
+  recovery_threshold = {
+    "awskms" = 3
+    "shamir" = null
+  }
+  seals = local.seal_secondary.type == "none" ? { primary = local.seal_primary } : {
+    primary   = local.seal_primary
+    secondary = local.seal_secondary
+  }
+  seals_primary = {
+    "awskms" = {
+      type = "awskms"
+      attributes = {
+        name       = "primary"
+        kms_key_id = var.seal_key_name
+      }
+    }
+    "shamir" = {
+      type       = "shamir"
+      attributes = null
+    }
+  }
+  seal_primary = local.seals_primary[var.seal_type]
+  seals_secondary = {
+    "awskms" = {
+      type = "awskms"
+      attributes = {
+        name       = "secondary"
+        kms_key_id = var.seal_key_name_secondary
+      }
+    }
+    "none" = {
+      type       = "none"
+      attributes = null
+    }
+  }
+  seal_secondary = local.seals_secondary[var.seal_type_secondary]
+  storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ?
+    merge(
+      {
+        node_id = "${var.storage_node_prefix}_${idx}"
+      },
+      var.storage_backend_attrs
+    ) :
+    {
+      address = "127.0.0.1:8500"
+      path    = "vault"
+    })
+  ]
+}
+
+resource "enos_vault_start" "leader" {
+  for_each = local.leader
+
+  bin_path    = local.bin_path
+  config_dir  = var.config_dir
+  environment = local.environment
+  config = {
+    api_addr     = "http://${var.target_hosts[each.value].private_ip}:8200"
+    cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
+    cluster_name = var.cluster_name
+    listener = {
+      type = "tcp"
+      attributes = {
+        address     = "0.0.0.0:8200"
+        tls_disable = "true"
+      }
+    }
+    log_level = var.log_level
+    storage = {
+      type       = var.storage_backend
+      attributes = ({ for key, value in local.storage_config[each.key] : key => value })
+    }
+    seals = local.seals
+    ui    = true
+  }
+  license        = var.license
+  manage_service = var.manage_service
+  username       = var.service_username
+  unit_name      = "vault"
+
+  transport = {
+    ssh = {
+      host = var.target_hosts[each.value].public_ip
+    }
+  }
+}
+
+resource "enos_vault_start" "followers" {
+  depends_on = [
+    enos_vault_start.leader,
+  ]
+  for_each = local.followers
+
+  bin_path    = local.bin_path
+  config_dir  = var.config_dir
+  environment = local.environment
+  config = {
+    api_addr     = "http://${var.target_hosts[each.value].private_ip}:8200"
+    cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
+    cluster_name = var.cluster_name
+    listener = {
+      type = "tcp"
+      attributes = {
+        address     = "0.0.0.0:8200"
+        tls_disable = "true"
+      }
+    }
+    log_level = var.log_level
+    storage = {
+      type       = var.storage_backend
+      attributes = { for key, value in local.storage_config[each.key] : key => value }
+    }
+    seals = local.seals
+    ui    = true
+  }
+  license        = var.license
+  manage_service = var.manage_service
+  username       = var.service_username
+  unit_name      = "vault"
+
+  transport = {
+    ssh = {
+      host = var.target_hosts[each.value].public_ip
+    }
+  }
+}

enos/modules/start_vault/outputs.tf

@@ -0,0 +1,33 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+output "cluster_name" {
+  description = "The Vault cluster name"
+  value       = var.cluster_name
+}
+
+output "followers" {
+  description = "The follower enos_vault_start resources"
+  value       = enos_vault_start.followers
+}
+
+output "leader" {
+  description = "The leader enos_vault_start resource"
+  value       = enos_vault_start.leader
+}
+
+output "private_ips" {
+  description = "Vault cluster target host private_ips"
+  value       = [for host in var.target_hosts : host.private_ip]
+}
+
+output "public_ips" {
+  description = "Vault cluster target host public_ips"
+  value       = [for host in var.target_hosts : host.public_ip]
+}
+
+output "target_hosts" {
+  description = "The vault cluster instances that were created"
+
+  value = var.target_hosts
+}

enos/modules/start_vault/variables.tf

@@ -0,0 +1,125 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "cluster_name" {
+  type        = string
+  description = "The Vault cluster name"
+}
+
+variable "config_dir" {
+  type        = string
+  description = "The directory to use for Vault configuration"
+  default     = "/etc/vault.d"
+}
+
+variable "environment" {
+  description = "Optional Vault configuration environment variables to set starting Vault"
+  type        = map(string)
+  default     = null
+}
+
+variable "install_dir" {
+  type        = string
+  description = "The directory where the vault binary will be installed"
+  default     = "/opt/vault/bin"
+}
+
+variable "license" {
+  type        = string
+  sensitive   = true
+  description = "The value of the Vault license"
+  default     = null
+}
+
+variable "log_level" {
+  type        = string
+  description = "The vault service log level"
+  default     = "info"
+
+  validation {
+    condition     = contains(["trace", "debug", "info", "warn", "error"], var.log_level)
+    error_message = "The log_level must be one of 'trace', 'debug', 'info', 'warn', or 'error'."
+  }
+}
+
+variable "manage_service" {
+  type        = bool
+  description = "Manage the Vault service users and systemd unit. Disable this to use configuration in RPM and Debian packages"
+  default     = true
+}
+
+variable "seal_ha_beta" {
+  description = "Enable using Seal HA on clusters that meet minimum version requirements and are enterprise editions"
+  default     = true
+}
+
+variable "seal_key_name" {
+  type        = string
+  description = "The auto-unseal key name"
+  default     = null
+}
+
+variable "seal_key_name_secondary" {
+  type        = string
+  description = "The secondary auto-unseal key name"
+  default     = null
+}
+
+variable "seal_type" {
+  type        = string
+  description = "The method by which to unseal the Vault cluster"
+  default     = "awskms"
+
+  validation {
+    condition     = contains(["awskms", "shamir"], var.seal_type)
+    error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported."
+  }
+}
+
+variable "seal_type_secondary" {
+  type        = string
+  description = "A secondary HA seal method. Only supported in Vault Enterprise >= 1.15"
+  default     = "none"
+
+  validation {
+    condition     = contains(["awskms", "none"], var.seal_type_secondary)
+    error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported."
+  }
+}
+
+variable "service_username" {
+  type        = string
+  description = "The host username to own the vault service"
+  default     = "vault"
+}
+
+variable "storage_backend" {
+  type        = string
+  description = "The storage backend to use"
+  default     = "raft"
+
+  validation {
+    condition     = contains(["raft", "consul"], var.storage_backend)
+    error_message = "The storage_backend must be either raft or consul. No other storage backends are supported."
+  }
+}
+
+variable "storage_backend_attrs" {
+  type        = map(any)
+  description = "An optional set of key value pairs to inject into the storage block"
+  default     = {}
+}
+
+variable "storage_node_prefix" {
+  type        = string
+  description = "A prefix to use for each node in the Vault storage configuration"
+  default     = "node"
+}
+
+variable "target_hosts" {
+  description = "The target machines host addresses to use for the Vault cluster"
+  type = map(object({
+    private_ip = string
+    public_ip  = string
+  }))
+}

enos/modules/stop_vault/main.tf

@@ -0,0 +1,38 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    # We need to specify the provider source in each module until we publish it
+    # to the public registry
+    enos = {
+      source  = "app.terraform.io/hashicorp-qti/enos"
+      version = ">= 0.4.0"
+    }
+  }
+}
+
+variable "service_name" {
+  type        = string
+  description = "The Vault systemd service name"
+  default     = "vault"
+}
+
+variable "target_hosts" {
+  description = "The target machines host addresses to use for the Vault cluster"
+  type = map(object({
+    private_ip = string
+    public_ip  = string
+  }))
+}
+
+resource "enos_remote_exec" "shutdown_multiple_nodes" {
+  for_each = var.target_hosts
+  inline   = ["sudo systemctl stop ${var.service_name}.service; sleep 5"]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}

enos/modules/target_ec2_fleet/main.tf

@@ -23,10 +23,6 @@ data "aws_subnets" "vpc" {
   }
 }
 
-data "aws_kms_key" "kms_key" {
-  key_id = var.awskms_unseal_key_arn
-}
-
 data "aws_iam_policy_document" "target" {
   statement {
     resources = ["*"]
@@ -37,8 +33,11 @@ data "aws_iam_policy_document" "target" {
     ]
   }
 
-  statement {
+  dynamic "statement" {
-    resources = [var.awskms_unseal_key_arn]
+    for_each = var.seal_key_names
+
+    content {
+      resources = [statement.value]
 
       actions = [
         "kms:DescribeKey",
@@ -48,6 +47,7 @@ data "aws_iam_policy_document" "target" {
         "kms:GenerateDataKey"
       ]
     }
+  }
 }
 
 data "aws_iam_policy_document" "target_role" {

enos/modules/target_ec2_fleet/variables.tf

@@ -6,12 +6,6 @@ variable "ami_id" {
   type        = string
 }
 
-variable "awskms_unseal_key_arn" {
-  type        = string
-  description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
-  default     = null
-}
-
 variable "cluster_name" {
   type        = string
   description = "A unique cluster identifier"
@@ -73,6 +67,12 @@ variable "project_name" {
   type        = string
 }
 
+variable "seal_key_names" {
+  type        = list(string)
+  description = "The key management seal key names"
+  default     = null
+}
+
 variable "ssh_allow_ips" {
   description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
   type        = list(string)

enos/modules/target_ec2_instances/main.tf

@@ -53,10 +53,6 @@ data "aws_subnets" "vpc" {
   }
 }
 
-data "aws_kms_key" "kms_key" {
-  key_id = var.awskms_unseal_key_arn
-}
-
 data "aws_iam_policy_document" "target" {
   statement {
     resources = ["*"]
@@ -67,8 +63,11 @@ data "aws_iam_policy_document" "target" {
     ]
   }
 
-  statement {
+  dynamic "statement" {
-    resources = [var.awskms_unseal_key_arn]
+    for_each = var.seal_key_names
+
+    content {
+      resources = [statement.value]
 
       actions = [
         "kms:DescribeKey",
@@ -78,6 +77,7 @@ data "aws_iam_policy_document" "target" {
         "kms:GenerateDataKey"
       ]
     }
+  }
 }
 
 data "aws_iam_policy_document" "target_instance_role" {

enos/modules/target_ec2_instances/variables.tf

@@ -6,12 +6,6 @@ variable "ami_id" {
   type        = string
 }
 
-variable "awskms_unseal_key_arn" {
-  type        = string
-  description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
-  default     = null
-}
-
 variable "cluster_name" {
   type        = string
   description = "A unique cluster identifier"
@@ -53,6 +47,12 @@ variable "project_name" {
   type        = string
 }
 
+variable "seal_key_names" {
+  type        = list(string)
+  description = "The key management seal key names"
+  default     = null
+}
+
 variable "ssh_allow_ips" {
   description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
   type        = list(string)

enos/modules/target_ec2_shim/main.tf

@@ -13,7 +13,6 @@ terraform {
 }
 
 variable "ami_id" { default = null }
-variable "awskms_unseal_key_arn" { default = null }
 variable "cluster_name" { default = null }
 variable "cluster_tag_key" { default = null }
 variable "common_tags" { default = null }
@@ -25,6 +24,7 @@ variable "instance_mem_min" { default = null }
 variable "instance_types" { default = null }
 variable "max_price" { default = null }
 variable "project_name" { default = null }
+variable "seal_key_names" { default = null }
 variable "ssh_allow_ips" { default = null }
 variable "ssh_keypair" { default = null }
 variable "vpc_id" { default = null }

enos/modules/target_ec2_spot_fleet/main.tf

@@ -23,10 +23,6 @@ data "aws_subnets" "vpc" {
   }
 }
 
-data "aws_kms_key" "kms_key" {
-  key_id = var.awskms_unseal_key_arn
-}
-
 data "aws_iam_policy_document" "target" {
   statement {
     resources = ["*"]
@@ -37,8 +33,11 @@ data "aws_iam_policy_document" "target" {
     ]
   }
 
-  statement {
+  dynamic "statement" {
-    resources = [var.awskms_unseal_key_arn]
+    for_each = var.seal_key_names
+
+    content {
+      resources = [statement.value]
 
       actions = [
         "kms:DescribeKey",
@@ -48,6 +47,7 @@ data "aws_iam_policy_document" "target" {
         "kms:GenerateDataKey"
       ]
     }
+  }
 }
 
 data "aws_iam_policy_document" "target_role" {

enos/modules/target_ec2_spot_fleet/variables.tf

@@ -6,12 +6,6 @@ variable "ami_id" {
   type        = string
 }
 
-variable "awskms_unseal_key_arn" {
-  type        = string
-  description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
-  default     = null
-}
-
 variable "cluster_name" {
   type        = string
   description = "A unique cluster identifier"
@@ -73,6 +67,12 @@ variable "max_price" {
   default     = "0.0416"
 }
 
+variable "seal_key_names" {
+  type        = list(string)
+  description = "The key management seal key names"
+  default     = null
+}
+
 variable "ssh_allow_ips" {
   description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
   type        = list(string)

enos/modules/vault_cluster/main.tf

@@ -42,30 +42,6 @@ locals {
     "awskms" = 3
     "shamir" = null
   }
-  seal = {
-    "awskms" = {
-      type = "awskms"
-      attributes = {
-        kms_key_id = var.awskms_unseal_key_arn
-      }
-    }
-    "shamir" = {
-      type       = "shamir"
-      attributes = null
-    }
-  }
-  storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ?
-    merge(
-      {
-        node_id = "${var.storage_node_prefix}_${idx}"
-      },
-      var.storage_backend_addl_config
-    ) :
-    {
-      address = "127.0.0.1:8500"
-      path    = "vault"
-    })
-  ]
   vault_service_user = "vault"
 }
 
@@ -148,90 +124,35 @@ resource "enos_consul_start" "consul" {
   }
 }
 
-resource "enos_vault_start" "leader" {
+module "start_vault" {
+  source = "../start_vault"
+
   depends_on = [
     enos_consul_start.consul,
     enos_bundle_install.vault,
   ]
-  for_each = local.leader
 
-  bin_path    = local.bin_path
-  config_dir  = var.config_dir
-  environment = var.config_env_vars
-  config = {
-    api_addr     = "http://${var.target_hosts[each.value].private_ip}:8200"
-    cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
   cluster_name            = var.cluster_name
-    listener = {
-      type = "tcp"
-      attributes = {
-        address     = "0.0.0.0:8200"
-        tls_disable = "true"
-      }
-    }
-    log_level = var.log_level
-    storage = {
-      type       = var.storage_backend
-      attributes = ({ for key, value in local.storage_config[each.key] : key => value })
-    }
-    seal = local.seal[var.unseal_method]
-    ui   = true
-  }
-  license        = var.license
-  manage_service = var.manage_service
-  username       = local.vault_service_user
-  unit_name      = "vault"
-
-  transport = {
-    ssh = {
-      host = var.target_hosts[each.value].public_ip
-    }
-  }
-}
-
-resource "enos_vault_start" "followers" {
-  depends_on = [
-    enos_vault_start.leader,
-  ]
-  for_each = local.followers
-
-  bin_path    = local.bin_path
   config_dir              = var.config_dir
-  environment = var.config_env_vars
+  install_dir             = var.install_dir
-  config = {
-    api_addr     = "http://${var.target_hosts[each.value].private_ip}:8200"
-    cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
-    cluster_name = var.cluster_name
-    listener = {
-      type = "tcp"
-      attributes = {
-        address     = "0.0.0.0:8200"
-        tls_disable = "true"
-      }
-    }
-    log_level = var.log_level
-    storage = {
-      type       = var.storage_backend
-      attributes = { for key, value in local.storage_config[each.key] : key => value }
-    }
-    seal = local.seal[var.unseal_method]
-    ui   = true
-  }
   license                 = var.license
+  log_level               = var.log_level
   manage_service          = var.manage_service
-  username       = local.vault_service_user
+  seal_ha_beta            = var.seal_ha_beta
-  unit_name      = "vault"
+  seal_key_name           = var.seal_key_name
-
+  seal_key_name_secondary = var.seal_key_name_secondary
-  transport = {
+  seal_type               = var.seal_type
-    ssh = {
+  seal_type_secondary     = var.seal_type_secondary
-      host = var.target_hosts[each.value].public_ip
+  service_username        = local.vault_service_user
-    }
+  storage_backend         = var.storage_backend
-  }
+  storage_backend_attrs   = var.storage_backend_addl_config
+  storage_node_prefix     = var.storage_node_prefix
+  target_hosts            = var.target_hosts
 }
 
 resource "enos_vault_init" "leader" {
   depends_on = [
-    enos_vault_start.followers,
+    module.start_vault,
   ]
   for_each = toset([
     for idx, leader in local.leader : leader
@@ -239,13 +160,13 @@ resource "enos_vault_init" "leader" {
   ])
 
   bin_path   = local.bin_path
-  vault_addr = enos_vault_start.leader[0].config.api_addr
+  vault_addr = module.start_vault.leader[0].config.api_addr
 
-  key_shares    = local.key_shares[var.unseal_method]
+  key_shares    = local.key_shares[var.seal_type]
-  key_threshold = local.key_threshold[var.unseal_method]
+  key_threshold = local.key_threshold[var.seal_type]
 
-  recovery_shares    = local.recovery_shares[var.unseal_method]
+  recovery_shares    = local.recovery_shares[var.seal_type]
-  recovery_threshold = local.recovery_threshold[var.unseal_method]
+  recovery_threshold = local.recovery_threshold[var.seal_type]
 
   transport = {
     ssh = {
@@ -256,15 +177,15 @@ resource "enos_vault_init" "leader" {
 
 resource "enos_vault_unseal" "leader" {
   depends_on = [
-    enos_vault_start.followers,
+    module.start_vault,
     enos_vault_init.leader,
   ]
   for_each = enos_vault_init.leader // only unseal the leader if we initialized it
 
   bin_path    = local.bin_path
-  vault_addr  = enos_vault_start.leader[each.key].config.api_addr
+  vault_addr  = module.start_vault.leader[each.key].config.api_addr
-  seal_type   = var.unseal_method
+  seal_type   = var.seal_type
-  unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
+  unseal_keys = var.seal_type != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
 
   transport = {
     ssh = {
@@ -282,13 +203,13 @@ resource "enos_vault_unseal" "followers" {
   // initialized the cluster
   for_each = toset([
     for idx, follower in local.followers : follower
-    if var.unseal_method == "shamir" && var.initialize_cluster
+    if var.seal_type == "shamir" && var.initialize_cluster
   ])
 
   bin_path    = local.bin_path
-  vault_addr  = enos_vault_start.followers[each.key].config.api_addr
+  vault_addr  = module.start_vault.followers[each.key].config.api_addr
-  seal_type   = var.unseal_method
+  seal_type   = var.seal_type
-  unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
+  unseal_keys = var.seal_type != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
 
   transport = {
     ssh = {
@@ -303,7 +224,7 @@ resource "enos_vault_unseal" "followers" {
 // force_unseal to true.
 resource "enos_vault_unseal" "maybe_force_unseal" {
   depends_on = [
-    enos_vault_start.followers,
+    module.start_vault.followers,
   ]
   for_each = {
     for idx, host in var.target_hosts : idx => host
@@ -312,7 +233,7 @@ resource "enos_vault_unseal" "maybe_force_unseal" {
 
   bin_path   = local.bin_path
   vault_addr = "http://localhost:8200"
-  seal_type  = var.unseal_method
+  seal_type  = var.seal_type
   unseal_keys = coalesce(
     var.shamir_unseal_keys,
     try(enos_vault_init.leader[0].unseal_keys_hex, null),
@@ -325,38 +246,11 @@ resource "enos_vault_unseal" "maybe_force_unseal" {
   }
 }
 
-resource "enos_remote_exec" "vault_write_license" {
-  for_each = toset([
-    for idx, leader in local.leader : leader
-    if var.initialize_cluster
-  ])
-
-  depends_on = [
-    enos_vault_unseal.leader,
-    enos_vault_unseal.maybe_force_unseal,
-  ]
-
-  environment = {
-    BIN_PATH    = local.bin_path,
-    LICENSE     = coalesce(var.license, "none")
-    VAULT_TOKEN = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none")
-  }
-
-  scripts = [abspath("${path.module}/scripts/vault-write-license.sh")]
-
-  transport = {
-    ssh = {
-      host = var.target_hosts[each.value].public_ip
-    }
-  }
-}
-
 # We need to ensure that the directory used for audit logs is present and accessible to the vault
 # user on all nodes, since logging will only happen on the leader.
 resource "enos_remote_exec" "create_audit_log_dir" {
   depends_on = [
-    enos_vault_start.leader,
+    module.start_vault,
-    enos_vault_start.followers,
     enos_vault_unseal.leader,
     enos_vault_unseal.followers,
     enos_vault_unseal.maybe_force_unseal,

enos/modules/vault_cluster/variables.tf

@@ -12,12 +12,6 @@ variable "artifactory_release" {
   default     = null
 }
 
-variable "awskms_unseal_key_arn" {
-  type        = string
-  description = "The AWSKMS key ARN if using the awskms unseal method"
-  default     = null
-}
-
 variable "backend_cluster_name" {
   type        = string
   description = "The name of the backend cluster"
@@ -171,6 +165,45 @@ variable "root_token" {
   default     = null
 }
 
+variable "seal_ha_beta" {
+  description = "Enable using Seal HA on clusters that meet minimum version requirements and are enterprise editions"
+  default     = true
+}
+
+variable "seal_key_name" {
+  type        = string
+  description = "The auto-unseal key name"
+  default     = null
+}
+
+variable "seal_key_name_secondary" {
+  type        = string
+  description = "The secondary auto-unseal key name"
+  default     = null
+}
+
+variable "seal_type" {
+  type        = string
+  description = "The method by which to unseal the Vault cluster"
+  default     = "awskms"
+
+  validation {
+    condition     = contains(["awskms", "shamir"], var.seal_type)
+    error_message = "The seal_type must be either awskms or shamir. No other unseal methods are supported."
+  }
+}
+
+variable "seal_type_secondary" {
+  type        = string
+  description = "A secondary HA seal method. Only supported in Vault Enterprise >= 1.15"
+  default     = "none"
+
+  validation {
+    condition     = contains(["awskms", "none"], var.seal_type_secondary)
+    error_message = "The secondary_seal_type must be 'awskms' or 'none'. No other secondary unseal methods are supported."
+  }
+}
+
 variable "shamir_unseal_keys" {
   type        = list(string)
   description = "Shamir unseal keys. Often only used adding additional nodes to an already initialized cluster."
@@ -207,14 +240,3 @@ variable "target_hosts" {
     public_ip  = string
   }))
 }
-
-variable "unseal_method" {
-  type        = string
-  description = "The method by which to unseal the Vault cluster"
-  default     = "awskms"
-
-  validation {
-    condition     = contains(["awskms", "shamir"], var.unseal_method)
-    error_message = "The unseal_method must be either awskms or shamir. No other unseal methods are supported."
-  }
-}

enos/modules/vault_wait_for_leader/scripts/wait-for-leader.sh

@@ -23,14 +23,14 @@ test -x "$binpath" || fail "unable to locate vault binary at $binpath"
 findLeaderInPrivateIPs() {
   # Find the leader private IP address
   local leader_private_ip
-  if ! leader_private_ip=$($binpath read sys/leader -format=json | jq -r '.data.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")') ; then
+  if ! leader_private_ip=$($binpath read sys/leader -format=json | jq -er '.data.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")') ; then
     # Some older versions of vault don't support reading sys/leader. Fallback to the cli status.
-    if leader_private_ip=$($binpath status -format json | jq '.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")'); then
+    if ! leader_private_ip=$($binpath status -format json | jq -er '.leader_address | scan("[0-9]+.[0-9]+.[0-9]+.[0-9]+")'); then
       return 1
     fi
   fi
 
-  if isIn=$(jq -r --arg ip "$leader_private_ip" 'map(select(. == $ip)) | length == 1' <<< "$VAULT_INSTANCE_PRIVATE_IPS"); then
+  if isIn=$(jq -er --arg ip "$leader_private_ip" 'map(select(. == $ip)) | length == 1' <<< "$VAULT_INSTANCE_PRIVATE_IPS"); then
     if [[ "$isIn" == "true" ]]; then
       echo "$leader_private_ip"
       return 0

enos/modules/vault_wait_for_seal_rewrap/main.tf

@@ -0,0 +1,67 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "vault_install_dir" {
+  type        = string
+  description = "The directory where the Vault binary will be installed"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "vault_instance_count" {
+  type        = number
+  description = "The number of instances in the vault cluster"
+}
+
+variable "vault_hosts" {
+  type = map(object({
+    private_ip = string
+    public_ip  = string
+  }))
+  description = "The vault cluster hosts that can be expected as a leader"
+}
+
+variable "timeout" {
+  type        = number
+  description = "The max number of seconds to wait before timing out"
+  default     = 60
+}
+
+variable "retry_interval" {
+  type        = number
+  description = "How many seconds to wait between each retry"
+  default     = 2
+}
+
+locals {
+  private_ips = [for k, v in values(tomap(var.vault_hosts)) : tostring(v["private_ip"])]
+}
+
+resource "enos_remote_exec" "wait_for_seal_rewrap_to_be_completed" {
+  environment = {
+    RETRY_INTERVAL    = var.retry_interval
+    TIMEOUT_SECONDS   = var.timeout
+    VAULT_ADDR        = "http://127.0.0.1:8200"
+    VAULT_TOKEN       = var.vault_root_token
+    VAULT_INSTALL_DIR = var.vault_install_dir
+  }
+
+  scripts = [abspath("${path.module}/scripts/wait-for-seal-rewrap.sh")]
+
+  transport = {
+    ssh = {
+      host = var.vault_hosts[0].public_ip
+    }
+  }
+}

enos/modules/vault_wait_for_seal_rewrap/scripts/wait-for-seal-rewrap.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+
+set -e
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "$RETRY_INTERVAL" ]] && fail "RETRY_INTERVAL env variable has not been set"
+[[ -z "$TIMEOUT_SECONDS" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
+[[ -z "$VAULT_ADDR" ]] && fail "VAULT_ADDR env variable has not been set"
+[[ -z "$VAULT_INSTALL_DIR" ]] && fail "VAULT_INSTALL_DIR env variable has not been set"
+[[ -z "$VAULT_TOKEN" ]] && fail "VAULT_TOKEN env variable has not been set"
+
+binpath=${VAULT_INSTALL_DIR}/vault
+test -x "$binpath" || fail "unable to locate vault binary at $binpath"
+
+getRewrapData() {
+  $binpath read sys/sealwrap/rewrap -format=json | jq -eMc '.data'
+}
+
+waitForRewrap() {
+  local data
+  if ! data=$(getRewrapData); then
+    echo "failed getting /v1/sys/sealwrap/rewrap data" 1>&2
+    return 1
+  fi
+
+  if ! jq -e '.is_running == false' <<< "$data" &> /dev/null; then
+    echo "rewrap is running" 1>&2
+    return 1
+  fi
+
+  if ! jq -e '.entries.failed == 0' <<< "$data" &> /dev/null; then
+    local entries
+    entries=$(jq -Mc '.entries.failed' <<< "$data")
+    echo "rewrap has $entries failed entries" 1>&2
+    return 1
+  fi
+
+  if ! jq -e '.entries.processed == .entries.succeeded' <<< "$data" &> /dev/null; then
+    local processed
+    local succeeded
+    processed=$(jq -Mc '.entries.processed' <<< "$data")
+    succeeded=$(jq -Mc '.entries.succeeded' <<< "$data")
+    echo "the number of processed entries ($processed) does not equal then number of succeeded ($succeeded)" 1>&2
+    return 1
+  fi
+
+  return 0
+}
+
+begin_time=$(date +%s)
+end_time=$((begin_time + TIMEOUT_SECONDS))
+while [ "$(date +%s)" -lt "$end_time" ]; do
+  if waitForRewrap; then
+    exit 0
+  fi
+
+  sleep "$RETRY_INTERVAL"
+done
+
+fail "Timed out waiting for seal rewrap to be completed. Data:\n\t$(getRewrapData)"