scottk/vault
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/vault.git synced 2025-10-29 17:52:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

[QT-525] enos: use spot instances for Vault targets (#20037)

Browse Source 
The previous strategy for provisioning infrastructure targets was to use
the cheapest instances that could reliably perform as Vault cluster
nodes. With this change we introduce a new model for target node
infrastructure. We've replaced on-demand instances for a spot
fleet. While the spot price fluctuates based on dynamic pricing, 
capacity, region, instance type, and platform, cost savings for our
most common combinations range between 20-70%.

This change only includes spot fleet targets for Vault clusters.
We'll be updating our Consul backend bidding in another PR.

* Create a new `vault_cluster` module that handles installation,
  configuration, initializing, and unsealing Vault clusters.
* Create a `target_ec2_instances` module that can provision a group of
  instances on-demand.
* Create a `target_ec2_spot_fleet` module that can bid on a fleet of
  spot instances.
* Extend every Enos scenario to utilize the spot fleet target acquisition
  strategy and the `vault_cluster` module.
* Update our Enos CI modules to handle both the `aws-nuke` permissions
  and also the privileges to provision spot fleets.
* Only use us-east-1 and us-west-2 in our scenario matrices as costs are
  lower than us-west-1.

Signed-off-by: Ryan Cragun <me@ryan.ec>
This commit is contained in:
Ryan Cragun
2023-04-13 13:44:43 -06:00
committed by GitHub
parent b2e1ff5b7a
commit 1329a6b506
25 changed files with 2216 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.github/enos-run-matrices/build-github-oss-linux-amd64-zip.json vendored
View File

@@ -2,7 +2,7 @@
"include": [
{
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 3
},
{
@@ -12,7 +12,7 @@
},
{
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 1
},
{
@@ -22,7 +22,7 @@
},
{
"scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -32,7 +32,7 @@
},
{
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 5
},
{
@@ -42,7 +42,7 @@
},
{
"scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{

10
.github/enos-run-matrices/build-github-oss-linux-arm64-zip.json vendored
View File

@@ -7,7 +7,7 @@
},
{
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -17,7 +17,7 @@
},
{
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 4
},
{
@@ -27,7 +27,7 @@
},
{
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 1
},
{
@@ -37,7 +37,7 @@
},
{
"scenario": "upgrade backend:consul consul_version:1.12.7 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 3
},
{
@@ -47,7 +47,7 @@
},
{
"scenario": "upgrade backend:consul consul_version:1.14.2 distro:rhel seal:awskms arch:arm64 artifact_source:crt edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 5
}
]

10
.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-amd64-zip.json vendored
View File

@@ -2,7 +2,7 @@
"include": [
{
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -12,7 +12,7 @@
},
{
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -22,7 +22,7 @@
},
{
"scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -32,7 +32,7 @@
},
{
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -42,7 +42,7 @@
},
{
"scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{

10
.github/enos-run-matrices/enos_release_testing_oss-artifactory-oss-linux-arm64-zip.json vendored
View File

@@ -7,17 +7,17 @@
},
{
"scenario": "smoke backend:raft consul_version:1.14.2 distro:ubuntu seal:awskms arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
"scenario": "smoke backend:consul consul_version:1.12.7 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 1
},
{
"scenario": "smoke backend:consul consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -27,7 +27,7 @@
},
{
"scenario": "upgrade backend:raft consul_version:1.14.2 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 2
},
{
@@ -42,7 +42,7 @@
},
{
"scenario": "upgrade backend:consul consul_version:1.13.4 distro:ubuntu seal:shamir arch:amd64 artifact_source:artifactory edition:oss artifact_type:bundle",
"aws_region": "us-west-1",
"aws_region": "us-east-1",
"test_group": 1
},
{

6
.gitignore vendored
View File

@@ -67,6 +67,10 @@ enos/.terraform/*
enos/.terraform.lock.hcl
enos/*.tfstate
enos/*.tfstate.*
enos/**/.terraform/*
enos/**/.terraform.lock.hcl
enos/**/*.tfstate
enos/**/*.tfstate.*
.DS_Store
.idea
@@ -127,4 +131,4 @@ website/components/node_modules
.releaser/
*.log
tools/godoctests/.bin
tools/godoctests/.bin

71
enos/ci/service-user-iam/main.tf
View File

@@ -31,6 +31,7 @@ resource "aws_iam_role" "role" {
data "aws_iam_policy_document" "assume_role_policy_document" {
provider = aws.us_east_1
statement {
effect = "Allow"
actions = ["sts:AssumeRole"]
@@ -46,11 +47,47 @@ resource "aws_iam_role_policy" "role_policy" {
provider = aws.us_east_1
role = aws_iam_role.role.name
name = "${local.service_user}_policy"
policy = data.aws_iam_policy_document.iam_policy_document.json
policy = data.aws_iam_policy_document.role_policy.json
}
data "aws_iam_policy_document" "iam_policy_document" {
data "aws_iam_policy_document" "role_policy" {
source_policy_documents = [
data.aws_iam_policy_document.enos_scenario.json,
data.aws_iam_policy_document.aws_nuke.json,
]
}
data "aws_iam_policy_document" "aws_nuke" {
provider = aws.us_east_1
statement {
effect = "Allow"
actions = [
"ec2:DescribeInternetGateways",
"ec2:DescribeNatGateways",
"ec2:DescribeRegions",
"ec2:DescribeVpnGateways",
"iam:DeleteAccessKey",
"iam:DeleteUser",
"iam:DeleteUserPolicy",
"iam:GetUser",
"iam:ListAccessKeys",
"iam:ListAccountAliases",
"iam:ListGroupsForUser",
"iam:ListUserPolicies",
"iam:ListUserTags",
"iam:ListUsers",
"iam:UntagUser",
"servicequotas:ListServiceQuotas"
]
resources = ["*"]
}
}
data "aws_iam_policy_document" "enos_scenario" {
provider = aws.us_east_1
statement {
effect = "Allow"
actions = [
@@ -58,19 +95,27 @@ data "aws_iam_policy_document" "iam_policy_document" {
"ec2:AttachInternetGateway",
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotFleetRequests",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateInternetGateway",
"ec2:CreateKeyPair",
"ec2:CreateLaunchTemplate",
"ec2:CreateLaunchTemplateVersion",
"ec2:CreateRoute",
"ec2:CreateRouteTable",
"ec2:CreateSecurityGroup",
"ec2:CreateSpotDatafeedSubscription",
"ec2:CreateSubnet",
"ec2:CreateTags",
"ec2:CreateVolume",
"ec2:CreateVPC",
"ec2:DeleteInternetGateway",
"ec2:DeleteLaunchTemplate",
"ec2:DeleteLaunchTemplateVersions",
"ec2:DeleteKeyPair",
"ec2:DeleteRouteTable",
"ec2:DeleteSecurityGroup",
"ec2:DeleteSpotDatafeedSubscription",
"ec2:DeleteSubnet",
"ec2:DeleteTags",
"ec2:DeleteVolume",
@@ -84,14 +129,22 @@ data "aws_iam_policy_document" "iam_policy_document" {
"ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeInstanceTypes",
"ec2:DescribeInternetGateways",
"ec2:DescribeInternetGateways",
"ec2:DescribeKeyPairs",
"ec2:DescribeLaunchTemplates",
"ec2:DescribeLaunchTemplateVersions",
"ec2:DescribeNatGateways",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeRegions",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotDatafeedSubscription",
"ec2:DescribeSpotFleetInstances",
"ec2:DescribeSpotFleetInstanceRequests",
"ec2:DescribeSpotFleetRequests",
"ec2:DescribeSpotFleetRequestHistory",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeTags",
"ec2:DescribeVolumes",
@@ -102,14 +155,21 @@ data "aws_iam_policy_document" "iam_policy_document" {
"ec2:DescribeVpnGateways",
"ec2:DetachInternetGateway",
"ec2:DisassociateRouteTable",
"ec2:GetLaunchTemplateData",
"ec2:GetSpotPlacementScores",
"ec2:ImportKeyPair",
"ec2:ModifyInstanceAttribute",
"ec2:ModifyLaunchTemplate",
"ec2:ModifySpotFleetRequest",
"ec2:ModifySubnetAttribute",
"ec2:ModifyVPCAttribute",
"ec2:RequestSpotInstances",
"ec2:RequestSpotFleet",
"ec2:ResetInstanceAttribute",
"ec2:RevokeSecurityGroupEgress",
"ec2:RevokeSecurityGroupIngress",
"ec2:RunInstances",
"ec2:SendSpotInstanceInterruptions",
"ec2:TerminateInstances",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeTargetGroups",
@@ -118,11 +178,10 @@ data "aws_iam_policy_document" "iam_policy_document" {
"iam:CreateInstanceProfile",
"iam:CreatePolicy",
"iam:CreateRole",
"iam:CreateRole",
"iam:CreateServiceLinkedRole",
"iam:DeleteInstanceProfile",
"iam:DeletePolicy",
"iam:DeleteRole",
"iam:DeleteRole",
"iam:DeleteRolePolicy",
"iam:DetachRolePolicy",
"iam:GetInstanceProfile",
@@ -135,7 +194,6 @@ data "aws_iam_policy_document" "iam_policy_document" {
"iam:ListPolicies",
"iam:ListRolePolicies",
"iam:ListRoles",
"iam:ListRoles",
"iam:PassRole",
"iam:PutRolePolicy",
"iam:RemoveRoleFromInstanceProfile",
@@ -153,6 +211,7 @@ data "aws_iam_policy_document" "iam_policy_document" {
"kms:ScheduleKeyDeletion",
"servicequotas:ListServiceQuotas"
]
resources = ["*"]
}
}

39
enos/ci/service-user-iam/service-quotas.tf
View File

@@ -4,33 +4,62 @@
locals {
// This is the code of the service quota to request a change for. Each adjustable limit has a
// unique code. See, https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/servicequotas_service_quota#quota_code
subnets_per_vps_quota = "L-F678F1CE"
subnets_per_vpcs_quota = "L-F678F1CE"
standard_spot_instance_requests_quota = "L-34B43A08"
}
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_1" {
provider = aws.us_east_2
quota_code = local.subnets_per_vps_quota
quota_code = local.subnets_per_vpcs_quota
service_code = "vpc"
value = 50
}
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_east_2" {
provider = aws.us_east_2
quota_code = local.subnets_per_vps_quota
quota_code = local.subnets_per_vpcs_quota
service_code = "vpc"
value = 50
}
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_1" {
provider = aws.us_west_1
quota_code = local.subnets_per_vps_quota
quota_code = local.subnets_per_vpcs_quota
service_code = "vpc"
value = 50
}
resource "aws_servicequotas_service_quota" "vpcs_per_region_us_west_2" {
provider = aws.us_west_2
quota_code = local.subnets_per_vps_quota
quota_code = local.subnets_per_vpcs_quota
service_code = "vpc"
value = 50
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_1" {
provider = aws.us_east_2
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_east_2" {
provider = aws.us_east_2
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_1" {
provider = aws.us_west_1
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}
resource "aws_servicequotas_service_quota" "spot_requests_per_region_us_west_2" {
provider = aws.us_west_2
quota_code = local.standard_spot_instance_requests_quota
service_code = "ec2"
value = 640
}

32
enos/enos-modules.hcl
View File

@@ -68,6 +68,27 @@ module "shutdown_multiple_nodes" {
source = "./modules/shutdown_multiple_nodes"
}
module "target_ec2_instances" {
source = "./modules/target_ec2_instances"
common_tags = var.tags
instance_count = var.vault_instance_count
project_name = var.project_name
ssh_keypair = var.aws_ssh_keypair_name
}
module "target_ec2_spot_fleet" {
source = "./modules/target_ec2_spot_fleet"
common_tags = var.tags
instance_mem_min = 4096
instance_cpu_min = 2
project_name = var.project_name
// Current on-demand cost of t3.medium in us-east.
spot_price_max = "0.0416"
ssh_keypair = var.aws_ssh_keypair_name
}
module "vault_agent" {
source = "./modules/vault_agent"
@@ -75,7 +96,6 @@ module "vault_agent" {
vault_instance_count = var.vault_instance_count
}
module "vault_verify_agent_output" {
source = "./modules/vault_verify_agent_output"
@@ -83,15 +103,9 @@ module "vault_verify_agent_output" {
}
module "vault_cluster" {
source = "app.terraform.io/hashicorp-qti/aws-vault/enos"
# source = "../../terraform-enos-aws-vault"
source = "./modules/vault_cluster"
common_tags = var.tags
environment = "ci"
instance_count = var.vault_instance_count
project_name = var.project_name
ssh_aws_keypair = var.aws_ssh_keypair_name
vault_install_dir = var.vault_install_dir
install_dir = var.vault_install_dir
}
module "vault_get_cluster_ips" {

148
enos/enos-scenario-agent.hcl
View File

@@ -25,13 +25,18 @@ scenario "agent" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
}
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"]
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
packages = ["jq"]
enos_provider = {
rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu
}
install_artifactory_artifact = local.bundle_path == null
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
@@ -54,21 +59,21 @@ scenario "agent" {
module = "build_${matrix.artifact_source}"
variables {
build_tags = try(var.vault_local_build_tags, local.build_tags[matrix.edition])
bundle_path = local.bundle_path
goarch = matrix.arch
goos = "linux"
artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
arch = matrix.artifact_source == "artifactory" ? matrix.arch : null
vault_product_version = var.vault_product_version
artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null
distro = matrix.artifact_source == "artifactory" ? matrix.distro : null
edition = matrix.artifact_source == "artifactory" ? matrix.edition : null
instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null
revision = var.vault_revision
build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : local.build_tags[matrix.edition]
bundle_path = local.bundle_path
goarch = matrix.arch
goos = "linux"
artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
arch = matrix.artifact_source == "artifactory" ? matrix.arch : null
product_version = var.vault_product_version
artifact_type = matrix.artifact_source == "artifactory" ? var.vault_artifact_type : null
distro = matrix.artifact_source == "artifactory" ? matrix.distro : null
edition = matrix.artifact_source == "artifactory" ? matrix.edition : null
instance_type = matrix.artifact_source == "artifactory" ? local.vault_instance_type : null
revision = var.vault_revision
}
}
@@ -102,28 +107,29 @@ scenario "agent" {
}
}
step "create_backend_cluster" {
module = "backend_raft"
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids["ubuntu"]["amd64"]
common_tags = local.tags
instance_type = var.backend_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
vpc_id = step.create_vpc.vpc_id
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
@@ -131,28 +137,25 @@ scenario "agent" {
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = "raft"
unseal_method = "shamir"
vault_local_artifact_path = local.bundle_path
vault_artifactory_release = local.install_artifactory_artifact ? step.build_vault.vault_artifactory_release : null
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = var.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
storage_backend = "raft"
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = "shamir"
}
}
step "start_vault_agent" {
module = "vault_agent"
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster,
]
@@ -162,8 +165,8 @@ scenario "agent" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.root_token
vault_agent_template_destination = "/tmp/agent_output.txt"
vault_agent_template_contents = "{{ with secret \\\"auth/token/lookup-self\\\" }}orphan={{ .Data.orphan }} display_name={{ .Data.display_name }}{{ end }}"
}
@@ -181,49 +184,64 @@ scenario "agent" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_agent_template_destination = "/tmp/agent_output.txt"
vault_agent_expected_output = "orphan=true display_name=approle"
}
}
output "vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.create_vault_cluster.instance_ids
output "awkms_unseal_key_arn" {
description = "The Vault cluster KMS key arn"
value = step.create_vpc.kms_key_arn
}
output "vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.instance_public_ips
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "vault_cluster_priv_ips" {
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips
value = step.create_vault_cluster.private_ips
}
output "vault_cluster_key_id" {
description = "The Vault cluster Key ID"
value = step.create_vault_cluster.key_id
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "vault_cluster_root_token" {
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token
value = step.create_vault_cluster.root_token
}
output "vault_cluster_unseal_keys_b64" {
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
value = step.create_vault_cluster.unseal_keys_b64
}
output "vault_cluster_unseal_keys_hex" {
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
value = step.create_vault_cluster.unseal_keys_hex
}
}

264
enos/enos-scenario-autopilot.hcl
View File

@@ -15,6 +15,12 @@ scenario "autopilot" {
edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"]
artifact_type = ["package"]
}
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
}
terraform_cli = terraform_cli.default
@@ -32,12 +38,17 @@ scenario "autopilot" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
}
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"]
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
packages = ["jq"]
enos_provider = {
rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu
}
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
@@ -108,36 +119,52 @@ scenario "autopilot" {
}
}
# This step creates a Vault cluster using a bundle downloaded from
# releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_vpc,
step.build_vault,
]
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = "raft"
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
packages = local.packages
release = var.vault_autopilot_initial_release
storage_backend = "raft"
storage_backend_addl_config = {
autopilot_upgrade_version = var.vault_autopilot_initial_release.version
}
unseal_method = matrix.seal
vault_install_dir = local.vault_install_dir
vault_release = var.vault_autopilot_initial_release
vault_license = step.read_license.license
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = matrix.seal
}
}
@@ -155,12 +182,13 @@ scenario "autopilot" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_write_test_data" {
module = module.vault_verify_write_data
depends_on = [
@@ -175,9 +203,9 @@ scenario "autopilot" {
variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -189,8 +217,25 @@ scenario "autopilot" {
}
}
# This step creates a new Vault cluster using a bundle or package
# from the matrix.artifact_source, with the var.vault_product_version
step "create_vault_cluster_upgrade_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
cluster_name = step.create_vault_cluster_targets.cluster_name
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "upgrade_vault_cluster_with_autopilot" {
module = module.vault_cluster
depends_on = [
@@ -205,28 +250,25 @@ scenario "autopilot" {
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = "raft"
storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config
unseal_method = matrix.seal
vault_cluster_tag = step.create_vault_cluster.vault_cluster_tag
vault_init = false
vault_install_dir = local.vault_install_dir
vault_license = step.read_license.license
vault_local_artifact_path = local.bundle_path
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_node_prefix = "upgrade_node"
vault_root_token = step.create_vault_cluster.vault_root_token
vault_unseal_when_no_init = matrix.seal == "shamir"
vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
force_unseal = matrix.seal == "shamir"
initialize_cluster = false
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
root_token = step.create_vault_cluster.root_token
shamir_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
storage_backend = "raft"
storage_backend_addl_config = step.create_autopilot_upgrade_storageconfig.storage_addl_config
storage_node_prefix = "upgrade_node"
target_hosts = step.create_vault_cluster_upgrade_targets.hosts
unseal_method = matrix.seal
}
}
@@ -234,6 +276,7 @@ scenario "autopilot" {
module = module.vault_verify_unsealed
depends_on = [
step.create_vault_cluster,
step.create_vault_cluster_upgrade_targets,
step.upgrade_vault_cluster_with_autopilot,
]
@@ -243,7 +286,7 @@ scenario "autopilot" {
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances
vault_instances = step.create_vault_cluster_upgrade_targets.hosts
}
}
@@ -260,14 +303,15 @@ scenario "autopilot" {
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances
vault_root_token = step.upgrade_vault_cluster_with_autopilot.vault_root_token
vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token
}
}
step "verify_autopilot_await_server_removal_state" {
module = module.vault_verify_autopilot
depends_on = [
step.create_vault_cluster_upgrade_targets,
step.upgrade_vault_cluster_with_autopilot,
step.verify_raft_auto_join_voter
]
@@ -280,8 +324,8 @@ scenario "autopilot" {
vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_autopilot_upgrade_status = "await-server-removal"
vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.upgrade_vault_cluster_with_autopilot.root_token
}
}
@@ -289,6 +333,7 @@ scenario "autopilot" {
module = module.vault_get_cluster_ips
depends_on = [
step.create_vault_cluster,
step.create_vault_cluster_upgrade_targets,
step.get_vault_cluster_ips,
step.upgrade_vault_cluster_with_autopilot
]
@@ -298,11 +343,11 @@ scenario "autopilot" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
added_vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
node_public_ip = step.get_vault_cluster_ips.leader_public_ip
added_vault_instances = step.create_vault_cluster_targets.hosts
}
}
@@ -329,6 +374,7 @@ scenario "autopilot" {
step "raft_remove_peers" {
module = module.vault_raft_remove_peer
depends_on = [
step.create_vault_cluster_upgrade_targets,
step.get_updated_vault_cluster_ips,
step.upgrade_vault_cluster_with_autopilot,
step.verify_autopilot_await_server_removal_state
@@ -339,11 +385,11 @@ scenario "autopilot" {
}
variables {
vault_install_dir = local.vault_install_dir
operator_instance = step.get_updated_vault_cluster_ips.leader_public_ip
remove_vault_instances = step.create_vault_cluster.vault_instances
remove_vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_instance_count = 3
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -359,7 +405,7 @@ scenario "autopilot" {
}
variables {
old_vault_instances = step.create_vault_cluster.vault_instances
old_vault_instances = step.create_vault_cluster_targets.hosts
vault_instance_count = 3
}
}
@@ -367,6 +413,7 @@ scenario "autopilot" {
step "verify_autopilot_idle_state" {
module = module.vault_verify_autopilot
depends_on = [
step.create_vault_cluster_upgrade_targets,
step.upgrade_vault_cluster_with_autopilot,
step.verify_raft_auto_join_voter,
step.remove_old_nodes
@@ -380,15 +427,16 @@ scenario "autopilot" {
vault_autopilot_upgrade_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_autopilot_upgrade_status = "idle"
vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_undo_logs_status" {
skip_step = try(semverconstraint(var.vault_product_version, "<1.13.0-0"), true)
skip_step = semverconstraint(var.vault_product_version, "<1.13.0-0")
module = module.vault_verify_undo_logs
depends_on = [
step.create_vault_cluster_upgrade_targets,
step.remove_old_nodes,
step.upgrade_vault_cluster_with_autopilot,
step.verify_autopilot_idle_state
@@ -400,78 +448,78 @@ scenario "autopilot" {
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.upgrade_vault_cluster_with_autopilot.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_instances = step.create_vault_cluster_upgrade_targets.hosts
vault_root_token = step.create_vault_cluster.root_token
}
}
output "vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.create_vault_cluster.instance_ids
output "awskms_unseal_key_arn" {
description = "The Vault cluster KMS key arn"
value = step.create_vpc.kms_key_arn
}
output "vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.instance_public_ips
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "vault_cluster_priv_ips" {
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips
value = step.create_vault_cluster.private_ips
}
output "vault_cluster_key_id" {
description = "The Vault cluster Key ID"
value = step.create_vault_cluster.key_id
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "vault_cluster_root_token" {
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token
value = step.create_vault_cluster.root_token
}
output "vault_cluster_unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
}
output "vault_cluster_recovery_key_shares" {
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.vault_recovery_key_shares
value = step.create_vault_cluster.recovery_key_shares
}
output "vault_cluster_recovery_keys_b64" {
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.vault_recovery_keys_b64
value = step.create_vault_cluster.recovery_keys_b64
}
output "vault_cluster_recovery_keys_hex" {
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.vault_recovery_keys_hex
value = step.create_vault_cluster.recovery_keys_hex
}
output "vault_cluster_unseal_keys_hex" {
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.unseal_keys_b64
}
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex
value = step.create_vault_cluster.unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
output "upgrade_hosts" {
description = "The Vault cluster target hosts"
value = step.upgrade_vault_cluster_with_autopilot.target_hosts
}
output "upgraded_vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.upgrade_vault_cluster_with_autopilot.instance_ids
}
output "upgraded_vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.upgrade_vault_cluster_with_autopilot.instance_public_ips
}
output "upgraded_vault_cluster_priv_ips" {
output "upgrade_private_ips" {
description = "The Vault cluster private IPs"
value = step.upgrade_vault_cluster_with_autopilot.instance_private_ips
value = step.upgrade_vault_cluster_with_autopilot.private_ips
}
output "upgrade_public_ips" {
description = "The Vault cluster public IPs"
value = step.upgrade_vault_cluster_with_autopilot.public_ips
}
}

376
enos/enos-scenario-replication.hcl
View File

@@ -22,6 +22,12 @@ scenario "replication" {
edition = ["ent.fips1402", "ent.hsm.fips1402"]
artifact_type = ["package"]
}
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
}
terraform_cli = terraform_cli.default
@@ -39,12 +45,17 @@ scenario "replication" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
}
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"]
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
packages = ["jq"]
enos_provider = {
rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu
}
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
@@ -134,37 +145,55 @@ scenario "replication" {
}
}
step "create_vault_primary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_primary_backend_cluster,
step.build_vault,
]
step "create_primary_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_primary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_primary_backend_cluster,
step.build_vault,
step.create_primary_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_primary_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag
consul_release = matrix.primary_backend == "consul" ? {
edition = var.backend_edition
version = matrix.consul_version
} : null
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.primary_backend
unseal_method = matrix.primary_seal
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_license = step.read_license.license
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
storage_backend = matrix.primary_backend
target_hosts = step.create_primary_cluster_targets.hosts
unseal_method = matrix.primary_seal
}
}
@@ -189,44 +218,62 @@ scenario "replication" {
}
}
step "create_vault_secondary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_secondary_backend_cluster,
step.build_vault,
]
step "create_secondary_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_secondary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_secondary_backend_cluster,
step.build_vault,
step.create_secondary_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_secondary_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_secondary_backend_cluster.consul_cluster_tag
consul_release = matrix.secondary_backend == "consul" ? {
edition = var.backend_edition
version = matrix.consul_version
} : null
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.secondary_backend
unseal_method = matrix.secondary_seal
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_license = step.read_license.license
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
storage_backend = matrix.secondary_backend
target_hosts = step.create_secondary_cluster_targets.hosts
unseal_method = matrix.secondary_seal
}
}
step "verify_vault_primary_unsealed" {
step "verify_that_vault_primary_cluster_is_unsealed" {
module = module.vault_verify_unsealed
depends_on = [
step.create_vault_primary_cluster
step.create_primary_cluster
]
providers = {
@@ -234,15 +281,15 @@ scenario "replication" {
}
variables {
vault_instances = step.create_vault_primary_cluster.vault_instances
vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
}
}
step "verify_vault_secondary_unsealed" {
step "verify_that_vault_secondary_cluster_is_unsealed" {
module = module.vault_verify_unsealed
depends_on = [
step.create_vault_secondary_cluster
step.create_secondary_cluster
]
providers = {
@@ -250,42 +297,42 @@ scenario "replication" {
}
variables {
vault_instances = step.create_vault_secondary_cluster.vault_instances
vault_instances = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
}
}
step "get_primary_cluster_ips" {
module = module.vault_get_cluster_ips
depends_on = [step.verify_vault_primary_unsealed]
depends_on = [step.verify_that_vault_primary_cluster_is_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_instances = step.create_vault_primary_cluster.vault_instances
vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_root_token = step.create_primary_cluster.root_token
}
}
step "get_secondary_cluster_ips" {
module = module.vault_get_cluster_ips
depends_on = [step.verify_vault_secondary_unsealed]
depends_on = [step.verify_that_vault_secondary_cluster_is_unsealed]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_instances = step.create_vault_secondary_cluster.vault_instances
vault_instances = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_secondary_cluster.vault_root_token
vault_root_token = step.create_secondary_cluster.root_token
}
}
step "verify_vault_primary_write_data" {
step "write_test_data_on_primary" {
module = module.vault_verify_write_data
depends_on = [step.get_primary_cluster_ips]
@@ -297,9 +344,9 @@ scenario "replication" {
variables {
leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
leader_private_ip = step.get_primary_cluster_ips.leader_private_ip
vault_instances = step.create_vault_primary_cluster.vault_instances
vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_root_token = step.create_primary_cluster.root_token
}
}
@@ -307,7 +354,7 @@ scenario "replication" {
module = module.vault_setup_perf_primary
depends_on = [
step.get_primary_cluster_ips,
step.verify_vault_primary_write_data
step.write_test_data_on_primary
]
providers = {
@@ -318,7 +365,7 @@ scenario "replication" {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
primary_leader_private_ip = step.get_primary_cluster_ips.leader_private_ip
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_root_token = step.create_primary_cluster.root_token
}
}
@@ -333,7 +380,7 @@ scenario "replication" {
variables {
primary_leader_public_ip = step.get_primary_cluster_ips.leader_public_ip
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_root_token = step.create_primary_cluster.root_token
}
}
@@ -349,7 +396,7 @@ scenario "replication" {
secondary_leader_public_ip = step.get_secondary_cluster_ips.leader_public_ip
secondary_leader_private_ip = step.get_secondary_cluster_ips.leader_private_ip
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_secondary_cluster.vault_root_token
vault_root_token = step.create_secondary_cluster.root_token
wrapping_token = step.generate_secondary_token.secondary_token
}
}
@@ -359,8 +406,8 @@ scenario "replication" {
step "unseal_secondary_followers" {
module = module.vault_unseal_nodes
depends_on = [
step.create_vault_primary_cluster,
step.create_vault_secondary_cluster,
step.create_primary_cluster,
step.create_secondary_cluster,
step.get_secondary_cluster_ips,
step.configure_performance_replication_secondary
]
@@ -372,12 +419,12 @@ scenario "replication" {
variables {
follower_public_ips = step.get_secondary_cluster_ips.follower_public_ips
vault_install_dir = local.vault_install_dir
vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : step.create_vault_primary_cluster.vault_recovery_keys_hex
vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : step.create_primary_cluster.recovery_keys_hex
vault_seal_type = matrix.primary_seal == "shamir" ? matrix.primary_seal : matrix.secondary_seal
}
}
step "verify_vault_secondary_unsealed_after_replication" {
step "verify_secondary_cluster_is_unsealed_after_enabling_replication" {
module = module.vault_verify_unsealed
depends_on = [
step.unseal_secondary_followers
@@ -388,14 +435,14 @@ scenario "replication" {
}
variables {
vault_instances = step.create_vault_secondary_cluster.vault_instances
vault_instances = step.create_secondary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
}
}
step "verify_performance_replication" {
module = module.vault_verify_performance_replication
depends_on = [step.verify_vault_secondary_unsealed_after_replication]
depends_on = [step.verify_secondary_cluster_is_unsealed_after_enabling_replication]
providers = {
enos = local.enos_provider[matrix.distro]
@@ -415,7 +462,7 @@ scenario "replication" {
depends_on = [
step.verify_performance_replication,
step.get_secondary_cluster_ips,
step.verify_vault_primary_write_data
step.write_test_data_on_primary
]
providers = {
@@ -428,13 +475,32 @@ scenario "replication" {
}
}
step "add_primary_cluster_nodes" {
step "create_more_primary_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "add_more_nodes_to_primary_cluster" {
module = module.vault_cluster
depends_on = [
step.create_vpc,
step.create_primary_backend_cluster,
step.create_vault_primary_cluster,
step.verify_replicated_data
step.create_primary_cluster,
step.verify_replicated_data,
step.create_more_primary_cluster_targets
]
providers = {
@@ -442,45 +508,42 @@ scenario "replication" {
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_primary_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_primary_backend_cluster.consul_cluster_tag
consul_release = matrix.primary_backend == "consul" ? {
edition = var.backend_edition
version = matrix.consul_version
} : null
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.primary_backend
unseal_method = matrix.primary_seal
vault_cluster_tag = step.create_vault_primary_cluster.vault_cluster_tag
vault_init = false
vault_license = step.read_license.license
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_node_prefix = "newprimary_node"
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_unseal_when_no_init = matrix.primary_seal == "shamir"
vault_unseal_keys = matrix.primary_seal == "shamir" ? step.create_vault_primary_cluster.vault_unseal_keys_hex : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
force_unseal = matrix.primary_seal == "shamir"
initialize_cluster = false
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
root_token = step.create_primary_cluster.root_token
shamir_unseal_keys = matrix.primary_seal == "shamir" ? step.create_primary_cluster.unseal_keys_hex : null
storage_backend = matrix.primary_backend
storage_node_prefix = "newprimary_node"
target_hosts = step.create_more_primary_cluster_targets.hosts
unseal_method = matrix.primary_seal
}
}
step "verify_add_node_unsealed" {
step "verify_more_primary_nodes_unsealed" {
module = module.vault_verify_unsealed
depends_on = [step.add_primary_cluster_nodes]
depends_on = [step.add_more_nodes_to_primary_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_instances = step.add_primary_cluster_nodes.vault_instances
vault_instances = step.create_more_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
}
}
@@ -489,9 +552,9 @@ scenario "replication" {
skip_step = matrix.primary_backend != "raft"
module = module.vault_verify_raft_auto_join_voter
depends_on = [
step.add_primary_cluster_nodes,
step.create_vault_primary_cluster,
step.verify_add_node_unsealed
step.add_more_nodes_to_primary_cluster,
step.create_primary_cluster,
step.verify_more_primary_nodes_unsealed
]
providers = {
@@ -499,9 +562,9 @@ scenario "replication" {
}
variables {
vault_instances = step.add_primary_cluster_nodes.vault_instances
vault_instances = step.create_more_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_primary_cluster.vault_root_token
vault_root_token = step.create_primary_cluster.root_token
}
}
@@ -509,7 +572,7 @@ scenario "replication" {
module = module.shutdown_node
depends_on = [
step.get_primary_cluster_ips,
step.verify_add_node_unsealed
step.verify_more_primary_nodes_unsealed
]
providers = {
@@ -540,7 +603,7 @@ scenario "replication" {
step "get_updated_primary_cluster_ips" {
module = module.vault_get_cluster_ips
depends_on = [
step.add_primary_cluster_nodes,
step.add_more_nodes_to_primary_cluster,
step.remove_primary_follower_1,
step.remove_primary_leader
]
@@ -550,10 +613,10 @@ scenario "replication" {
}
variables {
vault_instances = step.create_vault_primary_cluster.vault_instances
vault_instances = step.create_primary_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
added_vault_instances = step.add_primary_cluster_nodes.vault_instances
vault_root_token = step.create_vault_primary_cluster.vault_root_token
added_vault_instances = step.create_more_primary_cluster_targets.hosts
vault_root_token = step.create_primary_cluster.root_token
node_public_ip = step.get_primary_cluster_ips.follower_public_ip_2
}
}
@@ -575,112 +638,97 @@ scenario "replication" {
}
}
output "vault_primary_cluster_pub_ips" {
description = "The Vault primary cluster public IPs"
value = step.create_vault_primary_cluster.instance_public_ips
output "primary_cluster_hosts" {
description = "The Vault primary cluster target hosts"
value = step.create_primary_cluster_targets.hosts
}
output "vault_primary_cluster_priv_ips" {
description = "The Vault primary cluster private IPs"
value = step.create_vault_primary_cluster.instance_private_ips
output "primary_cluster_additional_hosts" {
description = "The Vault added new node on primary cluster target hosts"
value = step.create_more_primary_cluster_targets.hosts
}
output "vault_primary_newnode_pub_ip" {
description = "The Vault added new node on primary cluster public IP"
value = step.add_primary_cluster_nodes.instance_public_ips
}
output "vault_primary_newnode_priv_ip" {
description = "The Vault added new node on primary cluster private IP"
value = step.add_primary_cluster_nodes.instance_private_ips
}
output "vault_primary_cluster_root_token" {
output "primary_cluster_root_token" {
description = "The Vault primary cluster root token"
value = step.create_vault_primary_cluster.vault_root_token
value = step.create_primary_cluster.root_token
}
output "vault_primary_cluster_unseal_keys_b64" {
output "primary_cluster_unseal_keys_b64" {
description = "The Vault primary cluster unseal keys"
value = step.create_vault_primary_cluster.vault_unseal_keys_b64
value = step.create_primary_cluster.unseal_keys_b64
}
output "vault_primary_cluster_unseal_keys_hex" {
output "primary_cluster_unseal_keys_hex" {
description = "The Vault primary cluster unseal keys hex"
value = step.create_vault_primary_cluster.vault_unseal_keys_hex
value = step.create_primary_cluster.unseal_keys_hex
}
output "vault_primary_cluster_recovery_key_shares" {
output "primary_cluster_recovery_key_shares" {
description = "The Vault primary cluster recovery key shares"
value = step.create_vault_primary_cluster.vault_recovery_key_shares
value = step.create_primary_cluster.recovery_key_shares
}
output "vault_primary_cluster_recovery_keys_b64" {
output "primary_cluster_recovery_keys_b64" {
description = "The Vault primary cluster recovery keys b64"
value = step.create_vault_primary_cluster.vault_recovery_keys_b64
value = step.create_primary_cluster.recovery_keys_b64
}
output "vault_primary_cluster_recovery_keys_hex" {
output "primary_cluster_recovery_keys_hex" {
description = "The Vault primary cluster recovery keys hex"
value = step.create_vault_primary_cluster.vault_recovery_keys_hex
value = step.create_primary_cluster.recovery_keys_hex
}
output "vault_secondary_cluster_pub_ips" {
output "secondary_cluster_hosts" {
description = "The Vault secondary cluster public IPs"
value = step.create_vault_secondary_cluster.instance_public_ips
value = step.create_secondary_cluster_targets.hosts
}
output "vault_secondary_cluster_priv_ips" {
description = "The Vault secondary cluster private IPs"
value = step.create_vault_secondary_cluster.instance_private_ips
}
output "vault_primary_performance_replication_status" {
output "initial_primary_replication_status" {
description = "The Vault primary cluster performance replication status"
value = step.verify_performance_replication.primary_replication_status
}
output "vault_replication_known_primary_cluster_addrs" {
output "initial_known_primary_cluster_addresses" {
description = "The Vault secondary cluster performance replication status"
value = step.verify_performance_replication.known_primary_cluster_addrs
}
output "vault_secondary_performance_replication_status" {
output "initial_secondary_performance_replication_status" {
description = "The Vault secondary cluster performance replication status"
value = step.verify_performance_replication.secondary_replication_status
}
output "vault_primary_updated_performance_replication_status" {
description = "The Vault updated primary cluster performance replication status"
value = step.verify_updated_performance_replication.primary_replication_status
}
output "vault_updated_replication_known_primary_cluster_addrs" {
description = "The Vault secondary cluster performance replication status"
value = step.verify_updated_performance_replication.known_primary_cluster_addrs
}
output "verify_secondary_updated_performance_replication_status" {
description = "The Vault updated secondary cluster performance replication status"
value = step.verify_updated_performance_replication.secondary_replication_status
}
output "primary_replication_data_secondaries" {
output "intial_primary_replication_data_secondaries" {
description = "The Vault primary cluster secondaries connection status"
value = step.verify_performance_replication.primary_replication_data_secondaries
}
output "secondary_replication_data_primaries" {
output "initial_secondary_replication_data_primaries" {
description = "The Vault secondary cluster primaries connection status"
value = step.verify_performance_replication.secondary_replication_data_primaries
}
output "primary_updated_replication_data_secondaries" {
output "updated_primary_replication_status" {
description = "The Vault updated primary cluster performance replication status"
value = step.verify_updated_performance_replication.primary_replication_status
}
output "updated_known_primary_cluster_addresses" {
description = "The Vault secondary cluster performance replication status"
value = step.verify_updated_performance_replication.known_primary_cluster_addrs
}
output "updated_secondary_replication_status" {
description = "The Vault updated secondary cluster performance replication status"
value = step.verify_updated_performance_replication.secondary_replication_status
}
output "updated_primary_replication_data_secondaries" {
description = "The Vault updated primary cluster secondaries connection status"
value = step.verify_updated_performance_replication.primary_replication_data_secondaries
}
output "secondary_updated_replication_data_primaries" {
output "updated_secondary_replication_data_primaries" {
description = "The Vault updated secondary cluster primaries connection status"
value = step.verify_updated_performance_replication.secondary_replication_data_primaries
}

140
enos/enos-scenario-smoke.hcl
View File

@@ -17,6 +17,12 @@ scenario "smoke" {
edition = ["oss", "ent.fips1402", "ent.hsm.fips1402"]
artifact_type = ["package"]
}
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
}
terraform_cli = terraform_cli.default
@@ -35,12 +41,17 @@ scenario "smoke" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
}
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"]
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
packages = ["jq"]
enos_provider = {
rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu
}
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
@@ -137,11 +148,30 @@ scenario "smoke" {
}
}
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
@@ -149,26 +179,24 @@ scenario "smoke" {
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition
version = matrix.consul_version
} : null
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.backend
unseal_method = matrix.seal
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
packages = local.packages
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = matrix.seal
}
}
@@ -181,9 +209,9 @@ scenario "smoke" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -196,13 +224,13 @@ scenario "smoke" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -216,7 +244,7 @@ scenario "smoke" {
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
}
}
@@ -234,9 +262,9 @@ scenario "smoke" {
variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -251,8 +279,8 @@ scenario "smoke" {
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -267,7 +295,7 @@ scenario "smoke" {
variables {
vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
}
}
@@ -297,63 +325,63 @@ scenario "smoke" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
}
}
output "vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.create_vault_cluster.instance_ids
output "awskms_unseal_key_arn" {
description = "The Vault cluster KMS key arn"
value = step.create_vpc.kms_key_arn
}
output "vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.instance_public_ips
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "vault_cluster_priv_ips" {
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips
value = step.create_vault_cluster.private_ips
}
output "vault_cluster_key_id" {
description = "The Vault cluster Key ID"
value = step.create_vault_cluster.key_id
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "vault_cluster_root_token" {
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token
value = step.create_vault_cluster.root_token
}
output "vault_cluster_recovery_key_shares" {
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.vault_recovery_key_shares
value = step.create_vault_cluster.recovery_key_shares
}
output "vault_cluster_recovery_keys_b64" {
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.vault_recovery_keys_b64
value = step.create_vault_cluster.recovery_keys_b64
}
output "vault_cluster_recovery_keys_hex" {
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.vault_recovery_keys_hex
value = step.create_vault_cluster.recovery_keys_hex
}
output "vault_cluster_unseal_keys_b64" {
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
value = step.create_vault_cluster.unseal_keys_b64
}
output "vault_cluster_unseal_keys_hex" {
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
value = step.create_vault_cluster.unseal_keys_hex
}
}

117
enos/enos-scenario-ui.hcl
View File

@@ -113,11 +113,29 @@ scenario "ui" {
}
}
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.create_vpc.ami_ids[local.distro][local.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
@@ -125,20 +143,22 @@ scenario "ui" {
}
variables {
ami_id = step.create_vpc.ami_ids[local.distro][local.arch]
common_tags = local.tags
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.backend
unseal_method = local.seal
vault_local_artifact_path = local.bundle_path
vault_install_dir = local.vault_install_dir
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition
version = local.consul_version
} : null
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
local_artifact_path = local.bundle_path
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = local.seal
}
}
@@ -146,52 +166,72 @@ scenario "ui" {
module = module.vault_test_ui
variables {
vault_addr = step.create_vault_cluster.instance_public_ips[0]
vault_root_token = step.create_vault_cluster.vault_root_token
vault_unseal_keys = step.create_vault_cluster.vault_recovery_keys_b64
vault_recovery_threshold = step.create_vault_cluster.vault_recovery_threshold
vault_addr = step.create_vault_cluster_targets.hosts[0].public_ip
vault_root_token = step.create_vault_cluster.root_token
vault_unseal_keys = step.create_vault_cluster.recovery_keys_b64
vault_recovery_threshold = step.create_vault_cluster.recovery_threshold
ui_test_filter = local.ui_test_filter
}
}
output "vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.create_vault_cluster.instance_ids
output "awskms_unseal_key_arn" {
description = "The Vault cluster KMS key arn"
value = step.create_vpc.kms_key_arn
}
output "vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.instance_public_ips
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "vault_cluster_priv_ips" {
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips
value = step.create_vault_cluster.private_ips
}
output "vault_cluster_key_id" {
description = "The Vault cluster Key ID"
value = step.create_vault_cluster.key_id
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "vault_cluster_root_token" {
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token
value = step.create_vault_cluster.root_token
}
output "vault_cluster_unseal_keys_b64" {
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
value = step.create_vault_cluster.unseal_keys_b64
}
output "vault_cluster_unseal_keys_hex" {
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex
value = step.create_vault_cluster.unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
output "ui_test_environment" {
value = step.test_ui.ui_test_environment
description = "The environment variables that are required in order to run the test:enos yarn target"
}
output "ui_test_stderr" {
@@ -203,9 +243,4 @@ scenario "ui" {
description = "The stdout of the ui tests that ran"
value = step.test_ui.ui_test_stdout
}
output "ui_test_environment" {
value = step.test_ui.ui_test_environment
description = "The environment variables that are required in order to run the test:enos yarn target"
}
}

139
enos/enos-scenario-upgrade.hcl
View File

@@ -35,12 +35,17 @@ scenario "upgrade" {
"ent.hsm" = ["ui", "enterprise", "cgo", "hsm", "venthsm"]
"ent.hsm.fips1402" = ["ui", "enterprise", "cgo", "hsm", "fips", "fips_140_2", "ent.hsm.fips1402"]
}
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
dependencies_to_install = ["jq"]
bundle_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_bundle_path) : null
packages = ["jq"]
enos_provider = {
rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu
}
spot_price_max = {
// These prices are based on on-demand cost for t3.medium in us-east
"rhel" = "0.1016"
"ubuntu" = "0.0416"
}
tags = merge({
"Project Name" : var.project_name
"Project" : "Enos",
@@ -138,13 +143,30 @@ scenario "upgrade" {
}
}
# This step creates a Vault cluster using a bundle downloaded from
# releases.hashicorp.com, with the version specified in var.vault_autopilot_initial_release
step "create_vault_cluster_targets" {
module = module.target_ec2_spot_fleet // "target_ec2_instances" can be used for on-demand instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
common_tags = local.tags
instance_type = local.vault_instance_type // only used for on-demand instances
spot_price_max = local.spot_price_max[matrix.distro]
vpc_id = step.create_vpc.vpc_id
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
@@ -152,25 +174,24 @@ scenario "upgrade" {
}
variables {
ami_id = step.create_vpc.ami_ids[matrix.distro][matrix.arch]
common_tags = local.tags
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
awskms_unseal_key_arn = step.create_vpc.kms_key_arn
cluster_name = step.create_vault_cluster_targets.cluster_name
config_env_vars = {
VAULT_LOG_LEVEL = var.vault_log_level
}
consul_cluster_tag = step.create_backend_cluster.consul_cluster_tag
consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition
version = matrix.consul_version
} : null
dependencies_to_install = local.dependencies_to_install
instance_type = local.vault_instance_type
kms_key_arn = step.create_vpc.kms_key_arn
storage_backend = matrix.backend
unseal_method = matrix.seal
vault_install_dir = local.vault_install_dir
vault_release = var.vault_upgrade_initial_release
vault_license = matrix.edition != "oss" ? step.read_license.license : null
vpc_id = step.create_vpc.vpc_id
vault_environment = {
VAULT_LOG_LEVEL = var.vault_log_level
}
install_dir = local.vault_install_dir
license = matrix.edition != "oss" ? step.read_license.license : null
packages = local.packages
release = var.vault_upgrade_initial_release
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
unseal_method = matrix.seal
}
}
@@ -183,9 +204,9 @@ scenario "upgrade" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -203,9 +224,9 @@ scenario "upgrade" {
variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -223,11 +244,11 @@ scenario "upgrade" {
variables {
vault_api_addr = "http://localhost:8200"
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_local_artifact_path = local.bundle_path
vault_artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
vault_install_dir = local.vault_install_dir
vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.vault_unseal_keys_hex : null
vault_unseal_keys = matrix.seal == "shamir" ? step.create_vault_cluster.unseal_keys_hex : null
vault_seal_type = matrix.seal
}
}
@@ -244,13 +265,13 @@ scenario "upgrade" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -266,9 +287,9 @@ scenario "upgrade" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.vault_root_token
vault_root_token = step.create_vault_cluster.root_token
}
}
@@ -285,7 +306,7 @@ scenario "upgrade" {
}
variables {
vault_instances = step.create_vault_cluster.vault_instances
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
}
}
@@ -322,63 +343,63 @@ scenario "upgrade" {
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster.vault_instances
vault_root_token = step.create_vault_cluster.vault_root_token
vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.root_token
}
}
output "vault_cluster_instance_ids" {
description = "The Vault cluster instance IDs"
value = step.create_vault_cluster.instance_ids
output "awskms_unseal_key_arn" {
description = "The Vault cluster KMS key arn"
value = step.create_vpc.kms_key_arn
}
output "vault_cluster_pub_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.instance_public_ips
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "vault_cluster_priv_ips" {
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.instance_private_ips
value = step.create_vault_cluster.private_ips
}
output "vault_cluster_key_id" {
description = "The Vault cluster Key ID"
value = step.create_vault_cluster.key_id
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "vault_cluster_root_token" {
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.vault_root_token
value = step.create_vault_cluster.root_token
}
output "vault_cluster_recovery_key_shares" {
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.vault_recovery_key_shares
value = step.create_vault_cluster.recovery_key_shares
}
output "vault_cluster_recovery_keys_b64" {
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.vault_recovery_keys_b64
value = step.create_vault_cluster.recovery_keys_b64
}
output "vault_cluster_recovery_keys_hex" {
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.vault_recovery_keys_hex
value = step.create_vault_cluster.recovery_keys_hex
}
output "vault_cluster_unseal_keys_b64" {
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.vault_unseal_keys_b64
value = step.create_vault_cluster.unseal_keys_b64
}
output "vault_cluster_unseal_keys_hex" {
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.vault_unseal_keys_hex
}
output "vault_cluster_tag" {
description = "The Vault cluster tag"
value = step.create_vault_cluster.vault_cluster_tag
value = step.create_vault_cluster.unseal_keys_hex
}
}

171
enos/modules/target_ec2_instances/main.tf Normal file
View File

@@ -0,0 +1,171 @@
terraform {
required_providers {
# We need to specify the provider source in each module until we publish it
# to the public registry
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
version = ">= 0.3.2"
}
}
}
data "aws_vpc" "vpc" {
id = var.vpc_id
}
data "aws_subnets" "vpc" {
filter {
name = "vpc-id"
values = [var.vpc_id]
}
}
data "aws_kms_key" "kms_key" {
key_id = var.awskms_unseal_key_arn
}
data "aws_iam_policy_document" "target" {
statement {
resources = ["*"]
actions = [
"ec2:DescribeInstances",
"secretsmanager:*"
]
}
statement {
resources = [var.awskms_unseal_key_arn]
actions = [
"kms:DescribeKey",
"kms:ListKeys",
"kms:Encrypt",
"kms:Decrypt",
"kms:GenerateDataKey"
]
}
}
data "aws_iam_policy_document" "target_instance_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
data "enos_environment" "localhost" {}
resource "random_string" "cluster_name" {
length = 8
lower = true
upper = false
numeric = false
special = false
}
locals {
instances = toset([for idx in range(var.instance_count) : tostring(idx)])
cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result)
name_prefix = "${var.project_name}-${local.cluster_name}"
}
resource "aws_iam_role" "target_instance_role" {
name = "target_instance_role-${random_string.cluster_name.result}"
assume_role_policy = data.aws_iam_policy_document.target_instance_role.json
}
resource "aws_iam_instance_profile" "target" {
name = "${local.name_prefix}-target"
role = aws_iam_role.target_instance_role.name
}
resource "aws_iam_role_policy" "target" {
name = "${local.name_prefix}-target"
role = aws_iam_role.target_instance_role.id
policy = data.aws_iam_policy_document.target.json
}
resource "aws_security_group" "target" {
name = "${local.name_prefix}-target"
description = "Target instance security group"
vpc_id = var.vpc_id
# SSH traffic
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)]
}
# Vault traffic
ingress {
from_port = 8200
to_port = 8201
protocol = "tcp"
cidr_blocks = flatten([
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
formatlist("%s/32", var.ssh_allow_ips)])
}
# Consul traffic
ingress {
from_port = 8301
to_port = 8301
protocol = "tcp"
cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)]
}
ingress {
from_port = 8301
to_port = 8301
protocol = "udp"
cidr_blocks = ["${data.enos_environment.localhost.public_ip_address}/32", join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block)]
}
# Internal traffic
ingress {
from_port = 0
to_port = 0
protocol = "-1"
self = true
}
# External traffic
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = merge(
var.common_tags,
{
Name = "${local.name_prefix}-sg"
},
)
}
resource "aws_instance" "targets" {
for_each = local.instances
ami = var.ami_id
instance_type = var.instance_type
vpc_security_group_ids = [aws_security_group.target.id]
subnet_id = tolist(data.aws_subnets.vpc.ids)[each.key % length(data.aws_subnets.vpc.ids)]
key_name = var.ssh_keypair
iam_instance_profile = aws_iam_instance_profile.target.name
tags = merge(
var.common_tags,
{
Name = "${local.name_prefix}-target-instance"
Type = local.cluster_name
},
)
}

11
enos/modules/target_ec2_instances/outputs.tf Normal file
View File

@@ -0,0 +1,11 @@
output "cluster_name" {
value = local.cluster_name
}
output "hosts" {
description = "The ec2 instance target hosts"
value = { for idx in range(var.instance_count) : idx => {
public_ip = aws_instance.targets[idx].public_ip
private_ip = aws_instance.targets[idx].private_ip
} }
}

61
enos/modules/target_ec2_instances/variables.tf Normal file
View File

@@ -0,0 +1,61 @@
variable "ami_id" {
description = "The machine image identifier"
type = string
}
variable "awskms_unseal_key_arn" {
type = string
description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
default = null
}
variable "cluster_name" {
type = string
description = "A unique cluster identifier"
default = null
}
variable "common_tags" {
description = "Common tags for cloud resources"
type = map(string)
default = { "Project" : "Enos" }
}
variable "instance_count" {
description = "The number of target instances to create"
type = number
default = 3
}
variable "instance_type" {
description = "The instance machine type"
type = string
default = "t3.small"
}
variable "project_name" {
description = "A unique project name"
type = string
}
variable "spot_price_max" {
description = "Unused shim variable to match target_ec2_spot_fleet"
type = string
default = null
}
variable "ssh_allow_ips" {
description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
type = list(string)
default = []
}
variable "ssh_keypair" {
description = "SSH keypair used to connect to EC2 instances"
type = string
}
variable "vpc_id" {
description = "The identifier of the VPC where the target instances will be created"
type = string
}

388
enos/modules/target_ec2_spot_fleet/main.tf Normal file
View File

@@ -0,0 +1,388 @@
terraform {
required_providers {
# We need to specify the provider source in each module until we publish it
# to the public registry
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
version = ">= 0.3.2"
}
}
}
data "aws_vpc" "vpc" {
id = var.vpc_id
}
data "aws_subnets" "vpc" {
filter {
name = "vpc-id"
values = [var.vpc_id]
}
}
data "aws_kms_key" "kms_key" {
key_id = var.awskms_unseal_key_arn
}
data "aws_iam_policy_document" "target" {
statement {
resources = ["*"]
actions = [
"ec2:DescribeInstances",
"secretsmanager:*"
]
}
statement {
resources = [var.awskms_unseal_key_arn]
actions = [
"kms:DescribeKey",
"kms:ListKeys",
"kms:Encrypt",
"kms:Decrypt",
"kms:GenerateDataKey"
]
}
}
data "aws_iam_policy_document" "target_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
data "aws_iam_policy_document" "fleet" {
statement {
resources = ["*"]
actions = [
"ec2:DescribeImages",
"ec2:DescribeSubnets",
"ec2:RequestSpotInstances",
"ec2:TerminateInstances",
"ec2:DescribeInstanceStatus",
"ec2:CancelSpotFleetRequests",
"ec2:CreateTags",
"ec2:RunInstances",
"ec2:StartInstances",
"ec2:StopInstances",
]
}
statement {
effect = "Deny"
resources = [
"arn:aws:ec2:*:*:instance/*",
]
actions = [
"ec2:RunInstances",
]
condition {
test = "StringNotEquals"
variable = "ec2:InstanceMarketType"
values = ["spot"]
}
}
statement {
resources = ["*"]
actions = [
"iam:PassRole",
]
condition {
test = "StringEquals"
variable = "iam:PassedToService"
values = [
"ec2.amazonaws.com",
]
}
}
statement {
resources = [
"arn:aws:elasticloadbalancing:*:*:loadbalancer/*",
]
actions = [
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
]
}
statement {
resources = [
"arn:aws:elasticloadbalancing:*:*:*/*"
]
actions = [
"elasticloadbalancing:RegisterTargets"
]
}
}
data "aws_iam_policy_document" "fleet_role" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["spotfleet.amazonaws.com"]
}
}
}
data "enos_environment" "localhost" {}
resource "random_string" "cluster_name" {
length = 8
lower = true
upper = false
numeric = false
special = false
}
resource "random_string" "unique_id" {
length = 4
lower = true
upper = false
numeric = false
special = false
}
locals {
instances = toset([for idx in range(var.instance_count) : tostring(idx)])
cluster_name = coalesce(var.cluster_name, random_string.cluster_name.result)
name_prefix = "${var.project_name}-${local.cluster_name}-${random_string.unique_id.result}"
fleet_tag = "${local.name_prefix}-spot-fleet-target"
fleet_tags = {
Name = "${local.name_prefix}-target"
Type = local.cluster_name
SpotFleet = local.fleet_tag
}
}
resource "aws_iam_role" "target" {
name = "${local.name_prefix}-target-role"
assume_role_policy = data.aws_iam_policy_document.target_role.json
}
resource "aws_iam_instance_profile" "target" {
name = "${local.name_prefix}-target-profile"
role = aws_iam_role.target.name
}
resource "aws_iam_role_policy" "target" {
name = "${local.name_prefix}-target-policy"
role = aws_iam_role.target.id
policy = data.aws_iam_policy_document.target.json
}
resource "aws_iam_role" "fleet" {
name = "${local.name_prefix}-fleet-role"
assume_role_policy = data.aws_iam_policy_document.fleet_role.json
}
resource "aws_iam_role_policy" "fleet" {
name = "${local.name_prefix}-fleet-policy"
role = aws_iam_role.fleet.id
policy = data.aws_iam_policy_document.fleet.json
}
resource "aws_security_group" "target" {
name = "${local.name_prefix}-target"
description = "Target instance security group"
vpc_id = var.vpc_id
# SSH traffic
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = [
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
]
}
# Vault traffic
ingress {
from_port = 8200
to_port = 8201
protocol = "tcp"
cidr_blocks = flatten([
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
formatlist("%s/32", var.ssh_allow_ips)
])
}
# Consul traffic
ingress {
from_port = 8301
to_port = 8301
protocol = "tcp"
cidr_blocks = [
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
]
}
ingress {
from_port = 8301
to_port = 8301
protocol = "udp"
cidr_blocks = [
"${data.enos_environment.localhost.public_ip_address}/32",
join(",", data.aws_vpc.vpc.cidr_block_associations.*.cidr_block),
]
}
# Internal traffic
ingress {
from_port = 0
to_port = 0
protocol = "-1"
self = true
}
# External traffic
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
tags = merge(
var.common_tags,
{
Name = "${local.name_prefix}-sg"
},
)
}
resource "aws_launch_template" "target" {
name = "${local.name_prefix}-target"
image_id = var.ami_id
key_name = var.ssh_keypair
iam_instance_profile {
name = aws_iam_instance_profile.target.name
}
network_interfaces {
associate_public_ip_address = true
delete_on_termination = true
security_groups = [aws_security_group.target.id]
}
tag_specifications {
resource_type = "instance"
tags = merge(
var.common_tags,
local.fleet_tags,
)
}
}
# There are three primary knobs we can turn to try and optimize our costs by
# using a spot fleet: our min and max instance requirements, our max bid
# price, and the allocation strategy to use when fulfilling the spot request.
# We've currently configured our instance requirements to allow for anywhere
# from 2-4 vCPUs and 4-16GB of RAM. We intentionally have a wide range
# to allow for a large instance size pool to be considered. Our next knob is our
# max bid price. As we're using spot fleets to save on instance cost, we never
# want to pay more for an instance than we were on-demand. We've set the max price
# to equal what we pay for t3.medium instances on-demand, which are the smallest
# reliable size for Vault scenarios. The final knob is the allocation strategy
# that AWS will use when looking for instances that meet our resource and cost
# requirements. We're using the "lowestPrice" strategy to get the absolute
# cheapest machines that will fit the requirements, but it comes with a slightly
# higher capacity risk than say, "capacityOptimized" or "priceCapacityOptimized".
# Unless we see capacity issues or instances being shut down then we ought to
# stick with that strategy.
resource "aws_spot_fleet_request" "targets" {
allocation_strategy = "lowestPrice"
fleet_type = "request"
iam_fleet_role = aws_iam_role.fleet.arn
// Set this to zero so re-runs don't plan for replacement
instance_pools_to_use_count = 0
target_capacity = var.instance_count
terminate_instances_on_delete = true
wait_for_fulfillment = true
launch_template_config {
launch_template_specification {
id = aws_launch_template.target.id
version = aws_launch_template.target.latest_version
}
overrides {
spot_price = var.spot_price_max
subnet_id = data.aws_subnets.vpc.ids[0]
instance_requirements {
burstable_performance = "included"
memory_mib {
min = var.instance_mem_min
max = var.instance_mem_max
}
vcpu_count {
min = var.instance_cpu_min
max = var.instance_cpu_max
}
}
}
}
tags = merge(
var.common_tags,
local.fleet_tags,
)
}
data "aws_instances" "targets" {
depends_on = [
aws_spot_fleet_request.targets,
]
instance_tags = local.fleet_tags
instance_state_names = [
"pending",
"running",
]
filter {
name = "image-id"
values = [var.ami_id]
}
filter {
name = "iam-instance-profile.arn"
values = [aws_iam_instance_profile.target.arn]
}
}
data "aws_instance" "targets" {
depends_on = [
aws_spot_fleet_request.targets,
data.aws_instances.targets
]
for_each = local.instances
instance_id = data.aws_instances.targets.ids[each.key]
}

11
enos/modules/target_ec2_spot_fleet/outputs.tf Normal file
View File

@@ -0,0 +1,11 @@
output "cluster_name" {
value = local.cluster_name
}
output "hosts" {
description = "The spot fleet target hosts"
value = { for idx in range(var.instance_count) : idx => {
public_ip = data.aws_instance.targets[idx].public_ip
private_ip = data.aws_instance.targets[idx].private_ip
} }
}

88
enos/modules/target_ec2_spot_fleet/variables.tf Normal file
View File

@@ -0,0 +1,88 @@
variable "ami_id" {
description = "The machine image identifier"
type = string
}
variable "awskms_unseal_key_arn" {
type = string
description = "The AWSKMS key ARN if using the awskms unseal method. If specified the instances will be granted kms permissions to the key"
default = null
}
variable "cluster_name" {
type = string
description = "A unique cluster identifier"
default = null
}
variable "common_tags" {
description = "Common tags for cloud resources"
type = map(string)
default = {
Project = "Vault"
}
}
variable "instance_mem_min" {
description = "The minimum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)"
type = number
default = 4096 // ~4 GB
}
variable "instance_mem_max" {
description = "The maximum amount of memory in mebibytes for each instance in the fleet. (1 MiB = 1024 bytes)"
type = number
default = 16385 // ~16 GB
}
variable "instance_cpu_min" {
description = "The minimum number of vCPU's for each instance in the fleet"
type = number
default = 2
}
variable "instance_cpu_max" {
description = "The maximum number of vCPU's for each instance in the fleet"
type = number
default = 8 // Unlikely we'll ever get that high due to spot price bid protection
}
variable "instance_count" {
description = "The number of target instances to create"
type = number
default = 3
}
variable "instance_type" {
description = "Shim variable for target module variable compatibility that is not used. The spot fleet determines instance sizes"
type = string
default = null
}
variable "project_name" {
description = "A unique project name"
type = string
}
variable "spot_price_max" {
description = "The maximum hourly price to pay for each target instance"
type = string
// Current on-demand cost of linux t3.medium in us-east.
default = "0.0416"
}
variable "ssh_allow_ips" {
description = "Allowlisted IP addresses for SSH access to target nodes. The IP address of the machine running Enos will automatically allowlisted"
type = list(string)
default = []
}
variable "ssh_keypair" {
description = "SSH keypair used to connect to EC2 instances"
type = string
}
variable "vpc_id" {
description = "The identifier of the VPC where the target instances will be created"
type = string
}

335
enos/modules/vault_cluster/main.tf Normal file
View File

@@ -0,0 +1,335 @@
terraform {
required_providers {
# We need to specify the provider source in each module until we publish it
# to the public registry
enos = {
source = "app.terraform.io/hashicorp-qti/enos"
version = ">= 0.3.2"
}
}
}
data "enos_environment" "localhost" {}
locals {
bin_path = "${var.install_dir}/vault"
consul_bin_path = "${var.consul_install_dir}/consul"
key_shares = {
"awskms" = null
"shamir" = 5
}
key_threshold = {
"awskms" = null
"shamir" = 3
}
// In order to get Terraform to plan we have to use collections with keys
// that are known at plan time. In order for our module to work our var.target_hosts
// must be a map with known keys at plan time. Here we're creating locals
// that keep track of index values that point to our target hosts.
followers = toset(slice(local.instances, 1, length(local.instances)))
instances = [for idx in range(length(var.target_hosts)) : tostring(idx)]
leader = toset(slice(local.instances, 0, 1))
recovery_shares = {
"awskms" = 5
"shamir" = null
}
recovery_threshold = {
"awskms" = 3
"shamir" = null
}
seal = {
"awskms" = {
type = "awskms"
attributes = {
kms_key_id = var.awskms_unseal_key_arn
}
}
"shamir" = {
type = "shamir"
attributes = null
}
}
storage_config = [for idx, host in var.target_hosts : (var.storage_backend == "raft" ?
merge(
{
node_id = "${var.storage_node_prefix}_${idx}"
},
var.storage_backend_addl_config
) :
{
address = "127.0.0.1:8500"
path = "vault"
})
]
}
resource "enos_remote_exec" "install_packages" {
for_each = {
for idx, host in var.target_hosts : idx => var.target_hosts[idx]
if length(var.packages) > 0
}
content = templatefile("${path.module}/templates/install-packages.sh", {
packages = join(" ", var.packages)
})
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_bundle_install" "consul" {
for_each = {
for idx, host in var.target_hosts : idx => var.target_hosts[idx]
if var.storage_backend == "consul"
}
destination = var.consul_install_dir
release = merge(var.consul_release, { product = "consul" })
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_bundle_install" "vault" {
for_each = var.target_hosts
destination = var.install_dir
release = var.release == null ? var.release : merge({ product = "vault" }, var.release)
artifactory = var.artifactory_release
path = var.local_artifact_path
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_consul_start" "consul" {
for_each = enos_bundle_install.consul
bin_path = local.consul_bin_path
data_dir = var.consul_data_dir
config = {
data_dir = var.consul_data_dir
datacenter = "dc1"
retry_join = ["provider=aws tag_key=Type tag_value=${var.consul_cluster_tag}"]
server = false
bootstrap_expect = 0
log_level = "INFO"
log_file = var.consul_log_file
}
unit_name = "consul"
username = "consul"
transport = {
ssh = {
host = var.target_hosts[each.key].public_ip
}
}
}
resource "enos_vault_start" "leader" {
depends_on = [
enos_consul_start.consul,
enos_bundle_install.vault,
]
for_each = local.leader
bin_path = local.bin_path
config_dir = var.config_dir
environment = var.config_env_vars
config = {
api_addr = "http://${var.target_hosts[each.value].private_ip}:8200"
cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
cluster_name = var.cluster_name
listener = {
type = "tcp"
attributes = {
address = "0.0.0.0:8200"
tls_disable = "true"
}
}
storage = {
type = var.storage_backend
attributes = ({ for key, value in local.storage_config[each.key] : key => value })
}
seal = local.seal[var.unseal_method]
ui = true
}
license = var.license
manage_service = var.manage_service
username = "vault"
unit_name = "vault"
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
resource "enos_vault_start" "followers" {
depends_on = [
enos_vault_start.leader,
]
for_each = local.followers
bin_path = local.bin_path
config_dir = var.config_dir
environment = var.config_env_vars
config = {
api_addr = "http://${var.target_hosts[each.value].private_ip}:8200"
cluster_addr = "http://${var.target_hosts[each.value].private_ip}:8201"
cluster_name = var.cluster_name
listener = {
type = "tcp"
attributes = {
address = "0.0.0.0:8200"
tls_disable = "true"
}
}
storage = {
type = var.storage_backend
attributes = { for key, value in local.storage_config[each.key] : key => value }
}
seal = local.seal[var.unseal_method]
ui = true
}
license = var.license
manage_service = var.manage_service
username = "vault"
unit_name = "vault"
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
resource "enos_vault_init" "leader" {
depends_on = [
enos_vault_start.followers,
]
for_each = toset([
for idx, leader in local.leader : leader
if var.initialize_cluster
])
bin_path = local.bin_path
vault_addr = enos_vault_start.leader[0].config.api_addr
key_shares = local.key_shares[var.unseal_method]
key_threshold = local.key_threshold[var.unseal_method]
recovery_shares = local.recovery_shares[var.unseal_method]
recovery_threshold = local.recovery_threshold[var.unseal_method]
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
resource "enos_vault_unseal" "leader" {
depends_on = [
enos_vault_start.followers,
enos_vault_init.leader,
]
for_each = enos_vault_init.leader // only unseal the leader if we initialized it
bin_path = local.bin_path
vault_addr = enos_vault_start.leader[each.key].config.api_addr
seal_type = var.unseal_method
unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
transport = {
ssh = {
host = var.target_hosts[tolist(local.leader)[0]].public_ip
}
}
}
resource "enos_vault_unseal" "followers" {
depends_on = [
enos_vault_init.leader,
enos_vault_unseal.leader,
]
// Only unseal followers if we're not using an auto-unseal method and we've
// initialized the cluster
for_each = toset([
for idx, follower in local.followers : follower
if var.unseal_method == "shamir" && var.initialize_cluster
])
bin_path = local.bin_path
vault_addr = enos_vault_start.followers[each.key].config.api_addr
seal_type = var.unseal_method
unseal_keys = var.unseal_method != "shamir" ? null : coalesce(var.shamir_unseal_keys, enos_vault_init.leader[0].unseal_keys_hex)
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}
// Force unseal the cluster. This is used if the vault-cluster module is used
// to add additional nodes to a cluster via auto-pilot, or some other means.
// When that happens we'll want to set initialize_cluster to false and
// force_unseal to true.
resource "enos_vault_unseal" "maybe_force_unseal" {
depends_on = [
enos_vault_start.followers,
]
for_each = {
for idx, host in var.target_hosts : idx => host
if var.force_unseal && !var.initialize_cluster
}
bin_path = local.bin_path
vault_addr = "http://localhost:8200"
seal_type = var.unseal_method
unseal_keys = coalesce(
var.shamir_unseal_keys,
try(enos_vault_init.leader[0].unseal_keys_hex, null),
)
transport = {
ssh = {
host = each.value.public_ip
}
}
}
resource "enos_remote_exec" "vault_write_license" {
for_each = toset([
for idx, leader in local.leader : leader
if var.initialize_cluster
])
depends_on = [
enos_vault_unseal.leader,
enos_vault_unseal.maybe_force_unseal,
]
content = templatefile("${path.module}/templates/vault-write-license.sh", {
bin_path = local.bin_path,
root_token = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none")
license = coalesce(var.license, "none")
})
transport = {
ssh = {
host = var.target_hosts[each.value].public_ip
}
}
}

55
enos/modules/vault_cluster/outputs.tf Normal file
View File

@@ -0,0 +1,55 @@
output "public_ips" {
description = "Vault cluster target host public_ips"
value = [for host in var.target_hosts : host.public_ip]
}
output "private_ips" {
description = "Vault cluster target host private_ips"
value = [for host in var.target_hosts : host.private_ip]
}
output "target_hosts" {
description = "The vault cluster instances that were created"
value = var.target_hosts
}
output "root_token" {
value = coalesce(var.root_token, try(enos_vault_init.leader[0].root_token, null), "none")
}
output "unseal_keys_b64" {
value = try(enos_vault_init.leader[0].unseal_keys_b64, [])
}
output "unseal_keys_hex" {
value = try(enos_vault_init.leader[0].unseal_keys_hex, null)
}
output "unseal_shares" {
value = try(enos_vault_init.leader[0].unseal_keys_shares, -1)
}
output "unseal_threshold" {
value = try(enos_vault_init.leader[0].unseal_keys_threshold, -1)
}
output "recovery_keys_b64" {
value = try(enos_vault_init.leader[0].recovery_keys_b64, [])
}
output "recovery_keys_hex" {
value = try(enos_vault_init.leader[0].recovery_keys_hex, [])
}
output "recovery_key_shares" {
value = try(enos_vault_init.leader[0].recovery_keys_shares, -1)
}
output "recovery_threshold" {
value = try(enos_vault_init.leader[0].recovery_keys_threshold, -1)
}
output "cluster_name" {
description = "The Vault cluster name"
value = var.cluster_name
}

44
enos/modules/vault_cluster/templates/install-packages.sh Executable file
View File

@@ -0,0 +1,44 @@
#!/bin/bash
set -ex -o pipefail
packages="${packages}"
if [ "$packages" == "" ]
then
echo "No dependencies to install."
exit 0
fi
function retry {
local retries=$1
shift
local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ "$count" -lt "$retries" ]; then
sleep "$wait"
else
return "$exit"
fi
done
return 0
}
echo "Installing Dependencies: $packages"
if [ -f /etc/debian_version ]; then
# Make sure cloud-init is not modifying our sources list while we're trying
# to install.
retry 7 grep ec2 /etc/apt/sources.list
cd /tmp
retry 5 sudo apt update
retry 5 sudo apt install -y "$${packages[@]}"
else
cd /tmp
retry 7 sudo yum -y install "$${packages[@]}"
fi

38
enos/modules/vault_cluster/templates/vault-write-license.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/bin/bash
license='${license}'
if test $license = "none"; then
exit 0
fi
function retry {
local retries=$1
shift
local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ "$count" -lt "$retries" ]; then
sleep "$wait"
else
return "$exit"
fi
done
return 0
}
export VAULT_ADDR=http://localhost:8200
export VAULT_TOKEN='${root_token}'
# Temporary hack until we can make the unseal resource handle legacy license
# setting. If we're running 1.8 and above then we shouldn't try to set a license.
ver=$(${bin_path} version)
if [[ "$(echo "$ver" |awk '{print $2}' |awk -F'.' '{print $2}')" -ge 8 ]]; then
exit 0
fi
retry 5 ${bin_path} write /sys/license text="$license"

176
enos/modules/vault_cluster/variables.tf Normal file
View File

@@ -0,0 +1,176 @@
variable "artifactory_release" {
type = object({
username = string
token = string
url = string
sha256 = string
})
description = "The Artifactory release information to install Vault artifacts from Artifactory"
default = null
}
variable "awskms_unseal_key_arn" {
type = string
description = "The AWSKMS key ARN if using the awskms unseal method"
default = null
}
variable "cluster_name" {
type = string
description = "The Vault cluster name"
default = null
}
variable "config_dir" {
type = string
description = "The directory to use for Vault configuration"
default = "/etc/vault.d"
}
variable "config_env_vars" {
description = "Optional Vault configuration environment variables to set starting Vault"
type = map(string)
default = null
}
variable "consul_cluster_tag" {
type = string
description = "The retry_join tag to use for Consul"
default = null
}
variable "consul_data_dir" {
type = string
description = "The directory where the consul will store data"
default = "/opt/consul/data"
}
variable "consul_install_dir" {
type = string
description = "The directory where the consul binary will be installed"
default = "/opt/consul/bin"
}
variable "consul_log_file" {
type = string
description = "The file where the consul will write log output"
default = "/var/log/consul.log"
}
variable "consul_release" {
type = object({
version = string
edition = string
})
description = "Consul release version and edition to install from releases.hashicorp.com"
default = {
version = "1.15.1"
edition = "oss"
}
}
variable "force_unseal" {
type = bool
description = "Always unseal the Vault cluster even if we're not initializing it"
default = false
}
variable "initialize_cluster" {
type = bool
description = "Initialize the Vault cluster"
default = true
}
variable "install_dir" {
type = string
description = "The directory where the vault binary will be installed"
default = "/opt/vault/bin"
}
variable "license" {
type = string
sensitive = true
description = "The value of the Vault license"
default = null
}
variable "local_artifact_path" {
type = string
description = "The path to a locally built vault artifact to install. It can be a zip archive, RPM, or Debian package"
default = null
}
variable "manage_service" {
type = bool
description = "Manage the Vault service users and systemd unit. Disable this to use configuration in RPM and Debian packages"
default = true
}
variable "packages" {
type = list(string)
description = "A list of packages to install via the target host package manager"
default = []
}
variable "release" {
type = object({
version = string
edition = string
})
description = "Vault release version and edition to install from releases.hashicorp.com"
default = null
}
variable "root_token" {
type = string
description = "The Vault root token that we can use to intialize and configure the cluster"
default = null
}
variable "shamir_unseal_keys" {
type = list(string)
description = "Shamir unseal keys. Often only used adding additional nodes to an already initialized cluster."
default = null
}
variable "storage_backend" {
type = string
description = "The storage backend to use"
default = "raft"
validation {
condition = contains(["raft", "consul"], var.storage_backend)
error_message = "The storage_backend must be either raft or consul. No other storage backends are supported."
}
}
variable "storage_backend_addl_config" {
type = map(any)
description = "An optional set of key value pairs to inject into the storage block"
default = {}
}
variable "storage_node_prefix" {
type = string
description = "A prefix to use for each node in the Vault storage configuration"
default = "node"
}
variable "target_hosts" {
description = "The target machines host addresses to use for the Vault cluster"
type = map(object({
private_ip = string
public_ip = string
}))
}
variable "unseal_method" {
type = string
description = "The method by which to unseal the Vault cluster"
default = "awskms"
validation {
condition = contains(["awskms", "shamir"], var.unseal_method)
error_message = "The unseal_method must be either awskms or shamir. No other unseal methods are supported."
}
}